Skip to content

Reserve memory for host #6809

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: 4.19
Choose a base branch
from
Draft

Reserve memory for host #6809

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
605b8f7
Reserve memory for host
Oct 14, 2022
71da43b
Fix unclosed curly brace
benj-n Jun 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions engine/components-api/src/main/java/com/cloud/agent/AgentManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,4 +166,6 @@ public enum TapAgentsAction {
void notifyMonitorsOfRemovedHost(long hostId, long clusterId);

void propagateChangeToAgents(Map<String, String> params);

void updateCapacityOfHosts();
}
86 changes: 80 additions & 6 deletions engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Collections;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
Expand All @@ -34,6 +35,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.LongFunction;

import javax.inject.Inject;
import javax.naming.ConfigurationException;
Expand All @@ -51,6 +53,7 @@
import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
import org.apache.cloudstack.utils.bytescale.ByteScaleUtils;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.log4j.Logger;
Expand Down Expand Up @@ -81,7 +84,11 @@
import com.cloud.agent.transport.Request;
import com.cloud.agent.transport.Response;
import com.cloud.alert.AlertManager;
import com.cloud.capacity.Capacity;
import com.cloud.capacity.dao.CapacityDao;
import com.cloud.capacity.CapacityVO;
import com.cloud.configuration.ManagementServiceConfiguration;
import com.cloud.configuration.ConfigurationManagerImpl;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.HostPodVO;
Expand Down Expand Up @@ -124,6 +131,8 @@
import com.cloud.utils.nio.Task;
import com.cloud.utils.time.InaccurateClock;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.ThreadContext;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
import org.apache.logging.log4j.ThreadContext;

import org.apache.commons.collections.CollectionUtils;

/**
* Implementation of the Agent Manager. This class controls the connection to the agents.
Expand Down Expand Up @@ -172,6 +181,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl

@Inject
protected IndirectAgentLB indirectAgentLB;
@Inject
protected CapacityDao capacityDao;

protected int _retry = 2;

Expand Down Expand Up @@ -1874,6 +1885,7 @@ public void processConnect(final Host host, final StartupCommand cmd, final bool
params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString()));
params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString()));
params.put(NetworkOrchestrationService.TUNGSTEN_ENABLED.key(), String.valueOf(NetworkOrchestrationService.TUNGSTEN_ENABLED.valueIn(host.getDataCenterId())));
params.put(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key(), String.valueOf(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.valueIn(host.getClusterId())));

try {
SetHostParamsCommand cmds = new SetHostParamsCommand(params);
Expand Down Expand Up @@ -1932,25 +1944,87 @@ protected Map<Long, List<Long>> getHostsPerZone() {
return hostsByZone;
}

private void sendCommandToAgents(Map<Long, List<Long>> hostsPerZone, Map<String, String> params) {
SetHostParamsCommand cmds = new SetHostParamsCommand(params);
for (Long zoneId : hostsPerZone.keySet()) {
List<Long> hostIds = hostsPerZone.get(zoneId);
private void sendCommandToAgents(Map<Long, List<Long>> hostsPerZone, LongFunction<Map<String, String>> paramsGenerator ) {
for (List<Long> hostIds : hostsPerZone.values()) {
for (Long hostId : hostIds) {
Answer answer = easySend(hostId, cmds);
Answer answer = easySend(hostId, new SetHostParamsCommand(paramsGenerator.apply(hostId)));
if (answer == null || !answer.getResult()) {
s_logger.error("Error sending parameters to agent " + hostId);
}
}
}
}

private long calculateAvailableMemoryOfHost(HostVO host) {
long reservedMemory = ByteScaleUtils.mebibytesToBytes(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.valueIn(host.getClusterId()));
return host.getTotalMemory() + host.getDom0MinMemory() - reservedMemory;
}

private void updateMemoriesInDb(HostVO host, long newMemoryValue) {
host.setTotalMemory(newMemoryValue);

// Update "dom0_memory" in host table
host.setDom0MinMemory(ByteScaleUtils.mebibytesToBytes(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.valueIn(host.getClusterId())));
_hostDao.update(host.getId(), host);

// Update the "total_capacity" for all hosts in op_host_capacity
CapacityVO memCap = capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_MEMORY);
memCap.setTotalCapacity(host.getTotalMemory());
capacityDao.update(memCap.getId(), memCap);
}

private boolean updateHostMemory(HostVO host) {
try {
// Update the "ram" for all hosts
long newMemoryValue = calculateAvailableMemoryOfHost(host);
if (newMemoryValue > 0) {
updateMemoriesInDb(host, newMemoryValue);
return true;
}
} catch (Exception e) {
s_logger.error("Unable to update the reserved memory capacity for host id " + host.getId() + " : " + e.getMessage());
}
return false;
}

@Override
public void propagateChangeToAgents(Map<String, String> params) {
if (params != null && ! params.isEmpty()) {
s_logger.debug("Propagating changes on host parameters to the agents");
Map<Long, List<Long>> hostsPerZone = getHostsPerZone();
sendCommandToAgents(hostsPerZone, params);
sendCommandToAgents(hostsPerZone, id -> params);
}
}

@Override
public void updateCapacityOfHosts() {
Map<Long, List<Long>> hostsByZone = new HashMap<>();
boolean allHostMemoryValuesAreValid = true;

List<HostVO> allHosts = _resourceMgr.listAllHostsInAllZonesByType(Host.Type.Routing);
if (CollectionUtils.isEmpty(allHosts)) {
return;
}

for (HostVO host : allHosts) {
boolean updateWasSuccessful = updateHostMemory(host);

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change

if (!updateWasSuccessful) {
allHostMemoryValuesAreValid = false;
continue;
}

Long zoneId = host.getDataCenterId();
List<Long> hostIds = hostsByZone.getOrDefault(zoneId, new ArrayList<>());
hostIds.add(host.getId());
hostsByZone.put(zoneId, hostIds);
}

if (allHostMemoryValuesAreValid) {
sendCommandToAgents(hostsByZone,
hostId -> Collections.singletonMap(
ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key(),
ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.valueIn(_hostDao.findById(hostId).getClusterId()).toString()));
}
}

Expand Down
8 changes: 8 additions & 0 deletions engine/schema/src/main/java/com/cloud/host/HostVO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -782,4 +782,12 @@ public boolean checkHostServiceOfferingTags(ServiceOffering serviceOffering){
public PartitionType partitionType() {
return PartitionType.Host;
}

public long getDom0MinMemory() {
return dom0MinMemory;
}

public void setDom0MinMemory(long dom0MinMemory) {
this.dom0MinMemory = dom0MinMemory;
}
}
28 changes: 26 additions & 2 deletions ...rvisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@
import com.cloud.agent.resource.virtualnetwork.VirtualRouterDeployer;
import com.cloud.agent.resource.virtualnetwork.VirtualRoutingResource;
import com.cloud.configuration.Config;
import com.cloud.configuration.ConfigurationManagerImpl;
import com.cloud.dc.Vlan;
import com.cloud.exception.InternalErrorException;
import com.cloud.host.Host.Type;
Expand Down Expand Up @@ -1111,7 +1112,17 @@ public boolean configure(final String name, final Map<String, Object> params) th
videoRam = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_VIDEO_RAM);

// Reserve 1GB unless admin overrides
dom0MinMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_MEM_MB));
long reservedMemory = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_MEM_MB);

value = (String)params.get(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key());
Copy link
Member

@weizhouapache weizhouapache Jan 19, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

these lines (997 to 1002) are not needed I think.

@GutoVeronezi
can you confirm ?

Copy link
Member

@weizhouapache weizhouapache Jan 19, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@soreana
if the value in agent.properties is 0 or less than 0, use the default (1024) ?

long clusterReservedMemoryValue = NumbersUtil.parseInt(value, 1024);

if (clusterReservedMemoryValue != 1024) {
reservedMemory = clusterReservedMemoryValue;
}

dom0MinMem = ByteScaleUtils.mebibytesToBytes(reservedMemory);


dom0MinCpuCores = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_CPU_CORE_COUNT);

Expand Down Expand Up @@ -1453,6 +1464,15 @@ private void validateLocalStorageUUID(String localStorageUUID) throws Configurat
}
}

private void updateDom0MinMem(PropertiesStorage storage, final Map<String, String> params) {
long value = Long.parseLong(params.get(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key()));
s_logger.info("Reserved memory for host is " + value + "MB");
dom0MinMem = ByteScaleUtils.mebibytesToBytes(value);
if (!StringUtils.isEmpty(String.valueOf(value))) {
storage.persist(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key(), String.valueOf(value));
}
}

public boolean configureHostParams(final Map<String, String> params) {
final File file = PropertiesUtil.findConfigFile("agent.properties");
if (file == null) {
Expand All @@ -1468,7 +1488,7 @@ public boolean configureHostParams(final Map<String, String> params) {
}

if (params.get(Config.MigrateWait.toString()) != null) {
String value = (String)params.get(Config.MigrateWait.toString());
String value = (String) params.get(Config.MigrateWait.toString());
Integer intValue = NumbersUtil.parseInt(value, -1);
storage.persist("vm.migrate.wait", String.valueOf(intValue));
migrateWait = intValue;
Expand All @@ -1478,6 +1498,10 @@ public boolean configureHostParams(final Map<String, String> params) {
isTungstenEnabled = Boolean.parseBoolean(params.get(NetworkOrchestrationService.TUNGSTEN_ENABLED.key()));
}

if (params.get(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key()) != null) {
updateDom0MinMem(storage, params);
}

return true;
}

Expand Down
38 changes: 19 additions & 19 deletions server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,6 @@
import org.apache.cloudstack.framework.config.impl.ConfigurationSubGroupVO;
import org.apache.cloudstack.framework.config.impl.ConfigurationVO;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.MessageSubscriber;
import org.apache.cloudstack.framework.messagebus.PublishScope;
import org.apache.cloudstack.query.QueryService;
import org.apache.cloudstack.region.PortableIp;
Expand Down Expand Up @@ -493,6 +492,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
public static final ConfigKey<Boolean> ENABLE_DOMAIN_SETTINGS_FOR_CHILD_DOMAIN = new ConfigKey<Boolean>(Boolean.class, "enable.domain.settings.for.child.domain", "Advanced", "false",
"Indicates whether the settings of parent domain should be applied for child domain. If true, the child domain will get value from parent domain if its not configured in child domain else global value is taken.",
true, ConfigKey.Scope.Global, null);
public static final ConfigKey<Integer> HOST_RESERVED_MEM_MB = new ConfigKey<>("Advanced", Integer.class, "host.reserved.mem.mb", "1024",
Copy link
Contributor

@sureshanaparti sureshanaparti Jun 25, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
public static final ConfigKey<Integer> HOST_RESERVED_MEM_MB = new ConfigKey<>("Advanced", Integer.class, "host.reserved.mem.mb", "1024",
public static final ConfigKey<Integer> HOST_RESERVED_MEMORY_MB = new ConfigKey<>("Advanced", Integer.class, "host.reserved.memory.mb", "1024",

"Set an upper limit for memory in megabytes which will be reserved for host and not used for VM allocation.", true, ConfigKey.Scope.Cluster);

public static ConfigKey<Integer> VM_SERVICE_OFFERING_MAX_CPU_CORES = new ConfigKey<Integer>("Advanced", Integer.class, "vm.serviceoffering.cpu.cores.max", "0", "Maximum CPU cores "
+ "for vm service offering. If 0 - no limitation", true);
Expand Down Expand Up @@ -595,23 +596,22 @@ private void overProvisioningFactorsForValidation() {
}

private void initMessageBusListener() {
messageBus.subscribe(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, new MessageSubscriber() {
@Override
public void onPublishMessage(String serderAddress, String subject, Object args) {
String globalSettingUpdated = (String) args;
if (StringUtils.isEmpty(globalSettingUpdated)) {
return;
}
if (globalSettingUpdated.equals(ApiServiceConfiguration.ManagementServerAddresses.key()) ||
globalSettingUpdated.equals(IndirectAgentLBServiceImpl.IndirectAgentLBAlgorithm.key())) {
_indirectAgentLB.propagateMSListToAgents();
} else if (globalSettingUpdated.equals(Config.RouterAggregationCommandEachTimeout.toString())
|| globalSettingUpdated.equals(Config.MigrateWait.toString())) {
Map<String, String> params = new HashMap<String, String>();
params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString()));
params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString()));
_agentManager.propagateChangeToAgents(params);
}
messageBus.subscribe(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, (serverAddress, subject, args) -> {
String globalSettingUpdated = (String) args;
if (StringUtils.isEmpty(globalSettingUpdated)) {
return;
}
if (globalSettingUpdated.equals(ApiServiceConfiguration.ManagementServerAddresses.key()) ||
globalSettingUpdated.equals(IndirectAgentLBServiceImpl.IndirectAgentLBAlgorithm.key())) {
_indirectAgentLB.propagateMSListToAgents();
} else if (globalSettingUpdated.equals(Config.RouterAggregationCommandEachTimeout.toString())
|| globalSettingUpdated.equals(Config.MigrateWait.toString())) {
Map<String, String> params = new HashMap<String, String>();
params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString()));
params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString()));
_agentManager.propagateChangeToAgents(params);
} else if (globalSettingUpdated.equalsIgnoreCase(HOST_RESERVED_MEM_MB.key())) {
_agentManager.updateCapacityOfHosts();
}
});
}
Expand Down Expand Up @@ -7809,7 +7809,7 @@ public ConfigKey<?>[] getConfigKeys() {
BYTES_MAX_READ_LENGTH, BYTES_MAX_WRITE_LENGTH, ADD_HOST_ON_SERVICE_RESTART_KVM, SET_HOST_DOWN_TO_MAINTENANCE,
VM_SERVICE_OFFERING_MAX_CPU_CORES, VM_SERVICE_OFFERING_MAX_RAM_SIZE, MIGRATE_VM_ACROSS_CLUSTERS,
ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN, ENABLE_DOMAIN_SETTINGS_FOR_CHILD_DOMAIN,
ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS, DELETE_QUERY_BATCH_SIZE
ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS, DELETE_QUERY_BATCH_SIZE, HOST_RESERVED_MEM_MB
};
}

Expand Down
7 changes: 7 additions & 0 deletions server/src/main/java/com/cloud/resource/ResourceManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.utils.bytescale.ByteScaleUtils;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.ObjectUtils;
Expand Down Expand Up @@ -100,6 +101,7 @@
import com.cloud.cluster.ClusterManager;
import com.cloud.configuration.Config;
import com.cloud.configuration.ConfigurationManager;
import com.cloud.configuration.ConfigurationManagerImpl;
import com.cloud.dc.ClusterDetailsDao;
import com.cloud.dc.ClusterDetailsVO;
import com.cloud.dc.ClusterVO;
Expand Down Expand Up @@ -2350,6 +2352,11 @@ protected HostVO createHostVO(final StartupCommand[] cmds, final ServerResource
hostTags = implicitHostTags;
}
}
// Update host memory reported by agent
if (ssCmd.getHypervisorType().equals(HypervisorType.KVM) ||
ssCmd.getHypervisorType().equals(HypervisorType.LXC)) {
host.setDom0MinMemory(ByteScaleUtils.mebibytesToBytes(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.valueIn(host.getClusterId())));
}
}

host.setDataCenterId(dc.getId());
Expand Down
Loading