Skip to content

Commit

Permalink
feat: linux control group version 2 API support cgroup v2
Browse files Browse the repository at this point in the history
  • Loading branch information
@ChangxinDong
ChangxinDong committed Nov 8, 2022
1 parent 1b95b7d commit b8cd781
Show file tree
Hide file tree
Showing 8 changed files with 52 additions and 264 deletions.
5 changes: 2 additions & 3 deletions src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@
import com.aws.greengrass.lifecyclemanager.Kernel;
import com.aws.greengrass.testcommons.testutilities.GGExtension;
import com.aws.greengrass.testcommons.testutilities.TestUtils;
import com.aws.greengrass.util.platforms.unix.linux.Cgroup;
import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem;
import com.aws.greengrass.util.platforms.unix.linux.CGroupV1;
import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
Expand Down Expand Up @@ -116,7 +115,7 @@ void GIVEN_LifeCycleEventStreamClient_WHEN_pause_resume_component_THEN_target_se
private LinuxSystemResourceController.CgroupFreezerState getCgroupFreezerState(String serviceName)
throws IOException {
return LinuxSystemResourceController.CgroupFreezerState.valueOf(
new String(Files.readAllBytes(new Cgroup(CgroupSubSystem.Freezer).getCgroupFreezerStateFilePath(serviceName)),
new String(Files.readAllBytes(CGroupV1.Freezer.getCgroupFreezerStateFilePath(serviceName)),
StandardCharsets.UTF_8).trim());
}
}
Expand Down
87 changes: 11 additions & 76 deletions ...com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,9 @@
import com.aws.greengrass.testcommons.testutilities.GGExtension;
import com.aws.greengrass.testcommons.testutilities.NoOpPathOwnershipHandler;
import com.aws.greengrass.util.Pair;
import com.aws.greengrass.util.platforms.SystemResourceController;
import com.aws.greengrass.util.platforms.unix.linux.Cgroup;
import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem;
import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystemV2;
import com.aws.greengrass.util.platforms.unix.linux.LinuxPlatform;
import com.aws.greengrass.util.platforms.unix.linux.CGroupV1;
import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.SystemUtils;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand All @@ -40,11 +34,9 @@
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;

import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystemException;
import java.nio.file.Files;
Expand Down Expand Up @@ -95,14 +87,9 @@
class GenericExternalServiceIntegTest extends BaseITCase {
private Kernel kernel;

@Spy
LinuxPlatform linuxPlatform;

private final static String ROOT_PATH_STRING = "/systest21/fs/cgroup";
private final static String ROOT_PATH_STRING = "/sys/fs/cgroup";
private final static String GG_PATH_STRING = "greengrass";

SystemResourceController systemResourceController;

static Stream<Arguments> posixTestUserConfig() {
return Stream.of(
arguments("config_run_with_user.yaml", "nobody", "nobody"),
Expand Down Expand Up @@ -551,7 +538,7 @@ void GIVEN_posix_default_user_WHEN_runs_THEN_runs_with_default_user(String file,
String messageOnStdout = m.getMessage();
if (STDOUT.equals(m.getEventType()) && messageOnStdout != null
&& (messageOnStdout.contains("run as")
|| messageOnStdout.contains("install as") )) {
|| messageOnStdout.contains("install as") )) {
stdouts.add(messageOnStdout);
countDownLatch.countDown();
}
Expand Down Expand Up @@ -642,10 +629,9 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re
@EnabledOnOs({OS.LINUX})
@Test
void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_resource_limits_V2() throws Exception {
assumeTrue(ifCgroupV2(), "skip this test case if v1 is enabled.");

String echoComponentName = "echo_service";
String mainComponentName = "main";
String rootGGPathString = ROOT_PATH_STRING + "/" + GG_PATH_STRING;
String componentPathString = rootGGPathString + "/" + echoComponentName;
// Run with no resource limit
ConfigPlatformResolver.initKernelWithMultiPlatformConfig(kernel,
getClass().getResource("config_run_with_user.yaml"));
Expand All @@ -656,37 +642,6 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re
}
});

linuxPlatform = spy(kernel.getContext().get(LinuxPlatform.class));

createComponentData(echoComponentName);
createComponentData(mainComponentName);

// Due to cgroup v1 is active by default (in test platform), and the directories of cgroup v1 are read-only
// therefore, here create some directories and files as fake cgroup v2 files to support testing
Field controllerField = LinuxPlatform.class.getDeclaredField("CGROUP_CONTROLLERS");
setFinalStatic(controllerField, Paths.get(componentPathString + "/memory.max"));
systemResourceController = linuxPlatform.getSystemResourceController();
LinuxSystemResourceController controllerV2 = (LinuxSystemResourceController) systemResourceController;
Field memoryCgroupField = LinuxSystemResourceController.class.getSuperclass().getDeclaredField("memoryCgroup");
memoryCgroupField.setAccessible(true);
Cgroup memoryCgroup = (Cgroup) memoryCgroupField.get(controllerV2);
Field subsystem = memoryCgroup.getClass().getDeclaredField("subSystem");
subsystem.setAccessible(true);
CgroupSubSystemV2 cg = (CgroupSubSystemV2) subsystem.get(memoryCgroup);
Field f = cg.getClass().getInterfaces()[0].getDeclaredField("CGROUP_ROOT");
setFinalStatic(f, Paths.get(ROOT_PATH_STRING));

Field mountsField = LinuxSystemResourceController.class.getSuperclass().getDeclaredField("MOUNT_PATH");
mountsField.setAccessible(true);
String mountPathFile = rootGGPathString + "/mountPath.txt";
final Path mountPathFilePath = Paths.get(mountPathFile);
if (!Files.exists(mountPathFilePath)) {
Files.createFile(mountPathFilePath);
Files.write(mountPathFilePath, String.format("test1 %s test2 test3 test4 test5", ROOT_PATH_STRING).getBytes(StandardCharsets.UTF_8));

}
setFinalStatic(mountsField, mountPathFile);

kernel.launch();
assertResourceLimits_V2(10240l * 1024, 1.5);

Expand All @@ -707,14 +662,10 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re
kernel.getContext().waitForPublishQueueToClear();

assertResourceLimits_V2(10240l * 1024, 1.5);

FileUtils.deleteDirectory(Paths.get("/systest21").toFile());
}

private void setFinalStatic(Field field, Object newValue) throws Exception {
field.setAccessible(true);
FieldUtils.removeFinalModifier(field, true);
field.set(null, newValue);
private boolean ifCgroupV2() {
return Files.exists(Paths.get("/sys/fs/cgroup/cgroup.controllers"));
}

@Test
Expand Down Expand Up @@ -848,34 +799,18 @@ void GIVEN_service_starts_up_WHEN_startup_times_out_THEN_timeout_error_code_pers
}

private void assertResourceLimits(String componentName, long memory, double cpus) throws Exception {
byte[] buf1 = Files.readAllBytes(new Cgroup(CgroupSubSystem.Memory).getComponentMemoryLimitPath(componentName));
byte[] buf1 = Files.readAllBytes(CGroupV1.Memory.getComponentMemoryLimitPath(componentName));
assertThat(memory, equalTo(Long.parseLong(new String(buf1, StandardCharsets.UTF_8).trim())));

byte[] buf2 = Files.readAllBytes(new Cgroup(CgroupSubSystem.CPU).getComponentCpuQuotaPath(componentName));
byte[] buf3 = Files.readAllBytes(new Cgroup(CgroupSubSystem.CPU).getComponentCpuPeriodPath(componentName));
byte[] buf2 = Files.readAllBytes(CGroupV1.CPU.getComponentCpuQuotaPath(componentName));
byte[] buf3 = Files.readAllBytes(CGroupV1.CPU.getComponentCpuPeriodPath(componentName));

int quota = Integer.parseInt(new String(buf2, StandardCharsets.UTF_8).trim());
int period = Integer.parseInt(new String(buf3, StandardCharsets.UTF_8).trim());
int expectedQuota = (int) (cpus * period);
assertThat(expectedQuota, equalTo(quota));
}

private void createComponentData(String componentName) throws IOException {
Path path = Paths.get(ROOT_PATH_STRING).resolve(GG_PATH_STRING).resolve(componentName);
Files.createDirectories(path);

if (!Files.exists(path.resolve("memory.max"))) {
Files.createFile(path.resolve("memory.max"));
}
if (!Files.exists(path.resolve("cpu.max"))) {
Files.createFile(path.resolve("cpu.max"));
}
Files.write(path.resolve("cpu.max"), "max 100000".getBytes(StandardCharsets.UTF_8));
if (!Files.exists(path.resolve("cgroup.procs"))) {
Files.createFile(path.resolve("cgroup.procs"));
}
}

private void assertResourceLimits_V2(long memory, double cpus) throws Exception {
byte[] buf1 = Files.readAllBytes(Paths.get(String.format("%s/%s/echo_service/memory.max", ROOT_PATH_STRING, GG_PATH_STRING)));
assertThat(memory, equalTo(Long.parseLong(new String(buf1, StandardCharsets.UTF_8).trim())));
Expand Down Expand Up @@ -928,7 +863,7 @@ void GIVEN_running_service_WHEN_pause_resume_requested_THEN_pause_resume_Service
private LinuxSystemResourceController.CgroupFreezerState getCgroupFreezerState(String serviceName)
throws IOException {
return LinuxSystemResourceController.CgroupFreezerState
.valueOf(new String(Files.readAllBytes(new Cgroup(CgroupSubSystem.Freezer).getCgroupFreezerStateFilePath(serviceName))
.valueOf(new String(Files.readAllBytes(CGroupV1.Freezer.getCgroupFreezerStateFilePath(serviceName))
, StandardCharsets.UTF_8).trim());
}
}
16 changes: 11 additions & 5 deletions ...forms/unix/linux/CGroupSubSystemPath.java → ...orms/unix/linux/CGroupSubSystemPaths.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME",
justification = "CGroupSubSystemPath virtual filesystem path cannot be relative")
public interface CGroupSubSystemPath {
public interface CGroupSubSystemPaths {
Path CGROUP_ROOT = Paths.get("/sys/fs/cgroup");
String GG_NAMESPACE = "greengrass";
String CGROUP_MEMORY_LIMITS = "memory.limit_in_bytes";
Expand All @@ -45,9 +45,13 @@ default String subsystemMountCmd() {

Path getSubsystemRootPath();

Path getSubsystemGGPath();
default Path getSubsystemGGPath() {
return getSubsystemRootPath().resolve(GG_NAMESPACE);
}

Path getSubsystemComponentPath(String componentName);
default Path getSubsystemComponentPath(String componentName) {
return getSubsystemGGPath().resolve(componentName);
}

Path getComponentMemoryLimitPath(String componentName);

Expand All @@ -59,7 +63,9 @@ default Path getComponentCpuQuotaPath(String componentName) {
return null;
}

Path getCgroupProcsPath(String componentName);
default Path getCgroupProcsPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS);
}

Path getCgroupFreezerStateFilePath(String componentName);

Expand All @@ -83,7 +89,7 @@ default Path getCgroupFreezePath(String componentName) {

void handleCpuLimits(GreengrassService component, double cpu) throws IOException;

void pauseComponentProcessesCore(GreengrassService component, List<Process> processes) throws IOException;
void pauseComponentProcessesCore(GreengrassService component) throws IOException;

void resumeComponentProcesses(GreengrassService component) throws IOException;

Expand Down
23 changes: 4 additions & 19 deletions ...platforms/unix/linux/CgroupSubSystem.java → ...s/util/platforms/unix/linux/CGroupV1.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@
import java.util.Set;

@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME",
justification = "CgroupSubSystem virtual filesystem path cannot be relative")
public enum CgroupSubSystem implements CGroupSubSystemPath {
justification = "CGroupV1 virtual filesystem path cannot be relative")
public enum CGroupV1 implements CGroupSubSystemPaths {
Memory("memory", ""), CPU("cpu,cpuacct", ""), Freezer("freezer", "freezer");

private String osString;
private String mountSrc;

CgroupSubSystem(String osString, String mountSrc) {
CGroupV1(String osString, String mountSrc) {
this.osString = osString;
this.mountSrc = mountSrc;
}
Expand Down Expand Up @@ -63,16 +63,6 @@ public Path getSubsystemRootPath() {
return CGROUP_ROOT.resolve(osString);
}

@Override
public Path getSubsystemGGPath() {
return getSubsystemRootPath().resolve(GG_NAMESPACE);
}

@Override
public Path getSubsystemComponentPath(String componentName) {
return getSubsystemGGPath().resolve(componentName);
}

@Override
public Path getComponentMemoryLimitPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CGROUP_MEMORY_LIMITS);
Expand All @@ -88,11 +78,6 @@ public Path getComponentCpuQuotaPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CPU_CFS_QUOTA_US);
}

@Override
public Path getCgroupProcsPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS);
}

@Override
public Path getCgroupFreezerStateFilePath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(FREEZER_STATE_FILE);
Expand Down Expand Up @@ -134,7 +119,7 @@ public void handleCpuLimits(GreengrassService component, double cpu) throws IOEx
}

@Override
public void pauseComponentProcessesCore(GreengrassService component, List<Process> processes)
public void pauseComponentProcessesCore(GreengrassService component)
throws IOException {
if (LinuxSystemResourceController.CgroupFreezerState.FROZEN.equals(
currentFreezerCgroupState(component.getServiceName()))) {
Expand Down
22 changes: 3 additions & 19 deletions ...atforms/unix/linux/CgroupSubSystemV2.java → ...s/util/platforms/unix/linux/CGroupV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,11 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.List;
import java.util.Set;

@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME",
justification = "CgroupSubSystemV2 virtual filesystem path cannot be relative")
public enum CgroupSubSystemV2 implements CGroupSubSystemPath {
justification = "CGroupV2 virtual filesystem path cannot be relative")
public enum CGroupV2 implements CGroupSubSystemPaths {
Memory, CPU, Freezer, Unified;
private static final String CGROUP_SUBTREE_CONTROL_CONTENT = "+cpuset +cpu +io +memory +pids";

Expand All @@ -33,26 +32,11 @@ public Path getSubsystemRootPath() {
return CGROUP_ROOT;
}

@Override
public Path getSubsystemGGPath() {
return getSubsystemRootPath().resolve(GG_NAMESPACE);
}

@Override
public Path getSubsystemComponentPath(String componentName) {
return getSubsystemGGPath().resolve(componentName);
}

@Override
public Path getComponentMemoryLimitPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(MEMORY_MAX);
}

@Override
public Path getCgroupProcsPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS);
}

@Override
public Path getCgroupFreezerStateFilePath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE);
Expand Down Expand Up @@ -129,7 +113,7 @@ public void handleCpuLimits(GreengrassService component, double cpu) throws IOEx
}

@Override
public void pauseComponentProcessesCore(GreengrassService component, List<Process> processes)
public void pauseComponentProcessesCore(GreengrassService component)
throws IOException {
Files.write(getCgroupFreezerStateFilePath(component.getServiceName()),
String.valueOf(CgroupV2FreezerState.FROZEN.getIndex()).getBytes(StandardCharsets.UTF_8),
Expand Down
Loading

0 comments on commit b8cd781

Please sign in to comment.