From 1346100704fd9377af1c44abfe5c596371cba645 Mon Sep 17 00:00:00 2001 From: coderzc Date: Thu, 28 Jul 2022 11:19:04 +0800 Subject: [PATCH 1/8] Support cgroupV2 for by using `jdk.internal.platform.Metrics` in Pulsar Loadbalancer --- pom.xml | 1 + .../broker/loadbalance/LinuxInfoUtils.java | 40 +++++++++--------- .../loadbalance/SimpleBrokerStartTest.java | 41 +++++++++++++++++++ 3 files changed, 61 insertions(+), 21 deletions(-) diff --git a/pom.xml b/pom.xml index ebac8ee8129ad..e956c06b957d9 100644 --- a/pom.xml +++ b/pom.xml @@ -98,6 +98,7 @@ flexible messaging model and an intuitive client API. --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.management/sun.management=ALL-UNNAMED + --add-opens java.base/jdk.internal.platform=ALL-UNNAMED true 4 diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java index c23b3e8d2a69c..d5c1dabf68c3e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java @@ -29,6 +29,8 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; +import jdk.internal.platform.Container; +import jdk.internal.platform.Metrics; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -40,10 +42,6 @@ @Slf4j public class LinuxInfoUtils { - // CGROUP - private static final String CGROUPS_CPU_USAGE_PATH = "/sys/fs/cgroup/cpu/cpuacct.usage"; - private static final String CGROUPS_CPU_LIMIT_QUOTA_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"; - private static final String CGROUPS_CPU_LIMIT_PERIOD_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_period_us"; // proc states private static final String PROC_STAT_PATH = "/proc/stat"; private static final String NIC_PATH = "/sys/class/net/"; @@ -51,6 +49,11 @@ public class LinuxInfoUtils { private static final int ARPHRD_ETHER = 1; private static final String NIC_SPEED_TEMPLATE = "/sys/class/net/%s/speed"; + private static final Metrics METRICS; + + static { + METRICS = Container.metrics(); + } /** * Determine whether the OS is the linux kernel. @@ -65,9 +68,14 @@ public static boolean isLinux() { */ public static boolean isCGroupEnabled() { try { - return Files.exists(Paths.get(CGROUPS_CPU_USAGE_PATH)); + if (METRICS == null) { + return false; + } + String provider = METRICS.getProvider(); + log.info("[LinuxInfo] The system metrics provider is: {}", provider); + return provider.contains("cgroup"); } catch (Exception e) { - log.warn("[LinuxInfo] Failed to check cgroup CPU usage file: {}", e.getMessage()); + log.warn("[LinuxInfo] Failed to check cgroup CPU: {}", e.getMessage()); return false; } } @@ -79,15 +87,10 @@ public static boolean isCGroupEnabled() { */ public static double getTotalCpuLimit(boolean isCGroupsEnabled) { if (isCGroupsEnabled) { - try { - long quota = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_QUOTA_PATH)); - long period = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_PERIOD_PATH)); - if (quota > 0) { - return 100.0 * quota / period; - } - } catch (IOException e) { - log.warn("[LinuxInfo] Failed to read CPU quotas from cgroups", e); - // Fallback to availableProcessors + long quota = METRICS.getCpuQuota(); + long period = METRICS.getCpuPeriod(); + if (quota > 0) { + return 100.0 * quota / period; } } // Fallback to JVM reported CPU quota @@ -99,12 +102,7 @@ public static double getTotalCpuLimit(boolean isCGroupsEnabled) { * @return Cpu usage */ public static double getCpuUsageForCGroup() { - try { - return readLongFromFile(Paths.get(CGROUPS_CPU_USAGE_PATH)); - } catch (IOException e) { - log.error("[LinuxInfo] Failed to read CPU usage from {}", CGROUPS_CPU_USAGE_PATH, e); - return -1; - } + return METRICS.getCpuUsage(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java index c01aaec666b89..143c431e8728a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java @@ -21,14 +21,21 @@ import static org.mockito.Mockito.spy; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import com.google.common.base.Charsets; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Optional; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.SystemUtils; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.testng.Assert; import org.testng.annotations.Test; @Slf4j @@ -97,4 +104,38 @@ public void testNoNICSpeed() throws Exception { } + @Test + public void testCGroupMetrics() throws IOException { + if (!SystemUtils.IS_OS_LINUX) { + return; + } + + boolean existsCGroup = Files.exists(Paths.get("/sys/fs/cgroup/cpu/cpuacct.usage")); + boolean cGroupEnabled = LinuxInfoUtils.isCGroupEnabled(); + Assert.assertEquals(cGroupEnabled, existsCGroup); + + double totalCpuLimit = LinuxInfoUtils.getTotalCpuLimit(cGroupEnabled); + double expectTotalCpuLimit = getTotalCpuLimit(cGroupEnabled); + Assert.assertEquals(totalCpuLimit, expectTotalCpuLimit); + + if (cGroupEnabled) { + double cpuUsageForCGroup = LinuxInfoUtils.getCpuUsageForCGroup(); + log.info("cpuUsageForCGroup: {}", cpuUsageForCGroup); + } + } + + public static double getTotalCpuLimit(boolean isCGroupsEnabled) throws IOException { + if (isCGroupsEnabled) { + long quota = Long.parseLong( + Files.readString(Path.of("/sys/fs/cgroup/cpu/cpu.cfs_quota_us"), Charsets.UTF_8).trim()); + long period = Long.parseLong( + Files.readString(Path.of("/sys/fs/cgroup/cpu/cpu.cfs_period_us"), Charsets.UTF_8).trim()); + if (quota > 0) { + return 100.0 * quota / period; + } + } + // Fallback to JVM reported CPU quota + return 100 * Runtime.getRuntime().availableProcessors(); + } + } From 10b4d9b62b8d4d0d2dd544591be890b4f89fdd37 Mon Sep 17 00:00:00 2001 From: coderzc Date: Thu, 28 Jul 2022 12:23:12 +0800 Subject: [PATCH 2/8] exports `jdk.internal.platform` --- bin/pulsar | 2 ++ buildtools/pom.xml | 1 + pom.xml | 3 +++ 3 files changed, 6 insertions(+) diff --git a/bin/pulsar b/bin/pulsar index d56376a0bf6e3..8c0b3e86297bf 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -291,6 +291,8 @@ if [[ -z "$IS_JAVA_8" ]]; then # netty.DnsResolverUtil # JvmDefaultGCMetricsLogger OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.management/sun.management=ALL-UNNAMED" + # LinuxInfoUtils + OPTS="$OPTS --add-opens java.base/jdk.internal.platform=ALL-UNNAMED" fi OPTS="-cp $PULSAR_CLASSPATH $OPTS" diff --git a/buildtools/pom.xml b/buildtools/pom.xml index ccf2caaf4948e..32f9bca9a0321 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -55,6 +55,7 @@ --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/jdk.internal.platform=ALL-UNNAMED diff --git a/pom.xml b/pom.xml index e956c06b957d9..9e0b82f866fb9 100644 --- a/pom.xml +++ b/pom.xml @@ -1772,6 +1772,9 @@ flexible messaging model and an intuitive client API. ${maven-compiler-plugin.version} ${pulsar.broker.compiler.release} + + --add-opens java.base/jdk.internal.platform=ALL-UNNAMED + From 38bc31cc9904f6708fe2a40c90f41a12f7d0c7e8 Mon Sep 17 00:00:00 2001 From: coderzc Date: Thu, 28 Jul 2022 16:10:53 +0800 Subject: [PATCH 3/8] Use reflection to get metrics --- pom.xml | 3 -- .../broker/loadbalance/LinuxInfoUtils.java | 44 ++++++++++++++----- .../impl/LinuxBrokerHostUsageImpl.java | 2 +- .../loadbalance/SimpleBrokerStartTest.java | 28 +++--------- 4 files changed, 41 insertions(+), 36 deletions(-) diff --git a/pom.xml b/pom.xml index 9e0b82f866fb9..e956c06b957d9 100644 --- a/pom.xml +++ b/pom.xml @@ -1772,9 +1772,6 @@ flexible messaging model and an intuitive client API. ${maven-compiler-plugin.version} ${pulsar.broker.compiler.release} - - --add-opens java.base/jdk.internal.platform=ALL-UNNAMED - diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java index d5c1dabf68c3e..17c826787cec6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java @@ -20,6 +20,7 @@ import com.google.common.base.Charsets; import java.io.IOException; +import java.lang.reflect.Method; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -29,11 +30,10 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; -import jdk.internal.platform.Container; -import jdk.internal.platform.Metrics; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.SystemUtils; @@ -49,10 +49,29 @@ public class LinuxInfoUtils { private static final int ARPHRD_ETHER = 1; private static final String NIC_SPEED_TEMPLATE = "/sys/class/net/%s/speed"; - private static final Metrics METRICS; + private static Object /*jdk.internal.platform.Metrics*/ metrics; + private static Method getMetricsProviderMethod; + private static Method getCpuQuotaMethod; + private static Method getCpuPeriodMethod; + private static Method getCpuUsageMethod; static { - METRICS = Container.metrics(); + try { + metrics = Class.forName("jdk.internal.platform.Container").getMethod("metrics") + .invoke(null); + if (metrics != null) { + getMetricsProviderMethod = metrics.getClass().getMethod("getProvider"); + getMetricsProviderMethod.setAccessible(true); + getCpuQuotaMethod = metrics.getClass().getMethod("getCpuQuota"); + getCpuQuotaMethod.setAccessible(true); + getCpuPeriodMethod = metrics.getClass().getMethod("getCpuPeriod"); + getCpuPeriodMethod.setAccessible(true); + getCpuUsageMethod = metrics.getClass().getMethod("getCpuUsage"); + getCpuUsageMethod.setAccessible(true); + } + } catch (Throwable e) { + log.warn("Failed to get runtime metrics", e); + } } /** @@ -68,10 +87,10 @@ public static boolean isLinux() { */ public static boolean isCGroupEnabled() { try { - if (METRICS == null) { + if (metrics == null) { return false; } - String provider = METRICS.getProvider(); + String provider = (String) getMetricsProviderMethod.invoke(metrics); log.info("[LinuxInfo] The system metrics provider is: {}", provider); return provider.contains("cgroup"); } catch (Exception e) { @@ -85,10 +104,11 @@ public static boolean isCGroupEnabled() { * @param isCGroupsEnabled Whether CGroup is enabled * @return Total cpu limit */ + @SneakyThrows public static double getTotalCpuLimit(boolean isCGroupsEnabled) { if (isCGroupsEnabled) { - long quota = METRICS.getCpuQuota(); - long period = METRICS.getCpuPeriod(); + long quota = (long) getCpuQuotaMethod.invoke(metrics); + long period = (long) getCpuPeriodMethod.invoke(metrics); if (quota > 0) { return 100.0 * quota / period; } @@ -101,8 +121,12 @@ public static double getTotalCpuLimit(boolean isCGroupsEnabled) { * Get CGroup cpu usage. * @return Cpu usage */ - public static double getCpuUsageForCGroup() { - return METRICS.getCpuUsage(); + @SneakyThrows + public static long getCpuUsageForCGroup() { + if (getCpuUsageMethod == null) { + return -1; + } + return (long) getCpuUsageMethod.invoke(metrics); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java index b605c523ddc40..e7aeaa6156f21 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java @@ -137,7 +137,7 @@ private double getTotalCpuUsage(double elapsedTimeSeconds) { } private double getTotalCpuUsageForCGroup(double elapsedTimeSeconds) { - double usage = getCpuUsageForCGroup(); + double usage = (double) getCpuUsageForCGroup(); double currentUsage = usage - lastCpuUsage; lastCpuUsage = usage; return 100 * currentUsage / elapsedTimeSeconds / TimeUnit.SECONDS.toNanos(1); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java index 143c431e8728a..9f108409c8ade 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java @@ -21,15 +21,12 @@ import static org.mockito.Mockito.spy; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.google.common.base.Charsets; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.SystemUtils; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -106,36 +103,23 @@ public void testNoNICSpeed() throws Exception { @Test public void testCGroupMetrics() throws IOException { - if (!SystemUtils.IS_OS_LINUX) { + if (!LinuxInfoUtils.isLinux()) { return; } - boolean existsCGroup = Files.exists(Paths.get("/sys/fs/cgroup/cpu/cpuacct.usage")); + boolean existsCGroup = Files.exists(Paths.get("/sys/fs/cgroup")); boolean cGroupEnabled = LinuxInfoUtils.isCGroupEnabled(); Assert.assertEquals(cGroupEnabled, existsCGroup); double totalCpuLimit = LinuxInfoUtils.getTotalCpuLimit(cGroupEnabled); - double expectTotalCpuLimit = getTotalCpuLimit(cGroupEnabled); - Assert.assertEquals(totalCpuLimit, expectTotalCpuLimit); + log.info("totalCpuLimit: {}", totalCpuLimit); + Assert.assertTrue(totalCpuLimit > 0.0); if (cGroupEnabled) { - double cpuUsageForCGroup = LinuxInfoUtils.getCpuUsageForCGroup(); + long cpuUsageForCGroup = LinuxInfoUtils.getCpuUsageForCGroup(); log.info("cpuUsageForCGroup: {}", cpuUsageForCGroup); + Assert.assertTrue(cpuUsageForCGroup > 0); } } - public static double getTotalCpuLimit(boolean isCGroupsEnabled) throws IOException { - if (isCGroupsEnabled) { - long quota = Long.parseLong( - Files.readString(Path.of("/sys/fs/cgroup/cpu/cpu.cfs_quota_us"), Charsets.UTF_8).trim()); - long period = Long.parseLong( - Files.readString(Path.of("/sys/fs/cgroup/cpu/cpu.cfs_period_us"), Charsets.UTF_8).trim()); - if (quota > 0) { - return 100.0 * quota / period; - } - } - // Fallback to JVM reported CPU quota - return 100 * Runtime.getRuntime().availableProcessors(); - } - } From c58105a5b4e20438bb6b529897267606186a6fb3 Mon Sep 17 00:00:00 2001 From: coderzc Date: Wed, 3 Aug 2022 00:02:53 +0800 Subject: [PATCH 4/8] fall back get metrics by path --- .../broker/loadbalance/LinuxInfoUtils.java | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java index 17c826787cec6..5ef6549cbb209 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java @@ -33,7 +33,6 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.SystemUtils; @@ -42,6 +41,11 @@ @Slf4j public class LinuxInfoUtils { + // CGROUP + private static final String CGROUPS_CPU_USAGE_PATH = "/sys/fs/cgroup/cpu/cpuacct.usage"; + private static final String CGROUPS_CPU_LIMIT_QUOTA_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"; + private static final String CGROUPS_CPU_LIMIT_PERIOD_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_period_us"; + // proc states private static final String PROC_STAT_PATH = "/proc/stat"; private static final String NIC_PATH = "/sys/class/net/"; @@ -88,7 +92,7 @@ public static boolean isLinux() { public static boolean isCGroupEnabled() { try { if (metrics == null) { - return false; + return Files.exists(Paths.get(CGROUPS_CPU_USAGE_PATH)); } String provider = (String) getMetricsProviderMethod.invoke(metrics); log.info("[LinuxInfo] The system metrics provider is: {}", provider); @@ -104,13 +108,25 @@ public static boolean isCGroupEnabled() { * @param isCGroupsEnabled Whether CGroup is enabled * @return Total cpu limit */ - @SneakyThrows public static double getTotalCpuLimit(boolean isCGroupsEnabled) { if (isCGroupsEnabled) { - long quota = (long) getCpuQuotaMethod.invoke(metrics); - long period = (long) getCpuPeriodMethod.invoke(metrics); - if (quota > 0) { - return 100.0 * quota / period; + try { + long quota; + long period; + if (metrics != null && getCpuQuotaMethod != null && getCpuPeriodMethod != null) { + quota = (long) getCpuQuotaMethod.invoke(metrics); + period = (long) getCpuPeriodMethod.invoke(metrics); + } else { + quota = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_QUOTA_PATH)); + period = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_PERIOD_PATH)); + } + + if (quota > 0) { + return 100.0 * quota / period; + } + } catch (Exception e) { + log.warn("[LinuxInfo] Failed to read CPU quotas from cgroup", e); + // Fallback to availableProcessors } } // Fallback to JVM reported CPU quota @@ -121,12 +137,16 @@ public static double getTotalCpuLimit(boolean isCGroupsEnabled) { * Get CGroup cpu usage. * @return Cpu usage */ - @SneakyThrows public static long getCpuUsageForCGroup() { - if (getCpuUsageMethod == null) { - return -1; + try { + if (metrics != null && getCpuUsageMethod != null) { + return (long) getCpuUsageMethod.invoke(metrics); + } + return readLongFromFile(Paths.get(CGROUPS_CPU_USAGE_PATH)); + } catch (Exception e) { + log.error("[LinuxInfo] Failed to read CPU usage from cgroup", e); + return -1; } - return (long) getCpuUsageMethod.invoke(metrics); } From 70b721d9f57b02eda5524d91e7e241ebe027be03 Mon Sep 17 00:00:00 2001 From: coderzc Date: Mon, 8 Aug 2022 20:30:23 +0800 Subject: [PATCH 5/8] Make sure really uses the jdk.internal.platform.Metrics --- .../broker/loadbalance/SimpleBrokerStartTest.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java index 9f108409c8ade..ed38c2bb3d0f0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java @@ -21,12 +21,12 @@ import static org.mockito.Mockito.spy; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Optional; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -102,7 +102,7 @@ public void testNoNICSpeed() throws Exception { @Test - public void testCGroupMetrics() throws IOException { + public void testCGroupMetrics() throws IllegalAccessException { if (!LinuxInfoUtils.isLinux()) { return; } @@ -116,6 +116,12 @@ public void testCGroupMetrics() throws IOException { Assert.assertTrue(totalCpuLimit > 0.0); if (cGroupEnabled) { + Assert.assertNotNull(FieldUtils.readStaticField(LinuxInfoUtils.class, "metrics", true)); + Assert.assertNotNull(FieldUtils.readStaticField(LinuxInfoUtils.class, "getMetricsProviderMethod", true)); + Assert.assertNotNull(FieldUtils.readStaticField(LinuxInfoUtils.class, "getCpuQuotaMethod", true)); + Assert.assertNotNull(FieldUtils.readStaticField(LinuxInfoUtils.class, "getCpuPeriodMethod", true)); + Assert.assertNotNull(FieldUtils.readStaticField(LinuxInfoUtils.class, "getCpuUsageMethod", true)); + long cpuUsageForCGroup = LinuxInfoUtils.getCpuUsageForCGroup(); log.info("cpuUsageForCGroup: {}", cpuUsageForCGroup); Assert.assertTrue(cpuUsageForCGroup > 0); From 39062b8026d5557496fbacb68b3ea9a061a08db0 Mon Sep 17 00:00:00 2001 From: coderzc Date: Mon, 15 Aug 2022 10:53:21 +0800 Subject: [PATCH 6/8] Use `@VisibleForTesting` instead of reflection. --- .../apache/pulsar/broker/loadbalance/LinuxInfoUtils.java | 4 +++- .../pulsar/broker/loadbalance/SimpleBrokerStartTest.java | 9 ++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java index 5ef6549cbb209..a2dbf8f98104a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Charsets; import java.io.IOException; import java.lang.reflect.Method; @@ -53,7 +54,8 @@ public class LinuxInfoUtils { private static final int ARPHRD_ETHER = 1; private static final String NIC_SPEED_TEMPLATE = "/sys/class/net/%s/speed"; - private static Object /*jdk.internal.platform.Metrics*/ metrics; + @VisibleForTesting + public static Object /*jdk.internal.platform.Metrics*/ metrics; private static Method getMetricsProviderMethod; private static Method getCpuQuotaMethod; private static Method getCpuPeriodMethod; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java index ed38c2bb3d0f0..2b0251abb5869 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java @@ -26,7 +26,6 @@ import java.util.Optional; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -102,7 +101,7 @@ public void testNoNICSpeed() throws Exception { @Test - public void testCGroupMetrics() throws IllegalAccessException { + public void testCGroupMetrics() { if (!LinuxInfoUtils.isLinux()) { return; } @@ -116,11 +115,7 @@ public void testCGroupMetrics() throws IllegalAccessException { Assert.assertTrue(totalCpuLimit > 0.0); if (cGroupEnabled) { - Assert.assertNotNull(FieldUtils.readStaticField(LinuxInfoUtils.class, "metrics", true)); - Assert.assertNotNull(FieldUtils.readStaticField(LinuxInfoUtils.class, "getMetricsProviderMethod", true)); - Assert.assertNotNull(FieldUtils.readStaticField(LinuxInfoUtils.class, "getCpuQuotaMethod", true)); - Assert.assertNotNull(FieldUtils.readStaticField(LinuxInfoUtils.class, "getCpuPeriodMethod", true)); - Assert.assertNotNull(FieldUtils.readStaticField(LinuxInfoUtils.class, "getCpuUsageMethod", true)); + Assert.assertNotNull(LinuxInfoUtils.metrics); long cpuUsageForCGroup = LinuxInfoUtils.getCpuUsageForCGroup(); log.info("cpuUsageForCGroup: {}", cpuUsageForCGroup); From 09297edbdd79c96c184776eb666ad95f88fb4279 Mon Sep 17 00:00:00 2001 From: coderzc Date: Mon, 15 Aug 2022 12:05:01 +0800 Subject: [PATCH 7/8] improve somme comment --- .../apache/pulsar/broker/loadbalance/LinuxInfoUtils.java | 8 ++++++-- .../pulsar/broker/loadbalance/SimpleBrokerStartTest.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java index a2dbf8f98104a..a6f48cc2d39ef 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java @@ -54,8 +54,7 @@ public class LinuxInfoUtils { private static final int ARPHRD_ETHER = 1; private static final String NIC_SPEED_TEMPLATE = "/sys/class/net/%s/speed"; - @VisibleForTesting - public static Object /*jdk.internal.platform.Metrics*/ metrics; + private static Object /*jdk.internal.platform.Metrics*/ metrics; private static Method getMetricsProviderMethod; private static Method getCpuQuotaMethod; private static Method getCpuPeriodMethod; @@ -286,6 +285,11 @@ private static double readDoubleFromFile(Path path) throws IOException { return Double.parseDouble(readTrimStringFromFile(path)); } + @VisibleForTesting + public static Object getMetrics() { + return metrics; + } + @AllArgsConstructor public enum NICUsageType { // transport diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java index 2b0251abb5869..aaf7eb8725861 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java @@ -115,7 +115,7 @@ public void testCGroupMetrics() { Assert.assertTrue(totalCpuLimit > 0.0); if (cGroupEnabled) { - Assert.assertNotNull(LinuxInfoUtils.metrics); + Assert.assertNotNull(LinuxInfoUtils.getMetrics()); long cpuUsageForCGroup = LinuxInfoUtils.getCpuUsageForCGroup(); log.info("cpuUsageForCGroup: {}", cpuUsageForCGroup); From 3ccdaba87ce876cd709b8a39ae058431d389f6a2 Mon Sep 17 00:00:00 2001 From: coderzc Date: Fri, 28 Apr 2023 21:33:42 +0800 Subject: [PATCH 8/8] fix code style add test --- .../broker/loadbalance/LinuxInfoUtils.java | 1 - .../impl/LinuxBrokerHostUsageImplTest.java | 37 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java index daba6d695ff75..1405979ae3b12 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.loadbalance; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Charsets; import java.io.IOException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java index 39298dce0b16a..563f707c445b2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java @@ -18,15 +18,19 @@ */ package org.apache.pulsar.broker.loadbalance.impl; -import lombok.Cleanup; -import org.testng.Assert; -import org.testng.annotations.Test; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.loadbalance.LinuxInfoUtils; +import org.testng.Assert; +import org.testng.annotations.Test; +@Slf4j public class LinuxBrokerHostUsageImplTest { @Test @@ -42,4 +46,31 @@ public void checkOverrideBrokerNicSpeedGbps() { double totalLimit = linuxBrokerHostUsage.getTotalNicLimitWithConfiguration(nics); Assert.assertEquals(totalLimit, 3.0 * 1000 * 1000 * 3); } + + @Test + public void testCpuUsage() throws InterruptedException { + if (!LinuxInfoUtils.isLinux()) { + return; + } + + @Cleanup("shutdown") + ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + LinuxBrokerHostUsageImpl linuxBrokerHostUsage = + new LinuxBrokerHostUsageImpl(Integer.MAX_VALUE, Optional.empty(), executorService); + + linuxBrokerHostUsage.calculateBrokerHostUsage(); + TimeUnit.SECONDS.sleep(1); + linuxBrokerHostUsage.calculateBrokerHostUsage(); + + double usage = linuxBrokerHostUsage.getBrokerHostUsage().getCpu().usage; + double limit = linuxBrokerHostUsage.getBrokerHostUsage().getCpu().limit; + float percentUsage = linuxBrokerHostUsage.getBrokerHostUsage().getCpu().percentUsage(); + + Assert.assertTrue(usage > 0); + Assert.assertTrue(limit > 0); + Assert.assertTrue(limit >= usage); + Assert.assertTrue(percentUsage > 0); + + log.info("usage: {}, limit: {}, percentUsage: {}", usage, limit, percentUsage); + } }