From d89b78e038fbda6b29a9f0350c57b035f2de7dc0 Mon Sep 17 00:00:00 2001 From: wcf Date: Wed, 31 May 2023 16:14:09 +0800 Subject: [PATCH 1/3] fix deploy with multi-network card #142 --- .../polardbx/common/utils/AddressUtils.java | 74 ++++++++++++++++- .../polardbx/common/AddressUtilsTest.java | 77 ++++++++++++++++++ .../test/resources/serialize/inet_address.txt | Bin 0 -> 122 bytes .../polardbx/gms/node/GmsNodeManager.java | 8 ++ polardbx-server/src/main/conf/logback.xml | 3 + .../src/main/resources/logback.xml | 3 + 6 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 polardbx-common/src/test/java/com/alibaba/polardbx/common/AddressUtilsTest.java create mode 100644 polardbx-common/src/test/resources/serialize/inet_address.txt diff --git a/polardbx-common/src/main/java/com/alibaba/polardbx/common/utils/AddressUtils.java b/polardbx-common/src/main/java/com/alibaba/polardbx/common/utils/AddressUtils.java index 99110c120..e4fc8fd79 100644 --- a/polardbx-common/src/main/java/com/alibaba/polardbx/common/utils/AddressUtils.java +++ b/polardbx-common/src/main/java/com/alibaba/polardbx/common/utils/AddressUtils.java @@ -20,11 +20,13 @@ import com.alibaba.polardbx.common.utils.logger.LoggerFactory; import com.google.common.base.Splitter; import lombok.Data; +import org.apache.commons.lang.StringUtils; import java.io.IOException; import java.net.InetAddress; import java.net.NetworkInterface; import java.net.ServerSocket; +import java.net.SocketException; import java.util.Enumeration; import java.util.List; import java.util.regex.Pattern; @@ -88,6 +90,10 @@ private static boolean isValidHostAddress(InetAddress address) { return false; } String name = address.getHostAddress(); + return isIpValid(name); + } + + private static boolean isIpValid(String name) { return (name != null && !EMPTY_IP.equals(name) && !LOCALHOST_IP.equals(name) && IP_PATTERN.matcher(name) .matches()); } @@ -148,6 +154,7 @@ public static String getPaxosAddressByStorageAddress(String storageAddr) { } public static InetAddress localAddress = null; + public final static String POD_IP = "POD_IP"; public static InetAddress getHostAddress() { if (localAddress != null) { @@ -155,6 +162,22 @@ public static InetAddress getHostAddress() { } try { + String podIp = System.getenv(POD_IP); + if (!StringUtils.isBlank(podIp)) { + podIp = podIp.trim(); + if (!isIpValid(podIp)) { + logger.error("pod ip not empty but not valid, pod ip is: " + podIp); + } else { + localAddress = getMatchedAddress(podIp); + if (localAddress != null) { + logger.info("get matched address from pod ip: " + podIp); + return localAddress; + } else { + // local address should not null + logger.error("Failed to matched pod ip to address."); + } + } + } localAddress = InetAddress.getLocalHost(); if (isValidHostAddress(localAddress)) { return localAddress; @@ -164,7 +187,7 @@ public static InetAddress getHostAddress() { + e.getMessage()); } try { - Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); + Enumeration interfaces = getNetInterface(); if (interfaces != null) { while (interfaces.hasMoreElements()) { try { @@ -194,4 +217,53 @@ public static InetAddress getHostAddress() { logger.error("Could not get local host ip address, will use 127.0.0.1 instead."); return localAddress; } + + public static InetAddress getMatchedAddress(String podIp) { + InetAddress matchedHost = null; + try { + matchedHost = tryMatchFromLocalHost(podIp); + if (matchedHost != null) { + return matchedHost; + } + + matchedHost = tryMatchFromAllNet(podIp, getNetInterface()); + } catch (Throwable e) { + logger.error("Failed to matched pod ip to address. cause: " + e.getMessage()); + } + return matchedHost; + } + + public static InetAddress tryMatchFromLocalHost(String podIp) { + try { + InetAddress localHost = InetAddress.getLocalHost(); + if (isValidHostAddress(localHost) && podIp.equalsIgnoreCase(localHost.getHostAddress())) { + return localHost; + } + } catch (Throwable e) { + // just ignore this + } + return null; + } + + public static InetAddress tryMatchFromAllNet(String podIp, Enumeration interfaces) { + if (interfaces != null) { + while (interfaces.hasMoreElements()) { + NetworkInterface network = interfaces.nextElement(); + Enumeration addresses = network.getInetAddresses(); + if (addresses != null) { + while (addresses.hasMoreElements()) { + InetAddress address = addresses.nextElement(); + if (isValidHostAddress(address) && podIp.equalsIgnoreCase(address.getHostAddress())) { + return address; + } + } + } + } + } + return null; + } + + public static Enumeration getNetInterface() throws SocketException { + return NetworkInterface.getNetworkInterfaces(); + } } diff --git a/polardbx-common/src/test/java/com/alibaba/polardbx/common/AddressUtilsTest.java b/polardbx-common/src/test/java/com/alibaba/polardbx/common/AddressUtilsTest.java new file mode 100644 index 000000000..e403a7e67 --- /dev/null +++ b/polardbx-common/src/test/java/com/alibaba/polardbx/common/AddressUtilsTest.java @@ -0,0 +1,77 @@ +package com.alibaba.polardbx.common; + +import com.alibaba.polardbx.common.utils.AddressUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class AddressUtilsTest { + static final String MOCK_IP = "30.221.112.137"; + static final String SERIALIZED_INET_ADDRESS_PATH = "serialize/inet_address.txt"; + + @Test + public void testMatchPodIpFromLocalHost() { + Assert.assertNull(AddressUtils.tryMatchFromLocalHost(MOCK_IP)); + } + + @Test + public void testMatchPodIpFromAllNet() + throws IOException, ClassNotFoundException, InvocationTargetException, NoSuchMethodException, + InstantiationException, IllegalAccessException { + + Assert.assertNull(AddressUtils.tryMatchFromAllNet(MOCK_IP, AddressUtils.getNetInterface())); + + // test mock interface + InetAddress inetAddress = AddressUtils.tryMatchFromAllNet( + MOCK_IP, Collections.enumeration(generateMockInterfaces())); + Assert.assertEquals(inetAddress.getHostAddress(), MOCK_IP); + } + + @Test + public void checkNotMatchedAddress() { + Assert.assertNull(AddressUtils.getMatchedAddress("127.0.0.1")); + Assert.assertNull(AddressUtils.getMatchedAddress("0.0.0.0")); + Assert.assertNull(AddressUtils.getMatchedAddress("123.xx.23.1")); + } + + private List generateMockInterfaces() + throws IOException, InvocationTargetException, NoSuchMethodException, InstantiationException, + IllegalAccessException, ClassNotFoundException { + List interfaceList = + new ArrayList<>(Collections.list(NetworkInterface.getNetworkInterfaces())); + NetworkInterface mockNetInterface = mockNetInterface(); + interfaceList.add(mockNetInterface); + return interfaceList; + } + + private NetworkInterface mockNetInterface() + throws NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, + IOException, ClassNotFoundException { + Class clazz = NetworkInterface.class; + Constructor constructor = clazz.getDeclaredConstructor(String.class, int.class, InetAddress[].class); + constructor.setAccessible(true); + NetworkInterface mockInterface = (NetworkInterface) constructor.newInstance("mock_ens", 1001, + new InetAddress[] {getMockInetAddress()}); + return mockInterface; + } + + private InetAddress getMockInetAddress() throws IOException, ClassNotFoundException { + String path = this.getClass().getClassLoader().getResource(SERIALIZED_INET_ADDRESS_PATH).getPath(); + try (FileInputStream fileIn = new FileInputStream(path); + ObjectInputStream objIn = new ObjectInputStream(fileIn)) { + + InetAddress inetAddress = (InetAddress) objIn.readObject(); + return inetAddress; + } + } +} diff --git a/polardbx-common/src/test/resources/serialize/inet_address.txt b/polardbx-common/src/test/resources/serialize/inet_address.txt new file mode 100644 index 0000000000000000000000000000000000000000..03692358a14deaf1ece7cb8249bc7bddcc8c5aa1 GIT binary patch literal 122 zcmaFAfA9PKdl^`Zix@<*63Y_x@={CmJb{E`N=i{`aj`B?HXMkN5i>xu$7pdkf7%)r3FR8Ro` DQtC8v literal 0 HcmV?d00001 diff --git a/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/node/GmsNodeManager.java b/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/node/GmsNodeManager.java index 28580c750..15d629793 100644 --- a/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/node/GmsNodeManager.java +++ b/polardbx-gms/src/main/java/com/alibaba/polardbx/gms/node/GmsNodeManager.java @@ -45,6 +45,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; public class GmsNodeManager extends AbstractLifecycle { @@ -323,6 +324,13 @@ private void loadRedundantNodes() { return gmsNode1.instId.compareTo(gmsNode2.instId); }); this.currentIndex = allNodes.indexOf(GmsNodeManager.getInstance().getLocalNode()); + if (currentIndex == -1) { + LOGGER.error(String.format( + "local node not found from allNodes, local node is %s, while all node is %s", + GmsNodeManager.getInstance().getLocalNode(), + allNodes.stream().map(node -> node.toString()).collect(Collectors.joining(","))) + ); + } } private GmsNode buildNode(ServerInfoRecord record, int uniqueId) { diff --git a/polardbx-server/src/main/conf/logback.xml b/polardbx-server/src/main/conf/logback.xml index 15cdc9029..a30e5aa21 100644 --- a/polardbx-server/src/main/conf/logback.xml +++ b/polardbx-server/src/main/conf/logback.xml @@ -991,6 +991,9 @@ + + + diff --git a/polardbx-server/src/main/resources/logback.xml b/polardbx-server/src/main/resources/logback.xml index a378b17ee..348b6517c 100644 --- a/polardbx-server/src/main/resources/logback.xml +++ b/polardbx-server/src/main/resources/logback.xml @@ -967,6 +967,9 @@ + + + From c250dc69d357e80e77e6d3398304400963011f43 Mon Sep 17 00:00:00 2001 From: wcf Date: Fri, 2 Jun 2023 15:54:33 +0800 Subject: [PATCH 2/3] update docker file #142 --- docker/20-nproc.conf | 6 ++++++ docker/Dockerfile | 1 + 2 files changed, 7 insertions(+) create mode 100644 docker/20-nproc.conf diff --git a/docker/20-nproc.conf b/docker/20-nproc.conf new file mode 100644 index 000000000..87dc0bdd2 --- /dev/null +++ b/docker/20-nproc.conf @@ -0,0 +1,6 @@ +# Default limit for number of user's processes to prevent +# accidental fork bombs. +# See rhbz #432903 for reasoning. + +* soft nproc 4096 +root soft nproc unlimited \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 38fe9c79e..29b7d3b51 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,6 +6,7 @@ COPY polardbx-server/target/polardbx-server drds-server COPY docker/admin/bin/* /home/admin/bin/ COPY docker/etc/* /tmp/ COPY docker/entrypoint.sh entrypoint.sh +COPY docker/20-nproc.conf /etc/security/limits.d/20-nproc.conf RUN \ chmod a+x bin/* && \ From ff14d61aebddb5cceade1556324bd82cdf88f821 Mon Sep 17 00:00:00 2001 From: wcf Date: Fri, 2 Jun 2023 15:55:26 +0800 Subject: [PATCH 3/3] fix docker file #142 --- docker/20-nproc.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/20-nproc.conf b/docker/20-nproc.conf index 87dc0bdd2..db822fe44 100644 --- a/docker/20-nproc.conf +++ b/docker/20-nproc.conf @@ -2,5 +2,5 @@ # accidental fork bombs. # See rhbz #432903 for reasoning. -* soft nproc 4096 +* soft nproc 40960 root soft nproc unlimited \ No newline at end of file