fix deploy with multi-network card #142

docker/20-nproc.conf

@@ -0,0 +1,6 @@
+# Default limit for number of user's processes to prevent
+# accidental fork bombs.
+# See rhbz #432903 for reasoning.
+
+*          soft    nproc     40960
+root       soft    nproc     unlimited

docker/Dockerfile

@@ -6,6 +6,7 @@ COPY polardbx-server/target/polardbx-server drds-server
 COPY docker/admin/bin/* /home/admin/bin/
 COPY docker/etc/* /tmp/
 COPY docker/entrypoint.sh entrypoint.sh
+COPY docker/20-nproc.conf /etc/security/limits.d/20-nproc.conf
 
 RUN \
     chmod a+x bin/* && \

polardbx-common/src/main/java/com/alibaba/polardbx/common/utils/AddressUtils.java

@@ -20,11 +20,13 @@
 import com.alibaba.polardbx.common.utils.logger.LoggerFactory;
 import com.google.common.base.Splitter;
 import lombok.Data;
+import org.apache.commons.lang.StringUtils;
 
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.NetworkInterface;
 import java.net.ServerSocket;
+import java.net.SocketException;
 import java.util.Enumeration;
 import java.util.List;
 import java.util.regex.Pattern;
@@ -88,6 +90,10 @@ private static boolean isValidHostAddress(InetAddress address) {
             return false;
         }
         String name = address.getHostAddress();
+        return isIpValid(name);
+    }
+
+    private static boolean isIpValid(String name) {
         return (name != null && !EMPTY_IP.equals(name) && !LOCALHOST_IP.equals(name) && IP_PATTERN.matcher(name)
             .matches());
     }
@@ -148,13 +154,30 @@ public static String getPaxosAddressByStorageAddress(String storageAddr) {
     }
 
     public static InetAddress localAddress = null;
+    public final static String POD_IP = "POD_IP";
 
     public static InetAddress getHostAddress() {
         if (localAddress != null) {
             return localAddress;
         }
 
         try {
+            String podIp = System.getenv(POD_IP);
+            if (!StringUtils.isBlank(podIp)) {
+                podIp = podIp.trim();
+                if (!isIpValid(podIp)) {
+                    logger.error("pod ip not empty but not valid, pod ip is: " + podIp);
+                } else {
+                    localAddress = getMatchedAddress(podIp);
+                    if (localAddress != null) {
+                        logger.info("get matched address from pod ip: " + podIp);
+                        return localAddress;
+                    } else {
+                        // local address should not null
+                        logger.error("Failed to matched pod ip to address.");
+                    }
+                }
+            }
             localAddress = InetAddress.getLocalHost();
             if (isValidHostAddress(localAddress)) {
                 return localAddress;
@@ -164,7 +187,7 @@ public static InetAddress getHostAddress() {
                 + e.getMessage());
         }
         try {
-            Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces();
+            Enumeration<NetworkInterface> interfaces = getNetInterface();
             if (interfaces != null) {
                 while (interfaces.hasMoreElements()) {
                     try {
@@ -194,4 +217,53 @@ public static InetAddress getHostAddress() {
         logger.error("Could not get local host ip address, will use 127.0.0.1 instead.");
         return localAddress;
     }
+
+    public static InetAddress getMatchedAddress(String podIp) {
+        InetAddress matchedHost = null;
+        try {
+            matchedHost = tryMatchFromLocalHost(podIp);
+            if (matchedHost != null) {
+                return matchedHost;
+            }
+
+            matchedHost = tryMatchFromAllNet(podIp, getNetInterface());
+        } catch (Throwable e) {
+            logger.error("Failed to matched pod ip to address. cause: " + e.getMessage());
+        }
+        return matchedHost;
+    }
+
+    public static InetAddress tryMatchFromLocalHost(String podIp) {
+        try {
+            InetAddress localHost = InetAddress.getLocalHost();
+            if (isValidHostAddress(localHost) && podIp.equalsIgnoreCase(localHost.getHostAddress())) {
+                return localHost;
+            }
+        } catch (Throwable e) {
+            // just ignore this
+        }
+        return null;
+    }
+
+    public static InetAddress tryMatchFromAllNet(String podIp, Enumeration<NetworkInterface> interfaces) {
+        if (interfaces != null) {
+            while (interfaces.hasMoreElements()) {
+                NetworkInterface network = interfaces.nextElement();
+                Enumeration<InetAddress> addresses = network.getInetAddresses();
+                if (addresses != null) {
+                    while (addresses.hasMoreElements()) {
+                        InetAddress address = addresses.nextElement();
+                        if (isValidHostAddress(address) && podIp.equalsIgnoreCase(address.getHostAddress())) {
+                            return address;
+                        }
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
+    public static Enumeration<NetworkInterface> getNetInterface() throws SocketException {
+        return NetworkInterface.getNetworkInterfaces();
+    }
 }

polardbx-common/src/test/java/com/alibaba/polardbx/common/AddressUtilsTest.java

@@ -0,0 +1,77 @@
+package com.alibaba.polardbx.common;
+
+import com.alibaba.polardbx.common.utils.AddressUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class AddressUtilsTest {
+    static final String MOCK_IP = "30.221.112.137";
+    static final String SERIALIZED_INET_ADDRESS_PATH = "serialize/inet_address.txt";
+
+    @Test
+    public void testMatchPodIpFromLocalHost() {
+        Assert.assertNull(AddressUtils.tryMatchFromLocalHost(MOCK_IP));
+    }
+
+    @Test
+    public void testMatchPodIpFromAllNet()
+        throws IOException, ClassNotFoundException, InvocationTargetException, NoSuchMethodException,
+        InstantiationException, IllegalAccessException {
+
+        Assert.assertNull(AddressUtils.tryMatchFromAllNet(MOCK_IP, AddressUtils.getNetInterface()));
+
+        // test mock interface
+        InetAddress inetAddress = AddressUtils.tryMatchFromAllNet(
+            MOCK_IP, Collections.enumeration(generateMockInterfaces()));
+        Assert.assertEquals(inetAddress.getHostAddress(), MOCK_IP);
+    }
+
+    @Test
+    public void checkNotMatchedAddress() {
+        Assert.assertNull(AddressUtils.getMatchedAddress("127.0.0.1"));
+        Assert.assertNull(AddressUtils.getMatchedAddress("0.0.0.0"));
+        Assert.assertNull(AddressUtils.getMatchedAddress("123.xx.23.1"));
+    }
+
+    private List<NetworkInterface> generateMockInterfaces()
+        throws IOException, InvocationTargetException, NoSuchMethodException, InstantiationException,
+        IllegalAccessException, ClassNotFoundException {
+        List<NetworkInterface> interfaceList =
+            new ArrayList<>(Collections.list(NetworkInterface.getNetworkInterfaces()));
+        NetworkInterface mockNetInterface = mockNetInterface();
+        interfaceList.add(mockNetInterface);
+        return interfaceList;
+    }
+
+    private NetworkInterface mockNetInterface()
+        throws NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException,
+        IOException, ClassNotFoundException {
+        Class<?> clazz = NetworkInterface.class;
+        Constructor<?> constructor = clazz.getDeclaredConstructor(String.class, int.class, InetAddress[].class);
+        constructor.setAccessible(true);
+        NetworkInterface mockInterface = (NetworkInterface) constructor.newInstance("mock_ens", 1001,
+            new InetAddress[] {getMockInetAddress()});
+        return mockInterface;
+    }
+
+    private InetAddress getMockInetAddress() throws IOException, ClassNotFoundException {
+        String path = this.getClass().getClassLoader().getResource(SERIALIZED_INET_ADDRESS_PATH).getPath();
+        try (FileInputStream fileIn = new FileInputStream(path);
+            ObjectInputStream objIn = new ObjectInputStream(fileIn)) {
+
+            InetAddress inetAddress = (InetAddress) objIn.readObject();
+            return inetAddress;
+        }
+    }
+}

polardbx-gms/src/main/java/com/alibaba/polardbx/gms/node/GmsNodeManager.java

@@ -45,6 +45,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
 
 public class GmsNodeManager extends AbstractLifecycle {
 
@@ -323,6 +324,13 @@ private void loadRedundantNodes() {
             return gmsNode1.instId.compareTo(gmsNode2.instId);
         });
         this.currentIndex = allNodes.indexOf(GmsNodeManager.getInstance().getLocalNode());
+        if (currentIndex == -1) {
+            LOGGER.error(String.format(
+                "local node not found from allNodes, local node is %s, while all node is %s",
+                GmsNodeManager.getInstance().getLocalNode(),
+                allNodes.stream().map(node -> node.toString()).collect(Collectors.joining(",")))
+            );
+        }
     }
 
     private GmsNode buildNode(ServerInfoRecord record, int uniqueId) {

polardbx-server/src/main/conf/logback.xml

@@ -991,6 +991,9 @@
     <logger name="com.alibaba.polardbx.net.handler.FrontendAuthenticator" additivity="true">
         <level value="info"/>
     </logger>
+    <logger name="com.alibaba.polardbx.common.utils.AddressUtils" additivity="true">
+        <level value="info"/>
+    </logger>
     <logger name="com.alibaba.polardbx.config.manager" additivity="true">
         <level value="warn"/>
     </logger>

polardbx-server/src/main/resources/logback.xml

@@ -967,6 +967,9 @@
     <logger name="com.alibaba.polardbx.net.handler.FrontendAuthenticator" additivity="true">
         <level value="info"/>
     </logger>
+    <logger name="com.alibaba.polardbx.common.utils.AddressUtils" additivity="true">
+        <level value="info"/>
+    </logger>
     <logger name="com.alibaba.polardbx.config.manager" additivity="true">
         <level value="warn"/>
     </logger>