Fix addr conflict.

infiniflow · Dec 18, 2024 · 6b27d2b · 6b27d2b
1 parent d347170
commit 6b27d2b
Show file tree

Hide file tree

Showing 12 changed files with 315 additions and 693 deletions.
diff --git a/.github/workflows/slow_test.yml b/.github/workflows/slow_test.yml
@@ -60,8 +60,6 @@ jobs:
           echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> $GITHUB_ENV
           echo "CPUS=${CPUS}" >> $GITHUB_ENV
           TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
-          INF_DIRECTORY=$PWD
-          echo "${INF_DIRECTORY}" | sudo tee .tester_env
           sudo docker pull infiniflow/infinity_builder:centos7_clang18
           sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --net=host --privileged --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot -v /var/run/docker.sock:/var/run/docker.sock --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18
 
@@ -113,7 +111,7 @@ jobs:
           # Fix sanitizer: https://github.com/ClickHouse/ClickHouse/issues/64086
           old_value=$(sudo sysctl -n vm.mmap_rnd_bits)
           sudo sysctl -w vm.mmap_rnd_bits=28
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006"
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity"
           sudo sysctl -w vm.mmap_rnd_bits=$old_value
 
       - name: Collect thread sanitizer output in cluster test

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -54,10 +54,21 @@ jobs:
           echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> $GITHUB_ENV
           echo "CPUS=${CPUS}" >> $GITHUB_ENV
           TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
-          INF_DIRECTORY=$PWD
-          echo "${INF_DIRECTORY}" | sudo tee .tester_env
           sudo docker pull infiniflow/infinity_builder:centos7_clang18
-          sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --net=host --privileged --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot -v /var/run/docker.sock:/var/run/docker.sock --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18
+          sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot -v /var/run/docker.sock:/var/run/docker.sock --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18
+
+      - name: Start minio container
+        if: ${{ !cancelled() && !failure() }}
+        run: |
+          MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
+          MINIO_DIR=~/minio_data_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
+          echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> $GITHUB_ENV
+          echo "MINIO_DIR=${MINIO_DIR}" >> $GITHUB_ENV
+          sudo docker rm -f -v ${MINIO_CONTAINER} && sudo rm -fr ${MINIO_DIR} && sudo mkdir ${MINIO_DIR} && sudo docker run -d --net=container:${BUILDER_CONTAINER} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin"  -v ${MINIO_DIR}:/data    quay.io/minio/minio server /data --console-address ":9006" --address ":9005" && sleep 5s
+          if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then
+              echo "Minio container is not running"
+              exit 1
+          fi
 
       - name: Build debug version
         if: ${{ !cancelled() && !failure() }}
@@ -71,7 +82,7 @@ jobs:
         if: ${{ !cancelled() && !failure() }}
         id: run_cluster_test
         run: |
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006"
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity"
 
       - name: Collect cluster test output
         if: ${{ !cancelled() }}
@@ -96,19 +107,6 @@ jobs:
           failure="${{ steps.run_restart_test.outcome == 'failure'}}"
           sudo python3 scripts/collect_restart_log.py --executable_path=cmake-build-debug/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
-      - name: Start minio container
-        if: ${{ !cancelled() && !failure() }}
-        run: |
-          MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
-          MINIO_DIR=~/minio_data_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
-          echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> $GITHUB_ENV
-          echo "MINIO_DIR=${MINIO_DIR}" >> $GITHUB_ENV
-          sudo docker rm -f -v ${MINIO_CONTAINER} && sudo rm -fr ${MINIO_DIR} && sudo mkdir ${MINIO_DIR} && sudo docker run -d --net=container:${BUILDER_CONTAINER} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin"  -v ${MINIO_DIR}:/data    quay.io/minio/minio server /data --console-address ":9006" --address ":9005" && sleep 5s
-          if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then
-              echo "Minio container is not running"
-              exit 1
-          fi
-
       - name: Start infinity debug version with minio
         if: ${{ !cancelled() && !failure() }}
         run: |
@@ -235,6 +233,18 @@ jobs:
           TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
           sudo docker pull infiniflow/infinity_builder:centos7_clang18
           sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18
+      - name: Start minio container
+        if: ${{ !cancelled() && !failure() }}
+        run: |
+          MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
+          MINIO_DIR=~/minio_data_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
+          echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> $GITHUB_ENV
+          echo "MINIO_DIR=${MINIO_DIR}" >> $GITHUB_ENV
+          sudo docker rm -f -v ${MINIO_CONTAINER} && sudo rm -fr ${MINIO_DIR} && sudo mkdir ${MINIO_DIR} && sudo docker run -d --net=container:${BUILDER_CONTAINER} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin"  -v ${MINIO_DIR}:/data    quay.io/minio/minio server /data --console-address ":9006" --address ":9005" && sleep 5s
+          if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then
+              echo "Minio container is not running"
+              exit 1
+          fi
 
       - name: Build release version
         if: ${{ !cancelled() && !failure() }}
@@ -244,17 +254,17 @@ jobs:
         if: ${{ !cancelled() && !failure() }}
         run: sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk infinity-embedded-sdk && pip3 install . -v --config-settings=cmake.build-type='RelWithDebInfo' --config-settings=build-dir='cmake-build-release' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd python/infinity_sdk/ && pip3 install . -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.."
 
-    #   - name: Run cluster test
-    #     if: ${{ !cancelled() && !failure() }}
-    #     id: run_cluster_test
-    #     run: |
-    #       sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-release/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006"
-
-    #   - name: Collect cluster test output
-    #     if: ${{ !cancelled() }}
-    #     run: |
-    #       failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
-    #       sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-release/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
+      - name: Run cluster test
+        if: ${{ !cancelled() && !failure() }}
+        id: run_cluster_test
+        run: |
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && python3 tools/run_cluster_test.py --infinity_path=cmake-build-release/src/infinity"
+
+      - name: Collect cluster test output
+        if: ${{ !cancelled() }}
+        run: |
+          failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
+          sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-release/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
     
       - name: Prepare restart test data
         if: ${{ !cancelled() && !failure() }}
@@ -284,19 +294,6 @@ jobs:
           failure="${{ steps.run_pysdk_local_infinity_test.outcome == 'failure'}}"
           sudo python3 scripts/collect_embedded_log.py --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure} --log_path=/var/infinity/log/infinity.log
 
-      - name: Start minio container
-        if: ${{ !cancelled() && !failure() }}
-        run: |
-          MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
-          MINIO_DIR=~/minio_data_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
-          echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> $GITHUB_ENV
-          echo "MINIO_DIR=${MINIO_DIR}" >> $GITHUB_ENV
-          sudo docker rm -f -v ${MINIO_CONTAINER} && sudo rm -fr ${MINIO_DIR} && sudo mkdir ${MINIO_DIR} && sudo docker run -d --net=container:${BUILDER_CONTAINER} --name ${MINIO_CONTAINER} -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin"  -v ${MINIO_DIR}:/data    quay.io/minio/minio server /data --console-address ":9006" --address ":9005" && sleep 5s
-          if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then
-              echo "Minio container is not running"
-              exit 1
-          fi
-
       - name: Start infinity release version with minio
         if: ${{ !cancelled() && !failure() }}
         run: |

diff --git a/conf/mock_follower.toml b/conf/mock_follower.toml
@@ -0,0 +1,58 @@
+[general]
+version = "0.5.0"
+time_zone = "utc-8"
+server_mode = "admin"
+
+[network]
+server_address = "17.0.0.3"
+postgres_port = 5434
+http_port = 23822
+client_port = 23819
+connection_pool_size = 128
+peer_ip = "17.0.0.3"
+peer_port = 23852
+peer_retry_delay = 0
+peer_retry_count = 0
+peer_connect_timeout = 2000
+peer_recv_timeout = 0
+peer_send_timeout = 0
+
+[log]
+log_filename = "infinity.log"
+log_dir = "/var/infinity/follower/log"
+log_to_stdout = false
+log_file_max_size = "10GB"
+log_file_rotate_count = 10
+log_level = "trace"
+
+[storage]
+persistence_dir = "/var/infinity/follower/persistence"
+data_dir = "/var/infinity/follower/data"
+optimize_interval = "10s"
+cleanup_interval = "60s"
+compact_interval = "120s"
+mem_index_capacity = 65536
+storage_type = "minio"
+
+[storage.object_storage]
+url = "17.0.0.1:9005"
+bucket_name = "infinity"
+access_key = "minioadmin"
+secret_key = "minioadmin"
+enable_https = false
+
+[buffer]
+buffer_manager_size = "4GB"
+lru_num = 7
+temp_dir = "/var/infinity/follower/tmp"
+memindex_memory_quota = "1GB"
+
+[wal]
+wal_dir = "/var/infinity/follower/wal"
+full_checkpoint_interval = "86400s"
+delta_checkpoint_interval = "60s"
+wal_compact_threshold = "1GB"
+wal_flush = "only_write"
+
+[resource]
+resource_dir = "/var/infinity/follower/resource"
diff --git a/conf/mock_leader.toml b/conf/mock_leader.toml
@@ -0,0 +1,58 @@
+[general]
+version = "0.5.0"
+time_zone = "utc-8"
+server_mode = "admin"
+
+[network]
+server_address = "17.0.0.2"
+postgres_port = 5433
+http_port = 23821
+client_port = 23818
+connection_pool_size = 128
+peer_ip = "17.0.0.2"
+peer_port = 23851
+peer_retry_delay = 0
+peer_retry_count = 0
+peer_connect_timeout = 2000
+peer_recv_timeout = 0
+peer_send_timeout = 0
+
+[log]
+log_filename = "infinity.log"
+log_dir = "/var/infinity/leader/log"
+log_to_stdout = false
+log_file_max_size = "10GB"
+log_file_rotate_count = 10
+log_level = "trace"
+
+[storage]
+persistence_dir = "/var/infinity/leader/persistence"
+data_dir = "/var/infinity/leader/data"
+optimize_interval = "10s"
+cleanup_interval = "60s"
+compact_interval = "120s"
+mem_index_capacity = 65536
+storage_type = "minio"
+
+[storage.object_storage]
+url = "17.0.0.1:9005"
+bucket_name = "infinity"
+access_key = "minioadmin"
+secret_key = "minioadmin"
+enable_https = false
+
+[buffer]
+buffer_manager_size = "4GB"
+lru_num = 7
+temp_dir = "/var/infinity/leader/tmp"
+memindex_memory_quota = "1GB"
+
+[wal]
+wal_dir = "/var/infinity/leader/wal"
+full_checkpoint_interval = "86400s"
+delta_checkpoint_interval = "60s"
+wal_compact_threshold = "1GB"
+wal_flush = "only_write"
+
+[resource]
+resource_dir = "/var/infinity/leader/resource"
diff --git a/python/test_cluster/clear_docker.py b/python/test_cluster/clear_docker.py
diff --git a/python/test_cluster/conftest.py b/python/test_cluster/conftest.py
@@ -1,6 +1,5 @@
 import pytest
-from infinity_cluster import InfinityCluster, MinioParams
-from docker_infinity_cluster import DockerInfinityCluster
+from infinity_cluster import InfinityCluster
 from mocked_infinity_cluster import MockInfinityCluster
 
 
@@ -10,28 +9,6 @@ def pytest_addoption(parser):
         action="store",
         default="cmake-build-debug/src/infinity",
     )
-    parser.addoption(
-        "--minio_dir",
-        action="store",
-        default="minio",
-    )
-    parser.addoption(
-        "--minio_port",
-        action="store",
-        default=9000,
-    )
-    parser.addoption(
-        "--minio_console_port",
-        action="store",
-        default=9001,
-    )
-    parser.addoption(
-        "--infinity_dir",
-        action="store",
-        required=True,
-        help="Path to infinity directory. For local test, $pwd is ok",
-    )
-    parser.addoption("--docker", action="store_true", default=False)
     parser.addoption(
         "--use_sudo",
         action="store_true",
@@ -40,56 +17,19 @@ def pytest_addoption(parser):
     )
 
 
-def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "docker: mark test to run only when --docker option is provided"
-    )
-
-
-def pytest_collection_modifyitems(config, items):
-    if config.getoption("--docker"):
-        return  # do not skip docker test
-    skip_docker = pytest.mark.skip(reason="need --docker option to run")
-    for item in items:
-        if "docker" in item.keywords:
-            print(f"skip {item.name}")
-            item.add_marker(skip_docker)
-
-
 def pytest_generate_tests(metafunc):
     infinity_path = metafunc.config.getoption("infinity_path")
-    minio_dir = metafunc.config.getoption("minio_dir")
-    minio_port = metafunc.config.getoption("minio_port")
-    minio_console_port = metafunc.config.getoption("minio_console_port")
-    minio_params = MinioParams(minio_dir, minio_port, minio_console_port)
 
-    infinity_dir = metafunc.config.getoption("infinity_dir")
     use_sudo = metafunc.config.getoption("use_sudo")
 
-    if len(infinity_dir) == 0:
-        # raise ValueError("Please provide a valid infinity_dir")
-        pass
-    print("infinity_dir: ", infinity_dir)
-    # print(metafunc.fixturenames)
-
     test_name = metafunc.function.__name__
-    if "docker_cluster" in metafunc.fixturenames:
-        # skip if docker is in option and the testcase is marked with docker
-        if not metafunc.config.getoption("--docker"):
-            return
-        docker_infinity_cluster = DockerInfinityCluster(
-            infinity_path, minio_params=minio_params, infinity_dir=infinity_dir
-        )
-        metafunc.parametrize("docker_cluster", [docker_infinity_cluster])
-    elif "cluster" in metafunc.fixturenames:
-        infinity_cluster = InfinityCluster(
-            infinity_path, minio_params=minio_params, test_name=test_name
-        )
+
+    if "cluster" in metafunc.fixturenames:
+        infinity_cluster = InfinityCluster(infinity_path, test_name=test_name)
         metafunc.parametrize("cluster", [infinity_cluster])
     elif "mock_cluster" in metafunc.fixturenames:
         mock_infinity_cluster = MockInfinityCluster(
             infinity_path,
-            minio_params=minio_params,
             test_name=test_name,
             use_sudo=use_sudo,
         )