chore: add start cluster with docker script (#400)

* chore: add start cluster with docker script Signed-off-by: <[email protected]> * chore: remove print Signed-off-by: <[email protected]> * chore: rename folder Signed-off-by: <[email protected]> --------- Signed-off-by: <[email protected]>
wesql · Dec 29, 2023 · 0a4f94f · 0a4f94f
1 parent 4b63f2b
commit 0a4f94f
Show file tree

Hide file tree

Showing 11 changed files with 439 additions and 9 deletions.
diff --git a/.github/workflows/cluster_endtoend_mysqltester.yml b/.github/workflows/cluster_endtoend_mysqltester.yml
@@ -96,7 +96,7 @@ jobs:
         run: |
           
           # build image
-          DOCKER_BUILDKIT=0 docker build docker/wesqlscale/local/ -t apecloud/mysql-tester
+          DOCKER_BUILDKIT=0 docker build docker/wesqlscale/mysqltester/ -t apecloud/mysql-tester
           
           # failpoint
           chmod 755 ./test/failpoint/failpoints.sh && source ./test/failpoint/failpoints.sh

diff --git a/config/mycnf/default.cnf b/config/mycnf/default.cnf
@@ -37,5 +37,6 @@ connect_timeout = 30
 innodb_lock_wait_timeout = 20
 max_allowed_packet = 64M
 max_connections = 500
+bind-address=0.0.0.0
 
 
diff --git a/docker/wesqlscale/local/Dockerfile b/docker/wesqlscale/local/Dockerfile
@@ -13,10 +13,10 @@ RUN apt-get update
 RUN apt-get install -y sudo curl vim jq default-mysql-client
 
 # Install dependencies
-COPY ./install_dependencies.sh /vt/dist/install_dependencies.sh
+COPY install_dependencies.sh /vt/dist/install_dependencies.sh
 RUN /vt/dist/install_dependencies.sh mysql80
 
-COPY ./install_local_dependencies.sh /vt/dist/install_local_dependencies.sh
+COPY install_local_dependencies.sh /vt/dist/install_local_dependencies.sh
 RUN  /vt/dist/install_local_dependencies.sh
 
 # Allows some docker builds to disable CGO
@@ -42,9 +42,9 @@ VOLUME /vt/src/mount
 VOLUME /vt/vtdataroot
 
 
-ENV VTROOT /vt/src/mount
+ENV VTROOT /vt/src/mount/wesql-scala
 ENV VTDATAROOT /vt/vtdataroot
-ENV PATH $VTROOT/bin:$PATH
+ENV PATH $VTROOT/wesql-scala/bin/_amd64:$PATH
 ENV PATH="/var/opt/etcd:${PATH}"
 
 ARG GOPROXY=https://goproxy.cn
@@ -54,7 +54,7 @@ ENV GONOSUMDB=github.com/apecloud
 ENV GOPRIVATE=github.com/apecloud
 ENV GOPROXY=${GOPROXY}
 
-EXPOSE 15306 17100 17101 17102
+EXPOSE 15306 17100 17101 17102 2379
 
 WORKDIR /vt/src/mount
 

diff --git a/docker/wesqlscale/mysqltester/Dockerfile b/docker/wesqlscale/mysqltester/Dockerfile
@@ -0,0 +1,63 @@
+# Copyright ApeCloud, Inc.
+# Licensed under the Apache v2(found in the LICENSE file in the root directory).
+
+
+
+ARG bootstrap_version=14.1
+ARG image="vitess/bootstrap:${bootstrap_version}-common"
+
+FROM "${image}"
+
+
+RUN apt-get update
+RUN apt-get install -y sudo curl vim jq default-mysql-client
+
+# Install dependencies
+COPY install_dependencies.sh /vt/dist/install_dependencies.sh
+RUN /vt/dist/install_dependencies.sh mysql80
+
+COPY install_local_dependencies.sh /vt/dist/install_local_dependencies.sh
+RUN  /vt/dist/install_local_dependencies.sh
+
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Re-copy sources from working tree.
+# COPY --chown=vitess:vitess localdocker /vt/src/vitess.io/vitess
+
+# RUN mkdir /vt/src/mount  && chown vitess:vitess /vt/src/mount
+
+RUN echo "source /vt/src/mount/examples/common/env.sh" >> /etc/bash.bashrc
+# Build and install Vitess in a temporary output directory.
+USER vitess
+
+WORKDIR /vt/src/vitess.io/vitess
+
+# pre download go package
+#RUN go mod download
+
+# mount for github source code
+VOLUME /vt/src/mount
+# mount for cluster data
+VOLUME /vt/vtdataroot
+
+
+ENV VTROOT /vt/src/mount
+ENV VTDATAROOT /vt/vtdataroot
+ENV PATH $VTROOT/bin:$PATH
+ENV PATH="/var/opt/etcd:${PATH}"
+
+ARG GOPROXY=https://goproxy.cn
+ARG LD_FLAGS="-s -w"
+ENV GONOPROXY=github.com/apecloud
+ENV GONOSUMDB=github.com/apecloud
+ENV GOPRIVATE=github.com/apecloud
+ENV GOPROXY=${GOPROXY}
+
+EXPOSE 15306 17100 17101 17102
+
+WORKDIR /vt/src/mount
+
+CMD ["/bin/bash"]
+
+# CMD cd /vt/src/mount/examples/mysql/ && ./init_cluster.sh && /bin/bash
diff --git a/docker/wesqlscale/mysqltester/install_dependencies.sh b/docker/wesqlscale/mysqltester/install_dependencies.sh
@@ -0,0 +1,196 @@
+#!/bin/bash
+
+# This is a script that gets run as part of the Dockerfile build
+# to install dependencies for the vitess/lite family of images.
+#
+# Usage: install_dependencies.sh <flavor>
+
+set -euo pipefail
+
+FLAVOR="$1"
+export DEBIAN_FRONTEND=noninteractive
+
+KEYSERVERS=(
+    keyserver.ubuntu.com
+    hkp://keyserver.ubuntu.com:80
+)
+
+add_apt_key() {
+    for i in {1..3}; do
+        for keyserver in "${KEYSERVERS[@]}"; do
+            if apt-key adv --no-tty --keyserver "${keyserver}" --recv-keys "$1"; then return; fi
+        done
+    done
+}
+
+# Set number of times to retry a download
+MAX_RETRY=20
+
+do_fetch() {
+    wget \
+        --tries=$MAX_RETRY\
+        --read-timeout=30\
+        --timeout=30\
+        --retry-connrefused\
+        --waitretry=1\
+        --no-dns-cache \
+        $1 -O $2
+}
+
+# Install base packages that are common to all flavors.
+BASE_PACKAGES=(
+    bzip2
+    ca-certificates
+    dirmngr
+    gnupg
+    libaio1
+    libatomic1
+    libcurl4
+    libdbd-mysql-perl
+    libwww-perl
+    libev4
+    libjemalloc2
+    libtcmalloc-minimal4
+    procps
+    rsync
+    strace
+    sysstat
+    wget
+    curl
+    percona-toolkit
+    zstd
+)
+
+apt-get update
+apt-get install -y --no-install-recommends "${BASE_PACKAGES[@]}"
+
+# Packages specific to certain flavors.
+case "${FLAVOR}" in
+mysql57)
+    mysql57_version=5.7.31
+    do_fetch https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/libmysqlclient20_${mysql57_version}-1debian10_amd64.deb /tmp/libmysqlclient20_${mysql57_version}-1debian10_amd64.deb
+    do_fetch https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-community-client_${mysql57_version}-1debian10_amd64.deb /tmp/mysql-community-client_${mysql57_version}-1debian10_amd64.deb
+    do_fetch https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-client_${mysql57_version}-1debian10_amd64.deb /tmp/mysql-client_${mysql57_version}-1debian10_amd64.deb
+    do_fetch https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-community-server_${mysql57_version}-1debian10_amd64.deb /tmp/mysql-community-server_${mysql57_version}-1debian10_amd64.deb
+    do_fetch https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-server_${mysql57_version}-1debian10_amd64.deb /tmp/mysql-server_${mysql57_version}-1debian10_amd64.deb
+    PACKAGES=(
+        /tmp/libmysqlclient20_${mysql57_version}-1debian10_amd64.deb
+        /tmp/mysql-community-client_${mysql57_version}-1debian10_amd64.deb
+        /tmp/mysql-client_${mysql57_version}-1debian10_amd64.deb
+        /tmp/mysql-community-server_${mysql57_version}-1debian10_amd64.deb
+        /tmp/mysql-server_${mysql57_version}-1debian10_amd64.deb
+        percona-xtrabackup-24
+    )
+    ;;
+mysql80)
+    mysql8_version=8.0.30
+    do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-common_${mysql8_version}-1debian10_amd64.deb /tmp/mysql-common_${mysql8_version}-1debian10_amd64.deb
+    do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/libmysqlclient21_${mysql8_version}-1debian10_amd64.deb /tmp/libmysqlclient21_${mysql8_version}-1debian10_amd64.deb
+    do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-client-core_${mysql8_version}-1debian10_amd64.deb /tmp/mysql-community-client-core_${mysql8_version}-1debian10_amd64.deb
+    do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-client-plugins_${mysql8_version}-1debian10_amd64.deb /tmp/mysql-community-client-plugins_${mysql8_version}-1debian10_amd64.deb
+    do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-client_${mysql8_version}-1debian10_amd64.deb /tmp/mysql-community-client_${mysql8_version}-1debian10_amd64.deb
+    do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-client_${mysql8_version}-1debian10_amd64.deb /tmp/mysql-client_${mysql8_version}-1debian10_amd64.deb
+    do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-server-core_${mysql8_version}-1debian10_amd64.deb /tmp/mysql-community-server-core_${mysql8_version}-1debian10_amd64.deb
+    do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-server_${mysql8_version}-1debian10_amd64.deb /tmp/mysql-community-server_${mysql8_version}-1debian10_amd64.deb
+    do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-server_${mysql8_version}-1debian10_amd64.deb /tmp/mysql-server_${mysql8_version}-1debian10_amd64.deb
+    PACKAGES=(
+        /tmp/mysql-common_${mysql8_version}-1debian10_amd64.deb
+        /tmp/libmysqlclient21_${mysql8_version}-1debian10_amd64.deb
+        /tmp/mysql-community-client-core_${mysql8_version}-1debian10_amd64.deb
+        /tmp/mysql-community-client-plugins_${mysql8_version}-1debian10_amd64.deb
+        /tmp/mysql-community-client_${mysql8_version}-1debian10_amd64.deb
+        /tmp/mysql-client_${mysql8_version}-1debian10_amd64.deb
+        /tmp/mysql-community-server-core_${mysql8_version}-1debian10_amd64.deb
+        /tmp/mysql-community-server_${mysql8_version}-1debian10_amd64.deb
+        /tmp/mysql-server_${mysql8_version}-1debian10_amd64.deb
+#        percona-xtrabackup-80
+    )
+    ;;
+percona)
+    PACKAGES=(
+        libcurl3
+        percona-server-server-5.6
+        percona-xtrabackup
+    )
+    ;;
+percona57)
+    PACKAGES=(
+        libperconaserverclient20
+        percona-server-server-5.7
+        percona-xtrabackup-24
+    )
+    ;;
+percona80)
+    PACKAGES=(
+        libperconaserverclient21
+        percona-server-rocksdb
+        percona-server-server
+        percona-xtrabackup-80
+    )
+    ;;
+*)
+    echo "Unknown flavor ${FLAVOR}"
+    exit 1
+    ;;
+esac
+
+# Get GPG keys for extra apt repositories.
+# repo.mysql.com
+add_apt_key 8C718D3B5072E1F5
+add_apt_key A8D3785C
+
+# All flavors include Percona XtraBackup (from repo.percona.com).
+add_apt_key 9334A25F8507EFA5
+
+# Add extra apt repositories for MySQL.
+case "${FLAVOR}" in
+mysql57)
+    echo 'deb http://repo.mysql.com/apt/debian/ buster mysql-5.7' > /etc/apt/sources.list.d/mysql.list
+    ;;
+mysql80)
+    echo 'deb http://repo.mysql.com/apt/debian/ buster mysql-8.0' > /etc/apt/sources.list.d/mysql.list
+    ;;
+esac
+
+# Add extra apt repositories for Percona Server and/or Percona XtraBackup.
+case "${FLAVOR}" in
+mysql57|mysql80|percona57)
+    echo 'deb http://repo.percona.com/apt buster main' > /etc/apt/sources.list.d/percona.list
+    ;;
+percona80)
+    echo 'deb http://repo.percona.com/apt buster main' > /etc/apt/sources.list.d/percona.list
+    echo 'deb http://repo.percona.com/ps-80/apt buster main' > /etc/apt/sources.list.d/percona80.list
+    ;;
+esac
+
+# Pre-fill values for installation prompts that are normally interactive.
+case "${FLAVOR}" in
+percona57)
+    debconf-set-selections <<EOF
+debconf debconf/frontend select Noninteractive
+percona-server-server-5.7 percona-server-server/root_password password 'unused'
+percona-server-server-5.7 percona-server-server/root_password_again password 'unused'
+percona-server-server-5.7 percona-server-server-5.7/root-pass password 'unused'
+percona-server-server-5.7 percona-server-server-5.7/re-root-pass password 'unused'
+EOF
+    ;;
+percona80)
+    debconf-set-selections <<EOF
+debconf debconf/frontend select Noninteractive
+percona-server-server-8.0 percona-server-server/root_password password 'unused'
+percona-server-server-8.0 percona-server-server/root_password_again password 'unused'
+EOF
+    ;;
+esac
+
+# Install flavor-specific packages
+apt-get update
+for i in $(seq 1 $MAX_RETRY); do apt-get install -y --no-install-recommends "${PACKAGES[@]}" && break; done
+if [[ "$i" = "$MAX_RETRY" ]]; then
+    exit 1
+fi
+
+# Clean up files we won't need in the final image.
+rm -rf /var/lib/apt/lists/*
+rm -rf /var/lib/mysql/
+rm -rf /tmp/*.deb
diff --git a/docker/wesqlscale/mysqltester/install_local_dependencies.sh b/docker/wesqlscale/mysqltester/install_local_dependencies.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright ApeCloud, Inc.
+# Licensed under the Apache v2(found in the LICENSE file in the root directory).
+
+
+
+
+# This is a script that gets run as part of the Dockerfile build
+# to install dependencies for the vitess/mini image.
+#
+# Usage: install_mini_dependencies.sh
+
+set -euo pipefail
+
+# Install etcd
+ETCD_VER=v3.4.9
+GOOGLE_URL=https://storage.googleapis.com/etcd
+GITHUB_URL=https://github.com/etcd-io/etcd/releases/download
+DOWNLOAD_URL=${GITHUB_URL}
+
+curl -k -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
+mkdir -p /var/opt/etcd
+sudo tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /var/opt/etcd --strip-components=1
+rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
+
+mkdir -p /var/run/etcd && chown -R vitess:vitess /var/run/etcd
+
+# Clean up files we won't need in the final image.
+rm -rf /var/lib/apt/lists/*
diff --git a/examples/common/env.sh b/examples/common/env.sh
@@ -1,4 +1,9 @@
 #!/bin/bash
+# Copyright ApeCloud, Inc.
+# Licensed under the Apache v2(found in the LICENSE file in the root directory).
+
+
+
 
 # Copyright 2019 The Vitess Authors.
 # 
@@ -66,7 +71,9 @@ elif [ "${TOPO}" = "consul" ]; then
     TOPOLOGY_FLAGS="--topo_implementation consul --topo_global_server_address ${CONSUL_SERVER}:${CONSUL_HTTP_PORT} --topo_global_root vitess/global/"
     mkdir -p "${VTDATAROOT}/consul"
 else
-    ETCD_SERVER="localhost:2379"
+    if [[ -z "${ETCD_SERVER}" ]]; then
+      ETCD_SERVER="localhost:2379"
+    fi
     TOPOLOGY_FLAGS="--topo_implementation etcd2 --topo_global_server_address $ETCD_SERVER --topo_global_root /vitess/global"
 
     mkdir -p "${VTDATAROOT}/etcd"

diff --git a/examples/common/scripts/vttablet-up.sh b/examples/common/scripts/vttablet-up.sh
@@ -31,7 +31,6 @@ jaeger_args=${JAEGER_ARGS:-''}
 
 printf -v alias '%s-%010d' $cell $uid
 printf -v tablet_dir 'vt_%010d' $uid
-tablet_hostname=''
 printf -v tablet_logfile 'vttablet_%010d_querylog.txt' $uid
 
 tablet_type=replica