From 8692bdc27a83122b3be14efb572850ba25a436bf Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 May 2024 16:31:14 +0530 Subject: [PATCH 01/19] feat: add vtop exmaple ci Signed-off-by: Manan Gupta --- .github/workflows/vtop_example.yml | 98 ++++++++++++++++++++++++++++++ test/config.json | 11 ++++ 2 files changed, 109 insertions(+) create mode 100644 .github/workflows/vtop_example.yml diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml new file mode 100644 index 00000000000..7cfbfd9400c --- /dev/null +++ b/.github/workflows/vtop_example.yml @@ -0,0 +1,98 @@ +name: vtop_example +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'vtop_example') + cancel-in-progress: true + +jobs: + build: + name: VTop Example + runs-on: ubuntu-latest + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl -s\ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v4 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@v3.0.1 + id: changes + with: + token: '' + filters: | + examples: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - 'examples/**' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v5 + with: + go-version: 1.22.3 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Install everything we need, and configure + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + go mod download + + - name: Run make minimaltools + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make minimaltools + + - name: Build + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make build + + - name: Install kubectl & kind + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make install_kubectl_kind + + - name: vtop_example + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 60 + run: | + source build.env + eatmydata -- go run test.go -docker=false -skip-build -print-log -follow -retry=1 -timeout=60m vtop_example \ No newline at end of file diff --git a/test/config.json b/test/config.json index 2e612e57ca5..294d623ba5e 100644 --- a/test/config.json +++ b/test/config.json @@ -1175,6 +1175,17 @@ "RetryMax": 1, "Tags": [] }, + "vtop_example": { + "File": "", + "Args": [], + "Command": [ + "test/vtop_example.sh" + ], + "Manual": false, + "Shard": "", + "RetryMax": 1, + "Tags": [] + }, "vtorc_primary_failure": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vtorc/primaryfailure"], From b2043b3d23b8dfdf4fd96b4ed4bb87e413491bff Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 May 2024 18:28:14 +0530 Subject: [PATCH 02/19] feat: also run CI on changes to yaml file of the workflow Signed-off-by: Manan Gupta --- .github/workflows/vtop_example.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml index 7cfbfd9400c..efb43e4d865 100644 --- a/.github/workflows/vtop_example.yml +++ b/.github/workflows/vtop_example.yml @@ -56,6 +56,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - 'examples/**' + - '.github/workflows/vtop_example.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' From e4d9a80f345fb802b7b29a82ad868595b19576fa Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 May 2024 18:32:17 +0530 Subject: [PATCH 03/19] test: fix output name for the changes in relevant files step Signed-off-by: Manan Gupta --- .github/workflows/vtop_example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml index efb43e4d865..1d4f16e43f1 100644 --- a/.github/workflows/vtop_example.yml +++ b/.github/workflows/vtop_example.yml @@ -45,7 +45,7 @@ jobs: with: token: '' filters: | - examples: + end_to_end: - 'go/**/*.go' - 'test.go' - 'Makefile' From 7dbeffc7969b77ff72dc684d748dd2f0f8c21025 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 27 May 2024 13:00:13 +0530 Subject: [PATCH 04/19] feat: fix docker image building Signed-off-by: Manan Gupta --- .github/workflows/vtop_example.yml | 2 +- docker/binaries/vtadmin/Dockerfile | 2 +- test/vtop_example.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml index 1d4f16e43f1..dcdb8f2cdab 100644 --- a/.github/workflows/vtop_example.yml +++ b/.github/workflows/vtop_example.yml @@ -7,7 +7,7 @@ concurrency: jobs: build: name: VTop Example - runs-on: ubuntu-latest + runs-on: gh-hosted-runners-16cores-1 steps: - name: Skip CI diff --git a/docker/binaries/vtadmin/Dockerfile b/docker/binaries/vtadmin/Dockerfile index 9d30ba565e0..fe69237ea13 100644 --- a/docker/binaries/vtadmin/Dockerfile +++ b/docker/binaries/vtadmin/Dockerfile @@ -17,7 +17,7 @@ ARG DEBIAN_VER=bullseye-slim FROM vitess/lite:${VT_BASE_VER} AS lite -FROM node:18-${DEBIAN_VER} as node +FROM node:20-${DEBIAN_VER} as node # Prepare directory structure. RUN mkdir -p /vt/web diff --git a/test/vtop_example.sh b/test/vtop_example.sh index 5ff90a2be7e..4ca9ccde9e5 100755 --- a/test/vtop_example.sh +++ b/test/vtop_example.sh @@ -486,7 +486,7 @@ EOF # Build the docker image for vitess/lite using the local code docker build -f docker/lite/Dockerfile -t vitess/lite:pr . # Build the docker image for vitess/vtadmin using the local code -docker build -f docker/binaries/vtadmin/Dockerfile --build-arg VT_BASE_VER=pr -t vitess/vtadmin:pr . +docker build -f docker/binaries/vtadmin/Dockerfile --build-arg VT_BASE_VER=pr -t vitess/vtadmin:pr ./docker/binaries/vtadmin # Print the docker images available docker image ls From 5ae9c113fd23dbceef3574ef1e0236e055777741 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 27 May 2024 13:47:54 +0530 Subject: [PATCH 05/19] test: print the pod status of vttablet to see what the problem is Signed-off-by: Manan Gupta --- test/vtop_example.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/vtop_example.sh b/test/vtop_example.sh index 4ca9ccde9e5..aa534ee22bd 100755 --- a/test/vtop_example.sh +++ b/test/vtop_example.sh @@ -68,6 +68,8 @@ function checkPodStatusWithTimeout() { sleep 1 done echo -e "ERROR: checkPodStatusWithTimeout timeout to find pod matching:\ngot:\n$out\nfor regex: $regex" + vttabletPod=$(kubectl get pods | grep -E "vttablet" | head -n 1 | awk '{print $1}') + kubectl describe pod $vttabletPod exit 1 } From ad821699caffd3f9998199b1d71341b43f8f1a72 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Mon, 27 May 2024 16:01:19 -0600 Subject: [PATCH 06/19] Use 64 cores runners Signed-off-by: Florent Poinsard --- .github/workflows/vtop_example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml index dcdb8f2cdab..d7deb2d89da 100644 --- a/.github/workflows/vtop_example.yml +++ b/.github/workflows/vtop_example.yml @@ -7,7 +7,7 @@ concurrency: jobs: build: name: VTop Example - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-64cores-1 steps: - name: Skip CI From c42e1e9685c6f2c12277a22e716b461fd0b7fa07 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 May 2024 10:37:23 +0530 Subject: [PATCH 07/19] test: also print the vttablet logs Signed-off-by: Manan Gupta --- test/vtop_example.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/vtop_example.sh b/test/vtop_example.sh index aa534ee22bd..007634e4169 100755 --- a/test/vtop_example.sh +++ b/test/vtop_example.sh @@ -70,6 +70,7 @@ function checkPodStatusWithTimeout() { echo -e "ERROR: checkPodStatusWithTimeout timeout to find pod matching:\ngot:\n$out\nfor regex: $regex" vttabletPod=$(kubectl get pods | grep -E "vttablet" | head -n 1 | awk '{print $1}') kubectl describe pod $vttabletPod + kubectl logs $vttabletPod -c vttablet exit 1 } From acf74bc77e7a52c5966ea4d8a0fbdb6c8a1aaa86 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 May 2024 13:06:27 +0530 Subject: [PATCH 08/19] feat: fix panic in shutdown Signed-off-by: Manan Gupta --- examples/operator/101_initial_cluster.yaml | 28 +++++++--------------- go/cmd/vttablet/cli/cli.go | 2 -- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/examples/operator/101_initial_cluster.yaml b/examples/operator/101_initial_cluster.yaml index 2fc7ebe6a92..4a4f478adac 100644 --- a/examples/operator/101_initial_cluster.yaml +++ b/examples/operator/101_initial_cluster.yaml @@ -155,23 +155,6 @@ stringData: # Vitess defaults ############################################################################### - # Vitess-internal database. - CREATE DATABASE IF NOT EXISTS _vt; - # Note that definitions of local_metadata and shard_metadata should be the same - # as in production which is defined in go/vt/mysqlctl/metadata_tables.go. - CREATE TABLE IF NOT EXISTS _vt.local_metadata ( - name VARCHAR(255) NOT NULL, - value VARCHAR(255) NOT NULL, - db_name VARBINARY(255) NOT NULL, - PRIMARY KEY (db_name, name) - ) ENGINE=InnoDB; - CREATE TABLE IF NOT EXISTS _vt.shard_metadata ( - name VARCHAR(255) NOT NULL, - value MEDIUMBLOB NOT NULL, - db_name VARBINARY(255) NOT NULL, - PRIMARY KEY (db_name, name) - ) ENGINE=InnoDB; - # Admin user with all privileges. CREATE USER 'vt_dba'@'localhost'; GRANT ALL ON *.* TO 'vt_dba'@'localhost'; @@ -199,12 +182,10 @@ stringData: ON *.* TO 'vt_allprivs'@'localhost'; # User for slave replication connections. - # TODO: Should we set a password on this since it allows remote connections? CREATE USER 'vt_repl'@'%'; GRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%'; - # User for Vitess filtered replication (binlog player). - # Same permissions as vt_app. + # User for Vitess VReplication (base vstreamers and vplayer). CREATE USER 'vt_filtered'@'localhost'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE, REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES, @@ -212,6 +193,13 @@ stringData: SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER ON *.* TO 'vt_filtered'@'localhost'; + # User for general MySQL monitoring. + CREATE USER 'vt_monitoring'@'localhost'; + GRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD + ON *.* TO 'vt_monitoring'@'localhost'; + GRANT SELECT, UPDATE, DELETE, DROP + ON performance_schema.* TO 'vt_monitoring'@'localhost'; + # custom sql is used to add custom scripts like creating users/passwords. We use it in our tests # {{custom_sql}} diff --git a/go/cmd/vttablet/cli/cli.go b/go/cmd/vttablet/cli/cli.go index 3fb1e98877f..3b77b43d9cb 100644 --- a/go/cmd/vttablet/cli/cli.go +++ b/go/cmd/vttablet/cli/cli.go @@ -143,7 +143,6 @@ func run(cmd *cobra.Command, args []string) error { qsc, err := createTabletServer(ctx, env, config, ts, tabletAlias, srvTopoCounts) if err != nil { - ts.Close() return err } @@ -172,7 +171,6 @@ func run(cmd *cobra.Command, args []string) error { VDiffEngine: vdiff.NewEngine(ts, tablet, env.CollationEnv(), env.Parser()), } if err := tm.Start(tablet, config); err != nil { - ts.Close() return fmt.Errorf("failed to parse --tablet-path or initialize DB credentials: %w", err) } servenv.OnClose(func() { From dd9f22cce99a882e367fef2f4b12f0c725a73dcb Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 May 2024 13:39:44 +0530 Subject: [PATCH 09/19] test: also print show grants output to debug Signed-off-by: Manan Gupta --- go/vt/mysqlctl/mysqld.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 952c0987c82..be2cdd501bf 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -538,6 +538,7 @@ func (mysqld *Mysqld) WaitForDBAGrants(ctx context.Context, waitTime time.Durati if fetchErr != nil { log.Errorf("Error running SHOW GRANTS - %v", fetchErr) } + log.Errorf("SHOW GRANTS output - %v", res.Rows) if fetchErr == nil && res != nil && len(res.Rows) > 0 && len(res.Rows[0]) > 0 { privileges := res.Rows[0][0].ToString() // In MySQL 8.0, all the privileges are listed out explicitly, so we can search for SUPER in the output. From 42801f79166cc73df6b6a00c1b49daa5db99782a Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 May 2024 15:35:21 +0530 Subject: [PATCH 10/19] test: print more logs Signed-off-by: Manan Gupta --- go/vt/mysqlctl/mysqld.go | 3 +++ test/vtop_example.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index be2cdd501bf..f23e3bafec5 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -532,6 +532,9 @@ func (mysqld *Mysqld) WaitForDBAGrants(ctx context.Context, waitTime time.Durati defer cancel() for { conn, connErr := mysql.Connect(ctx, params) + if connErr != nil { + log.Errorf("Error connecting to mysql - %v", connErr) + } if connErr == nil { res, fetchErr := conn.ExecuteFetch("SHOW GRANTS", 1000, false) conn.Close() diff --git a/test/vtop_example.sh b/test/vtop_example.sh index 007634e4169..6883d41348d 100755 --- a/test/vtop_example.sh +++ b/test/vtop_example.sh @@ -71,6 +71,7 @@ function checkPodStatusWithTimeout() { vttabletPod=$(kubectl get pods | grep -E "vttablet" | head -n 1 | awk '{print $1}') kubectl describe pod $vttabletPod kubectl logs $vttabletPod -c vttablet + kubectl logs $vttabletPod -c mysqld exit 1 } From fb824dd4f4e3863088ba19b906d09ba6dcfb5d05 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 May 2024 17:00:45 +0530 Subject: [PATCH 11/19] test: print the cnf files Signed-off-by: Manan Gupta --- go/cmd/mysqlctld/cli/mysqlctld.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/go/cmd/mysqlctld/cli/mysqlctld.go b/go/cmd/mysqlctld/cli/mysqlctld.go index ee3fe241440..2734411746e 100644 --- a/go/cmd/mysqlctld/cli/mysqlctld.go +++ b/go/cmd/mysqlctld/cli/mysqlctld.go @@ -150,6 +150,9 @@ func run(cmd *cobra.Command, args []string) error { return fmt.Errorf("failed to refresh config: %w", err) } + printFile(cnf.Path) + printFile(cnf.Path + ".previous") + // check if we were interrupted during a previous restore if !mysqlctl.RestoreWasInterrupted(cnf) { if err := mysqld.Start(ctx, cnf); err != nil { @@ -190,3 +193,9 @@ func run(cmd *cobra.Command, args []string) error { return nil } + +func printFile(path string) { + log.Errorf("Printing file - %v", path) + bytes, err := os.ReadFile(path) + log.Errorf("Read - %v, %v", string(bytes), err) +} From d137ec4a26f1398af44e0b0ca678add6f8fd5458 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Mon, 3 Jun 2024 16:02:24 -0700 Subject: [PATCH 12/19] Trigger rebuild Signed-off-by: Rohit Nayak From 961eb4d9f36f6eed1cfe8c51152ccd6253483cc2 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 18 Jul 2024 15:52:33 +0530 Subject: [PATCH 13/19] feat: update kind version and the mysql image to be used Signed-off-by: Manan Gupta --- examples/operator/101_initial_cluster.yaml | 2 +- tools/get_kubectl_kind.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/operator/101_initial_cluster.yaml b/examples/operator/101_initial_cluster.yaml index 23369d060bd..4a4f478adac 100644 --- a/examples/operator/101_initial_cluster.yaml +++ b/examples/operator/101_initial_cluster.yaml @@ -15,7 +15,7 @@ spec: vtbackup: vitess/lite:latest vtorc: vitess/lite:latest mysqld: - mysql80Compatible: vitess/lite:latest + mysql80Compatible: mysql:8.0.30 mysqldExporter: prom/mysqld-exporter:v0.11.0 cells: - name: zone1 diff --git a/tools/get_kubectl_kind.sh b/tools/get_kubectl_kind.sh index 57df414fdd8..89b50b222b0 100755 --- a/tools/get_kubectl_kind.sh +++ b/tools/get_kubectl_kind.sh @@ -28,7 +28,7 @@ ln -sf "kubectl-${KUBE_VERSION}" kubectl if ! command -v kind &> /dev/null then echo "Downloading kind..." - curl -L https://kind.sigs.k8s.io/dl/v0.12.0/kind-linux-amd64 > "kind" + curl -L https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 > "kind" chmod +x "kind" echo "Installed kind" else From a4b0358620940c781a963341815c8d8d13c406ce Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 30 Oct 2024 15:12:12 -0600 Subject: [PATCH 14/19] Run the workflow on a self-hosted runner Signed-off-by: Florent Poinsard --- .github/workflows/vtop_example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml index d7deb2d89da..55dce4ca4fe 100644 --- a/.github/workflows/vtop_example.yml +++ b/.github/workflows/vtop_example.yml @@ -7,7 +7,7 @@ concurrency: jobs: build: name: VTop Example - runs-on: gh-hosted-runners-64cores-1 + runs-on: self-hosted steps: - name: Skip CI From 1a15326cf7054574604cb8b0443a299695b1c229 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 30 Oct 2024 15:15:35 -0600 Subject: [PATCH 15/19] Remove non-required steps Signed-off-by: Florent Poinsard --- .github/workflows/vtop_example.yml | 2 +- tools/get_kubectl_kind.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml index 55dce4ca4fe..3f63988541a 100644 --- a/.github/workflows/vtop_example.yml +++ b/.github/workflows/vtop_example.yml @@ -73,7 +73,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | # Install everything we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y eatmydata go mod download - name: Run make minimaltools diff --git a/tools/get_kubectl_kind.sh b/tools/get_kubectl_kind.sh index 89b50b222b0..169b120aaa0 100755 --- a/tools/get_kubectl_kind.sh +++ b/tools/get_kubectl_kind.sh @@ -12,7 +12,7 @@ source build.env mkdir -p "$VTROOT/bin" cd "$VTROOT/bin" -KUBE_VERSION="${KUBE_VERSION:-v1.21.1}" +KUBE_VERSION="${KUBE_VERSION:-v1.31.0}" KUBERNETES_RELEASE_URL="${KUBERNETES_RELEASE_URL:-https://dl.k8s.io}" # Download kubectl if needed. @@ -28,7 +28,7 @@ ln -sf "kubectl-${KUBE_VERSION}" kubectl if ! command -v kind &> /dev/null then echo "Downloading kind..." - curl -L https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 > "kind" + curl -L https://kind.sigs.k8s.io/dl/v0.22.0/kind-linux-amd64 > "kind" chmod +x "kind" echo "Installed kind" else From a7fdc8eceda95e294d3d1ecb9540e57672fbf2bd Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 30 Oct 2024 15:18:04 -0600 Subject: [PATCH 16/19] Simplify workflow Signed-off-by: Florent Poinsard --- .github/workflows/vtop_example.yml | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml index 3f63988541a..95dcf8dabda 100644 --- a/.github/workflows/vtop_example.yml +++ b/.github/workflows/vtop_example.yml @@ -60,9 +60,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v5 + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.22.3 + go-version-file: go.mod - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,19 +73,9 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | # Install everything we need, and configure - sudo apt-get install -y eatmydata + sudo apt-get install -y eatmydata make go mod download - - name: Run make minimaltools - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - make minimaltools - - - name: Build - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - make build - - name: Install kubectl & kind if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | From c5e95f893847356586928f7e9713d655d984a20c Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 30 Oct 2024 15:48:59 -0600 Subject: [PATCH 17/19] add vtctldclient to the workflow Signed-off-by: Florent Poinsard --- .github/workflows/vtop_example.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml index 95dcf8dabda..d322d998951 100644 --- a/.github/workflows/vtop_example.yml +++ b/.github/workflows/vtop_example.yml @@ -76,6 +76,12 @@ jobs: sudo apt-get install -y eatmydata make go mod download + # needed for vtctldclient + - name: Build vitess + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make build + - name: Install kubectl & kind if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | From e3ef5d708c5c23a9a243022f59980030ff55e0d4 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Wed, 30 Oct 2024 16:11:51 -0600 Subject: [PATCH 18/19] Self review + remove debug Signed-off-by: Florent Poinsard --- .github/workflows/vtop_example.yml | 6 ++++-- go/cmd/mysqlctld/cli/mysqlctld.go | 9 --------- go/vt/mysqlctl/mysqld.go | 4 ---- test/vtop_example.sh | 1 + 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml index d322d998951..fb5ae87c101 100644 --- a/.github/workflows/vtop_example.yml +++ b/.github/workflows/vtop_example.yml @@ -36,17 +36,18 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' - uses: actions/checkout@v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' - uses: dorny/paths-filter@v3.0.1 + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 id: changes with: token: '' filters: | end_to_end: - 'go/**/*.go' + - 'go/vt/sidecardb/**/*.sql' - 'test.go' - 'Makefile' - 'build.env' @@ -56,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - 'examples/**' + - 'test/**' - '.github/workflows/vtop_example.yml' - name: Set up Go diff --git a/go/cmd/mysqlctld/cli/mysqlctld.go b/go/cmd/mysqlctld/cli/mysqlctld.go index 2734411746e..ee3fe241440 100644 --- a/go/cmd/mysqlctld/cli/mysqlctld.go +++ b/go/cmd/mysqlctld/cli/mysqlctld.go @@ -150,9 +150,6 @@ func run(cmd *cobra.Command, args []string) error { return fmt.Errorf("failed to refresh config: %w", err) } - printFile(cnf.Path) - printFile(cnf.Path + ".previous") - // check if we were interrupted during a previous restore if !mysqlctl.RestoreWasInterrupted(cnf) { if err := mysqld.Start(ctx, cnf); err != nil { @@ -193,9 +190,3 @@ func run(cmd *cobra.Command, args []string) error { return nil } - -func printFile(path string) { - log.Errorf("Printing file - %v", path) - bytes, err := os.ReadFile(path) - log.Errorf("Read - %v, %v", string(bytes), err) -} diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index d6a6513d3c1..72c1d8f6658 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -534,16 +534,12 @@ func (mysqld *Mysqld) WaitForDBAGrants(ctx context.Context, waitTime time.Durati defer cancel() for { conn, connErr := mysql.Connect(ctx, params) - if connErr != nil { - log.Errorf("Error connecting to mysql - %v", connErr) - } if connErr == nil { res, fetchErr := conn.ExecuteFetch("SHOW GRANTS", 1000, false) conn.Close() if fetchErr != nil { log.Errorf("Error running SHOW GRANTS - %v", fetchErr) } - log.Errorf("SHOW GRANTS output - %v", res.Rows) if fetchErr == nil && res != nil && len(res.Rows) > 0 && len(res.Rows[0]) > 0 { privileges := res.Rows[0][0].ToString() // In MySQL 8.0, all the privileges are listed out explicitly, so we can search for SUPER in the output. diff --git a/test/vtop_example.sh b/test/vtop_example.sh index 6883d41348d..e4f7166c185 100755 --- a/test/vtop_example.sh +++ b/test/vtop_example.sh @@ -486,6 +486,7 @@ EOF waitForKeyspaceToBeServing customer 80- 1 } +kind delete cluster --name kind || true # Build the docker image for vitess/lite using the local code docker build -f docker/lite/Dockerfile -t vitess/lite:pr . From 5d12aae2ae1670416f4734b2fac5488af05d9896 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 28 Nov 2024 11:25:51 +0530 Subject: [PATCH 19/19] feat: remove debug only lines Signed-off-by: Manan Gupta --- test/vtop_example.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/vtop_example.sh b/test/vtop_example.sh index e4f7166c185..c537c0f844c 100755 --- a/test/vtop_example.sh +++ b/test/vtop_example.sh @@ -68,10 +68,6 @@ function checkPodStatusWithTimeout() { sleep 1 done echo -e "ERROR: checkPodStatusWithTimeout timeout to find pod matching:\ngot:\n$out\nfor regex: $regex" - vttabletPod=$(kubectl get pods | grep -E "vttablet" | head -n 1 | awk '{print $1}') - kubectl describe pod $vttabletPod - kubectl logs $vttabletPod -c vttablet - kubectl logs $vttabletPod -c mysqld exit 1 }