ci: install risedev to ci image (#16161)

ci/Dockerfile

@@ -64,3 +64,12 @@ RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari ca
     && rm -rf "/root/.cargo/git/db"
 RUN cargo install [email protected] [email protected]
 RUN cargo uninstall cargo-binstall cargo-cache
+
+# install risedev
+COPY <<EOF /root/.cargo/bin/risedev
+#!/usr/bin/env bash
+set -e
+cd "/risingwave"
+./risedev "\$@"
+EOF
+RUN chmod +x /root/.cargo/bin/risedev

ci/build-ci-image.sh

@@ -10,7 +10,7 @@ cat ../rust-toolchain
 # shellcheck disable=SC2155
 
 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml
-export BUILD_ENV_VERSION=v20240313
+export BUILD_ENV_VERSION=v20240405_1
 
 export BUILD_TAG="public.ecr.aws/w1p7b4n3/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -71,7 +71,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240313
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240405_1
     depends_on:
       - mysql
       - db
@@ -84,7 +84,7 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240313
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240405_1
     depends_on:
       - mysql
       - db
@@ -103,12 +103,12 @@ services:
 
 
   rw-build-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240313
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240405_1
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240313
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240405_1
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -119,7 +119,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240313
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240405_1
     depends_on:
       db:
         condition: service_healthy

ci/scripts/backwards-compat-test.sh

@@ -114,7 +114,7 @@ setup_old_cluster() {
   wget $OLD_URL
   if [[ "$?" -ne 0 ]]; then
     set -e
-    echo "Failed to download ${OLD_VERSION} from github releases, build from source later during ./risedev d"
+    echo "Failed to download ${OLD_VERSION} from github releases, build from source later during \`risedev d\`"
     configure_rw_build
   else
     set -e

ci/scripts/build-simulation.sh

@@ -9,14 +9,14 @@ echo "--- Generate RiseDev CI config"
 cp ci/risedev-components.ci.env risedev-components.user.env
 
 echo "--- Build deterministic simulation e2e test runner"
-cargo make sslt-build-all --profile ci-sim
+risedev sslt-build-all --profile ci-sim
 
 echo "--- Show sccache stats"
 sccache --show-stats
 sccache --zero-stats
 
 echo "--- Build and archive deterministic simulation integration tests"
-NEXTEST_PROFILE=ci-sim cargo make sarchive-it-test --cargo-profile ci-sim
+NEXTEST_PROFILE=ci-sim risedev sarchive-it-test --cargo-profile ci-sim
 
 echo "--- Show sccache stats"
 sccache --show-stats

ci/scripts/common.sh

@@ -84,8 +84,8 @@ function download_and_prepare_rw() {
 
   echo -e "\033[33mPrepare RiseDev dev cluster\033[0m"
 
-  cargo make pre-start-dev
-  cargo make --allow-private link-all-in-one-binaries
+  risedev pre-start-dev
+  risedev --allow-private link-all-in-one-binaries
 }
 
 function filter_stack_trace() {

ci/scripts/deterministic-unit-test.sh

@@ -9,4 +9,4 @@ echo "--- Generate RiseDev CI config"
 cp ci/risedev-components.ci.env risedev-components.user.env
 
 echo "--- Run unit tests in deterministic simulation mode"
-cargo make stest --no-fail-fast
+risedev stest --no-fail-fast

ci/scripts/e2e-cassandra-sink-test.sh

@@ -32,7 +32,7 @@ mkdir ./connector-node
 tar xf ./risingwave-connector.tar.gz -C ./connector-node
 
 echo "--- starting risingwave cluster"
-cargo make ci-start ci-sink-test
+risedev ci-start ci-sink-test
 sleep 1
 
 echo "--- create cassandra table"
@@ -62,4 +62,4 @@ fi
 
 echo "--- Kill cluster"
 cd ../../
-cargo make ci-kill
+risedev ci-kill

ci/scripts/e2e-clickhouse-sink-test.sh

@@ -24,7 +24,7 @@ shift $((OPTIND -1))
 download_and_prepare_rw "$profile" source
 
 echo "--- starting risingwave cluster"
-cargo make ci-start ci-sink-test
+risedev ci-start ci-sink-test
 sleep 1
 
 
@@ -57,4 +57,4 @@ else
 fi
 
 echo "--- Kill cluster"
-cargo make ci-kill
+risedev ci-kill

ci/scripts/e2e-deltalake-sink-rust-test.sh

@@ -32,7 +32,7 @@ mkdir ./connector-node
 tar xf ./risingwave-connector.tar.gz -C ./connector-node
 
 echo "--- starting risingwave cluster"
-cargo make ci-start ci-sink-test
+risedev ci-start ci-sink-test
 sleep 1
 
 # prepare minio deltalake sink
@@ -76,4 +76,4 @@ else
 fi
 
 echo "--- Kill cluster"
-cargo make ci-kill
+risedev ci-kill

ci/scripts/e2e-doris-sink-test.sh

@@ -24,7 +24,7 @@ shift $((OPTIND -1))
 download_and_prepare_rw "$profile" source
 
 echo "--- starting risingwave cluster"
-cargo make ci-start ci-sink-test
+risedev ci-start ci-sink-test
 sleep 1
 
 echo "--- create doris table"
@@ -56,4 +56,4 @@ else
 fi
 
 echo "--- Kill cluster"
-cargo make ci-kill
+risedev ci-kill

ci/scripts/e2e-iceberg-cdc.sh

@@ -36,7 +36,7 @@ echo "--- e2e, ci-1cn-1fe, iceberg cdc"
 echo "--- starting risingwave cluster"
 
 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
-cargo make ci-start ci-1cn-1fe-with-recovery
+risedev ci-start ci-1cn-1fe-with-recovery
 
 # prepare minio iceberg sink
 echo "--- preparing iceberg"

ci/scripts/e2e-iceberg-sink-test.sh

@@ -33,7 +33,7 @@ tar xf ./risingwave-connector.tar.gz -C ./connector-node
 
 echo "--- starting risingwave cluster"
 mkdir -p .risingwave/log
-cargo make ci-start ci-iceberg-test
+risedev ci-start ci-iceberg-test
 sleep 1
 
 # prepare minio iceberg sink
@@ -82,4 +82,4 @@ else
 fi
 
 echo "--- Kill cluster"
-cargo make ci-kill
+risedev ci-kill

ci/scripts/e2e-iceberg-sink-v2-test.sh

@@ -25,7 +25,7 @@ download_and_prepare_rw "$profile" source
 
 echo "--- starting risingwave cluster"
 mkdir -p .risingwave/log
-cargo make ci-start ci-iceberg-test
+risedev ci-start ci-iceberg-test
 sleep 1
 
 # prepare minio iceberg sink
@@ -49,4 +49,4 @@ bash ./start_spark_connect_server.sh
 
 echo "--- Kill cluster"
 cd ../../
-cargo make ci-kill
+risedev ci-kill

ci/scripts/e2e-pulsar-sink-test.sh

@@ -21,7 +21,7 @@ shift $((OPTIND -1))
 download_and_prepare_rw "$profile" source
 
 echo "--- starting risingwave cluster"
-cargo make ci-start ci-sink-test
+risedev ci-start ci-sink-test
 sleep 1
 
 echo "--- waiting until pulsar is healthy"
@@ -44,4 +44,4 @@ sqllogictest -p 4566 -d dev './e2e_test/sink/pulsar_sink.slt'
 sleep 1
 
 echo "--- Kill cluster"
-cargo make ci-kill
+risedev ci-kill

ci/scripts/e2e-redis-sink-test.sh

@@ -24,7 +24,7 @@ shift $((OPTIND -1))
 download_and_prepare_rw "$profile" source
 
 echo "--- starting risingwave cluster"
-cargo make ci-start ci-sink-test
+risedev ci-start ci-sink-test
 apt-get update -y && apt-get install -y redis-server
 sleep 1
 
@@ -45,4 +45,4 @@ else
 fi
 
 echo "--- Kill cluster"
-cargo make ci-kill
+risedev ci-kill

ci/scripts/e2e-sink-test.sh

@@ -54,7 +54,7 @@ psql -h db -U postgres -d test -c "create table t5 (v1 smallint primary key, v2
 psql -h db -U postgres -d test < ./e2e_test/sink/remote/pg_create_table.sql
 
 echo "--- starting risingwave cluster"
-cargo make ci-start ci-1cn-1fe
+risedev ci-start ci-1cn-1fe
 
 echo "--- testing common sinks"
 sqllogictest -p 4566 -d dev './e2e_test/sink/append_only_sink.slt'
@@ -118,13 +118,13 @@ else
 fi
 
 echo "--- Kill cluster"
-cargo make ci-kill
+risedev ci-kill
 
 echo "--- e2e, ci-1cn-1fe, nexmark endless"
 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
-cargo make ci-start ci-1cn-1fe
+risedev ci-start ci-1cn-1fe
 sqllogictest -p 4566 -d dev './e2e_test/source/nexmark_endless_mvs/*.slt'
 sqllogictest -p 4566 -d dev './e2e_test/source/nexmark_endless_sinks/*.slt'
 
 echo "--- Kill cluster"
-cargo make ci-kill
+risedev ci-kill

ci/scripts/e2e-source-test.sh

@@ -59,7 +59,7 @@ psql < ./e2e_test/source/cdc/postgres_cdc.sql
 
 echo "--- starting risingwave cluster"
 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
-cargo make ci-start ci-1cn-1fe-with-recovery
+risedev ci-start ci-1cn-1fe-with-recovery
 
 echo "--- mongodb cdc test"
 # install the mongo shell
@@ -107,7 +107,7 @@ sleep 10
 sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check.slt'
 
 # kill cluster
-cargo make kill
+risedev kill
 echo "> cluster killed "
 
 echo "--- mysql & postgres recovery check"
@@ -130,7 +130,7 @@ echo "> inserted new rows into postgres"
 unset RISINGWAVE_CI
 export RUST_LOG="events::stream::message::chunk=trace,risingwave_stream=debug,risingwave_batch=info,risingwave_storage=info" \
 
-cargo make dev ci-1cn-1fe-with-recovery
+risedev dev ci-1cn-1fe-with-recovery
 echo "> wait for cluster recovery finish"
 sleep 20
 echo "> check mviews after cluster recovery"
@@ -141,12 +141,12 @@ sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check_new_rows.slt'
 sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc_share_stream_drop.slt'
 
 echo "--- Kill cluster"
-cargo make ci-kill
+risedev ci-kill
 
 echo "--- e2e, ci-1cn-1fe, protobuf schema registry"
 export RISINGWAVE_CI=true
 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
-cargo make ci-start ci-1cn-1fe
+risedev ci-start ci-1cn-1fe
 python3 -m pip install requests protobuf confluent-kafka
 python3 e2e_test/schema_registry/pb.py "message_queue:29092" "http://message_queue:8081" "sr_pb_test" 20 user
 echo "make sure google/protobuf/source_context.proto is NOT in schema registry"
@@ -157,11 +157,11 @@ sqllogictest -p 4566 -d dev './e2e_test/schema_registry/pb.slt'
 sqllogictest -p 4566 -d dev './e2e_test/schema_registry/alter_sr.slt'
 
 echo "--- Kill cluster"
-cargo make ci-kill
+risedev ci-kill
 
 echo "--- e2e, ci-kafka-plus-pubsub, kafka and pubsub source"
 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
-cargo make ci-start ci-pubsub
+risedev ci-start ci-pubsub
 ./scripts/source/prepare_ci_kafka.sh
 cargo run --bin prepare_ci_pubsub
 sqllogictest -p 4566 -d dev './e2e_test/source/basic/*.slt'
@@ -178,5 +178,5 @@ echo "--- e2e, kafka alter source again"
 sqllogictest -p 4566 -d dev './e2e_test/source/basic/alter/kafka_after_new_data_2.slt'
 
 echo "--- Run CH-benCHmark"
-./risedev slt -p 4566 -d dev './e2e_test/ch_benchmark/batch/ch_benchmark.slt'
-./risedev slt -p 4566 -d dev './e2e_test/ch_benchmark/streaming/*.slt'
+risedev slt -p 4566 -d dev './e2e_test/ch_benchmark/batch/ch_benchmark.slt'
+risedev slt -p 4566 -d dev './e2e_test/ch_benchmark/streaming/*.slt'

ci/scripts/e2e-sql-migration.sh

@@ -49,16 +49,16 @@ download_and_prepare_rw "$profile" common
 
 echo "--- starting risingwave cluster, ci-1cn-1fe-with-recovery"
 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
-cargo make ci-start ci-1cn-1fe-with-recovery
+risedev ci-start ci-1cn-1fe-with-recovery
 
 echo "--- init cluster with some data & DDL"
 sqllogictest -d dev -h localhost -p 4566 './e2e_test/sql_migration/prepare.slt'
 
 echo "--- kill cluster"
-cargo make ci-kill
+risedev ci-kill
 
 echo "--- restart etcd"
-cargo make dev ci-meta-etcd-for-migration
+risedev dev ci-meta-etcd-for-migration
 
 echo "--- run migration"
 mkdir -p "${RW_PREFIX_DATA}/sqlite/"
@@ -70,10 +70,10 @@ migration \
 -f
 
 echo "--- kill etcd"
-cargo make ci-kill
+risedev ci-kill
 
 echo "--- starting risingwave cluster, meta-1cn-1fe-sqlite"
-cargo make dev meta-1cn-1fe-sqlite
+risedev dev meta-1cn-1fe-sqlite
 
 echo "--- wait for recovery"
 wait_for_recovery
@@ -82,8 +82,8 @@ echo "--- run check"
 sqllogictest -d dev -h localhost -p 4566 './e2e_test/sql_migration/check.slt'
 
 echo "--- kill cluster"
-cargo make kill
+risedev kill
 
 echo "--- clean data"
-cargo make clean-data
+risedev clean-data
 

ci/scripts/e2e-starrocks-sink-test.sh

@@ -24,7 +24,7 @@ shift $((OPTIND -1))
 download_and_prepare_rw "$profile" source
 
 echo "--- starting risingwave cluster"
-cargo make ci-start ci-sink-test
+risedev ci-start ci-sink-test
 sleep 1
 
 
@@ -55,4 +55,4 @@ else
 fi
 
 echo "--- Kill cluster"
-cargo make ci-kill
+risedev ci-kill