diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts
index f884d512d4f..025580b5e8e 100644
--- a/catalogs/catalog-jdbc-doris/build.gradle.kts
+++ b/catalogs/catalog-jdbc-doris/build.gradle.kts
@@ -70,7 +70,7 @@ tasks {
 tasks.test {
   val skipUTs = project.hasProperty("skipTests")
   doFirst {
-    environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.3")
+    environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.4")
   }
 
   if (skipUTs) {
diff --git a/dev/docker/doris/Dockerfile b/dev/docker/doris/Dockerfile
index 90016b11104..8b3acc9af5b 100644
--- a/dev/docker/doris/Dockerfile
+++ b/dev/docker/doris/Dockerfile
@@ -42,7 +42,8 @@ RUN ARCH=$(uname -m) && \
 #################################################################################
 ## add files
 ADD packages/doris-${TARGETARCH}.tar.xz /opt/
-RUN ln -s /opt/apache-doris-${DORIS_VERSION}-bin-* ${DORIS_HOME} 
+RUN ln -s /opt/apache-doris-${DORIS_VERSION}-bin-* ${DORIS_HOME} && \
+    chmod 755 "${DORIS_BE_HOME}/lib/doris_be"
 
 COPY start.sh ${DORIS_HOME}
 
diff --git a/dev/docker/doris/start.sh b/dev/docker/doris/start.sh
index 64971c88aa0..1a24ea7dfc5 100644
--- a/dev/docker/doris/start.sh
+++ b/dev/docker/doris/start.sh
@@ -30,6 +30,9 @@ DORIS_BE_SCRIPT="${DORIS_BE_HOME}/bin/start_be.sh"
 sed -i '/Please set vm.max_map_count/,/exit 1/{s/exit 1/#exit 1\n        echo "skip this"/}' ${DORIS_BE_SCRIPT}
 sed -i '/Please set the maximum number of open file descriptors/,/exit 1/{s/exit 1/#exit 1\n        echo "skip this"/}' ${DORIS_BE_SCRIPT}
 
+# remove chmod command in start_be.sh, it will cost a lot of time. Add it to Dockerfile
+sed -i 's/chmod 755 "${DORIS_HOME}\/lib\/doris_be"/#&/' ${DORIS_BE_SCRIPT}
+
 # update fe.conf & be.conf, set priority_networks
 CONTAINER_IP=$(hostname -i)
 PRIORITY_NETWORKS=$(echo "${CONTAINER_IP}" | awk -F '.' '{print$1"."$2"."$3".0/24"}')
diff --git a/docs/docker-image-details.md b/docs/docker-image-details.md
index cfe9da76824..ca2313f0188 100644
--- a/docs/docker-image-details.md
+++ b/docs/docker-image-details.md
@@ -213,6 +213,9 @@ You can use this image to test Apache Doris.
 
 Changelog
 
+- gravitino-ci-doris:0.1.4
+  - remove chmod in start.sh to accelerate the startup speed
+
 - gravitino-ci-doris:0.1.3
   - To adapt to the CI framework, don't exit container when start failed, logs are no longer printed to stdout. 
   - Add `report_disk_state_interval_seconds` config to decrease report interval.
diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts
index 8e31c3854a1..9baca46ceb6 100644
--- a/integration-test/build.gradle.kts
+++ b/integration-test/build.gradle.kts
@@ -146,7 +146,7 @@ tasks.test {
       environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.12")
       environment("GRAVITINO_CI_TRINO_DOCKER_IMAGE", "datastrato/gravitino-ci-trino:0.1.5")
       environment("GRAVITINO_CI_KAFKA_DOCKER_IMAGE", "apache/kafka:3.7.0")
-      environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.3")
+      environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.4")
       environment("GRAVITINO_CI_RANGER_DOCKER_IMAGE", "datastrato/gravitino-ci-ranger:0.1.0")
 
       copy {