benchmark on a100

InternLM · Dec 4, 2023 · 717f058 · 717f058
1 parent 4811d84
commit 717f058
Show file tree

Hide file tree

Showing 5 changed files with 150 additions and 7 deletions.
diff --git a/benchmark/benchmark-7b.sh b/benchmark/benchmark-7b.sh
@@ -1,4 +1,9 @@
 #!/bin/bash
+if [ - z "$1" ]
+then
+    echo
+fi
+
 tp=1
 model_name=llama2
 model_path=/workspace/models-140/llama2/huggingface/llama-2-7b-chat/
@@ -16,12 +21,12 @@ apt-get install crudini
 crudini --set ${config_path} llama max_context_token_num 4
 crudini --set ${config_path} llama cache_chunk_size -1
 crudini --set ${config_path} llama cache_max_entry_count 1000
-crudini --set ${config_path} llama max_batch_size 256
+crudini --set ${config_path} llama max_batch_size 128
 # end of update config
 
 benchmark_rpm () {
     output_path=$1
-    mkdir -p ${output_path}
+    mkdir -p "${output_path}"
 
     batches=(64 128)
     for batch in "${batches[@]}"
@@ -40,6 +45,8 @@ benchmark_rpm () {
 
 benchmark_generation () {
     output_path=$1
+    mkdir -p "${output_path}"
+
     python3 benchmark/profile_generation.py \
     ${turbomind_model_path} \
     --concurrency 1 16 32 64 \

diff --git a/benchmark/benchmark_13b.sh b/benchmark/benchmark_13b.sh
@@ -16,12 +16,12 @@ apt-get install crudini
 crudini --set ${config_path} llama max_context_token_num 4
 crudini --set ${config_path} llama cache_chunk_size -1
 crudini --set ${config_path} llama cache_max_entry_count 500
-crudini --set ${config_path} llama max_batch_size 256
+crudini --set ${config_path} llama max_batch_size 128
 # end of update config
 
 benchmark_rpm () {
     output_path=$1
-    mkdir -p ${output_path}
+    mkdir -p "${output_path}"
 
     batches=(64 128)
     for batch in "${batches[@]}"
@@ -40,6 +40,8 @@ benchmark_rpm () {
 
 benchmark_generation () {
     output_path=$1
+    mkdir -p "${output_path}"
+
     python3 benchmark/profile_generation.py \
     ${turbomind_model_path} \
     --concurrency 1 16 32 64 \

diff --git a/benchmark/benchmark_20b.sh b/benchmark/benchmark_20b.sh
@@ -16,12 +16,12 @@ apt-get install crudini
 crudini --set ${config_path} llama max_context_token_num 4
 crudini --set ${config_path} llama cache_chunk_size -1
 crudini --set ${config_path} llama cache_max_entry_count 700
-crudini --set ${config_path} llama max_batch_size 256
+crudini --set ${config_path} llama max_batch_size 128
 # end of update config
 
 benchmark_rpm () {
     output_path=$1
-    mkdir -p ${output_path}
+    mkdir -p "${output_path}"
 
     batches=(64 128)
     for batch in "${batches[@]}"
@@ -40,6 +40,8 @@ benchmark_rpm () {
 
 benchmark_generation () {
     output_path=$1
+    mkdir -p "${output_path}"
+
     python3 benchmark/profile_generation.py \
     ${turbomind_model_path} \
     --concurrency 1 16 32 64 \

diff --git a/benchmark/benchmark_70b.sh b/benchmark/benchmark_70b.sh
@@ -21,7 +21,7 @@ crudini --set ${config_path} llama max_batch_size 256
 
 benchmark_rpm () {
     output_path=$1
-    mkdir -p ${output_path}
+    mkdir -p "${output_path}"
 
     batches=(64 128 256)
     for batch in "${batches[@]}"
@@ -40,6 +40,8 @@ benchmark_rpm () {
 
 benchmark_generation () {
     output_path=$1
+    mkdir -p "${output_path}"
+
     python3 benchmark/profile_generation.py \
     ${turbomind_model_path} \
     --concurrency 1 64 128 256 \