add parameter sweeping

Author: KuntaiDu

benchmarks/disagg_benchmarks/analyze_benchmark_result.py

@@ -0,0 +1,47 @@
+
+import argparse
+import json
+import yaml
+import os
+from pathlib import Path
+
+def load(path):
+    
+    with open(str(path), 'r') as f:
+        return json.loads(f.read())
+
+def main(args):
+
+    results = Path(args.results_folder)
+
+    chunk = load(results / "chunked_prefill_tp4.json")
+    prefill = load(results / "disagg_prefill_tp4.json")
+    decode = load(results / "disagg_decode_tp4.json")
+
+    ttft_ratio = chunk["mean_ttft_ms"] / prefill["mean_ttft_ms"]
+    itl_ratio = chunk["mean_itl_ms"] / decode["mean_itl_ms"]
+    prefill_decode_ratio = prefill["mean_ttft_ms"] / (decode["mean_itl_ms"] * args.output_len)
+    
+    with open(results / args.output_file, 'a') as f:
+        f.write(yaml.dump([{
+            'qps': args.qps,
+            'output_len': args.output_len,
+            'prefill_decode_ratio': prefill_decode_ratio,
+            'ttft_ratio': ttft_ratio,
+            'itl_ratio': itl_ratio,
+            "chunk_ttft": chunk["mean_ttft_ms"],
+            "chunk_itl": chunk["mean_itl_ms"],
+            "disagg_ttft": prefill["mean_ttft_ms"],
+            "disagg_itl": decode["mean_itl_ms"]
+        }]))
+    
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Analyze benchmark results")
+    parser.add_argument("--results-folder", required=True, help="Path to the results folder")
+    parser.add_argument("--output-len", type=int, required=True, help="Target output length")
+    parser.add_argument("--qps", type=int, required=True, help="Target QPS")
+    parser.add_argument("--output-file", type=str, default="chunk_vs_disagg.yaml")
+
+    args = parser.parse_args()
+    main(args)

benchmarks/disagg_benchmarks/disagg_benchmark.sh

@@ -41,86 +41,31 @@ wait_for_server() {
 }
 
 
-main() {
-
-  (which wget && which curl) || (apt-get update && apt-get install -y wget curl)
-  (which jq) || (apt-get -y install jq)
-  (which socat) || (apt-get -y install socat)
+benchmark() {
 
-  cd "$(dirname "$0")"
+  # compare chunked prefill with disaggregated prefill
 
-  cd ..
-  # create sonnet-4x.txt
-  echo "" > sonnet_4x.txt
-  for _ in {1..4}
-  do
-    cat sonnet.txt >> sonnet_4x.txt
-  done
-  cd disagg_benchmarks
-
-
-  mkdir -p results
   results_folder="./results"
   model="neuralmagic/Meta-Llama-3-70B-Instruct-FP8"
   dataset_name="sonnet"
   dataset_path="../sonnet_4x.txt"
   num_prompts=500
-  qps=4
+  qps=$1
   prefix_len=64
   input_len=2048
-  output_len=11
-
-
-  # chunked prefill with tp=8
-  python3 -m vllm.entrypoints.openai.api_server \
-          --model $model \
-          --port 8000 \
-          -tp 8 \
-          --disable-log-stats \
-          --disable-log-requests \
-          --enable-chunked-prefill &
-  wait_for_server 8000
-
-  python3 ../benchmark_serving.py \
-          --backend vllm \
-          --model $model \
-          --dataset-name $dataset_name \
-          --dataset-path $dataset_path \
-          --sonnet-input-len $input_len \
-          --sonnet-output-len $output_len \
-          --sonnet-prefix-len $prefix_len \
-          --num-prompts $num_prompts \
-          --port 8000 \
-          --save-result \
-          --result-dir $results_folder \
-          --result-filename chunked_prefill_tp8.json \
-          --request-rate $qps
-  kill_gpu_processes
+  output_len=$2
 
 
   # chunked prefill with tp=4
   CUDA_VISIBLE_DEVICES=0,1,2,3 python3 \
     -m vllm.entrypoints.openai.api_server \
     --model $model \
-    --port 8100 \
+    --port 8000 \
     -tp 4 \
     --disable-log-stats \
     --disable-log-requests \
     --enable-chunked-prefill &
-
-  # CUDA_VISIBLE_DEVICES=4,5,6,7 python3 \
-  #   -m vllm.entrypoints.openai.api_server \
-  #   --model $model \
-  #   --port 8200 \
-  #   -tp 4 \
-  #   --disable-log-stats \
-  #   --disable-log-requests \
-  #   --enable-chunked-prefill &
-
-  wait_for_server 8100
-  # wait_for_server 8200
-  # # launch round robin proxy
-  # bash ./round_robin_proxy.sh &
+  wait_for_server 8000
 
   python3 ../benchmark_serving.py \
           --backend vllm \
@@ -131,17 +76,15 @@ main() {
           --sonnet-output-len $output_len \
           --sonnet-prefix-len $prefix_len \
           --num-prompts $((num_prompts / 2)) \
-          --port 8100 \
+          --port 8000 \
           --save-result \
           --result-dir $results_folder \
           --result-filename chunked_prefill_tp4.json \
           --request-rate $((qps / 2))
   kill_gpu_processes
-  # pkill -f round_robin_proxy.sh
 
 
   # disaggregated prefill
-
   # prefill with tp=4
   python3 -m vllm.entrypoints.openai.api_server \
           --model $model \
@@ -150,7 +93,6 @@ main() {
           --disable-log-stats \
           --disable-log-requests &
   wait_for_server 8000
-
   # set output-len to 1 so that it only do prefilling
   python3 ../benchmark_serving.py \
           --backend vllm \
@@ -177,7 +119,6 @@ main() {
           --disable-log-stats \
           --disable-log-requests &
   wait_for_server 8000
-
   # skip prefilling 
   # by enabling APC and force the input tokens be the same
   python3 ../benchmark_serving.py \
@@ -187,7 +128,7 @@ main() {
           --dataset-path $dataset_path \
           --sonnet-input-len $input_len \
           --sonnet-output-len $output_len \
-          --sonnet-prefix-len $((input_len - 1))  \
+          --sonnet-prefix-len $input_len  \
           --num-prompts $num_prompts \
           --port 8000 \
           --save-result \
@@ -196,7 +137,48 @@ main() {
           --request-rate $qps
   kill_gpu_processes
 
+  python3 analyze_results.py \
+          --results-folder $results_folder \
+          --output-len $output_len \
+          --qps $qps
+
+}
+
+
+main() {
+
+  (which wget && which curl) || (apt-get update && apt-get install -y wget curl)
+  (which jq) || (apt-get -y install jq)
+  (which socat) || (apt-get -y install socat)
+
+  cd "$(dirname "$0")"
+
+  cd ..
+  # create sonnet-4x.txt
+  echo "" > sonnet_4x.txt
+  for _ in {1..4}
+  do
+    cat sonnet.txt >> sonnet_4x.txt
+  done
+  cd disagg_benchmarks
+
+  rm -rf results
+  mkdir results
+
+  default_qps=4
+  default_output_len=12
+
+  for target_qps in 1 2 4 8 16
+  do
+    benchmark $target_qps $default_output_len
+  done
+
+  for output_len in 5 10 20 40 80
+  do
+    benchmark $default_qps $output_len
+  done
+
 }
 
 
-main "$@"
+main "$@"