Add batch size assert

triton-inference-server · Mar 6, 2024 · 7df09ef · 7df09ef
1 parent 7827fee
commit 7df09ef
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/src/c++/perf_analyzer/command_line_parser.cc b/src/c++/perf_analyzer/command_line_parser.cc
@@ -1942,6 +1942,9 @@ CLParser::VerifyOptions()
     if (!params_->async) {
       Usage("Only async mode is currently supported for OpenAI service-kind");
     }
+    if (params_->batch_size != 1) {
+      Usage("Batching is not currently supported with OpenAI service-kind");
+    }
   }
 
   if (params_->should_collect_metrics &&