From 8f9878966468664b39263db7f7d0dbb09db09ecb Mon Sep 17 00:00:00 2001
From: Olga Andreeva <124622579+oandreeva-nv@users.noreply.github.com>
Date: Thu, 1 Feb 2024 09:56:17 -0800
Subject: [PATCH] Adding OpenTelemetry Batch Span Processor (#6842)

Adding OpenTelemetry Batch Span Processor
---------

Co-authored-by: Theo Clark <theoclark101@gmail.com>
Co-authored-by: Ryan McCormick <rmccormick@nvidia.com>
---
 docs/user_guide/trace.md              |  82 +++++++++++++++-
 qa/L0_cmdline_trace/test.sh           |  98 ++++++++++++++++++-
 qa/L0_trace/opentelemetry_unittest.py |  29 +++---
 qa/L0_trace/test.sh                   | 130 +++++++++++++++++++++++++-
 src/command_line_parser.cc            | 119 ++++++++++++++++++-----
 src/command_line_parser.h             |   7 +-
 src/tracer.cc                         |  78 ++++++++++------
 src/tracer.h                          |  22 ++++-
 8 files changed, 493 insertions(+), 72 deletions(-)

diff --git a/docs/user_guide/trace.md b/docs/user_guide/trace.md
index da623fab30..1abe019ea1 100644
--- a/docs/user_guide/trace.md
+++ b/docs/user_guide/trace.md
@@ -1,5 +1,5 @@
 <!--
-# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -456,6 +456,46 @@ flag as follows:
 $ tritonserver --trace-config mode=opentelemetry \
     --trace-config opentelemetry,url=<endpoint> ...
 ```
+
+Triton's OpenTelemetry trace mode uses
+[Batch Span Processor](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#batch-span-processor),
+which batches ended spans and sends them in bulk. Batching helps
+with data compression and reduces the number of outgoing connections
+required to transmit the data. This processor supports both size and
+time based batching. Size-based batching is controlled by 2 parameters:
+`bsp_max_export_batch_size` and `bsp_max_queue_size`, while time-based batching
+is controlled by `bsp_schedule_delay`. Collected spans will be exported when
+the batch size reaches `bsp_max_export_batch_size`, or delay since last export
+reaches `bsp_schedule_delay`, whatever comes first. Additionally, user should
+make sure that `bsp_max_export_batch_size` is always less than
+`bsp_max_queue_size`, otherwise the excessive spans will be dropped
+and trace data will be lost.
+
+Default parameters for the Batch Span Processor are provided in
+[`OpenTelemetry trace APIs settings`](#opentelemetry-trace-apis-settings).
+As a general recommendation, make sure that `bsp_max_queue_size` is large enough
+to hold all collected spans, and `bsp_schedule_delay` does not cause frequent
+exports, which will affect Triton Server's latency. A minimal Triton trace
+consists of 3 spans: top level span, model span, and compute span.
+
+* __Top level span__: The top-level span collects timestamps for when
+request was received by Triton, and when the response was sent. Any Triton
+trace contains only 1 top level span.
+* __Model span__: Model spans collect information, when request for
+this model was started, when it was placed in a queue, and when it was ended.
+A minimal Triton trace contains 1 model span.
+* __Compute span__: Compute spans record compute timestamps. A minimal
+Triton trace contains 1 compute span.
+
+The total amount of spans depends on the complexity of your model.
+A general rule is any base model - a single model that performs computations -
+produces 1 model span and one compute span. For ensembles, every composing
+model produces model and compute spans in addition to one model span for the
+ensemble. [BLS](#tracing-for-bls-models) models produce the same number of
+model and compute spans as the total amount of models involved in the BLS request,
+including the main BLS model.
+
+
 ### Differences in trace contents from Triton's trace [output](#json-trace-output)
 
 OpenTelemetry APIs produce [spans](https://opentelemetry.io/docs/concepts/observability-primer/#spans)
@@ -509,6 +549,46 @@ The following table shows available OpenTelemetry trace APIs settings for
       environment variable.
     </td>
     </tr>
+    <tr>
+    <td><a href="https://opentelemetry.io/docs/specs/otel/trace/sdk/#batching-processor">
+      Batch Span Processor</a>
+    </td>
+    <td></td><td></td>
+    </tr>
+    <tr>
+    <td><code>bsp_max_queue_size</code></td>
+    <td align="center">2048</td>
+    <td>
+      Maximum queue size. <br/>
+      This setting can also be specified through <br/>
+      <a href="https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#batch-span-processor">
+      OTEL_BSP_MAX_QUEUE_SIZE</a>
+      environment variable.
+    </td>
+    </tr>
+    <tr>
+    <td><code>bsp_schedule_delay</code></td>
+    <td align="center">5000</td>
+    <td>
+      Delay interval (in milliseconds) between two consecutive exports. <br/>
+      This setting can also be specified through <br/>
+      <a href="https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#batch-span-processor">
+      OTEL_BSP_SCHEDULE_DELAY</a>
+      environment variable.
+    </td>
+    </tr>
+    <tr>
+    <td><code>bsp_max_export_batch_size</code></td>
+    <td align="center">512</td>
+    <td>
+      Maximum batch size. Must be less than or equal to
+      <code>bsp_max_queue_size</code>.<br/>
+      This setting can also be specified through <br/>
+      <a href="https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#batch-span-processor">
+      OTEL_BSP_MAX_EXPORT_BATCH_SIZE</a>
+      environment variable.
+    </td>
+    </tr>
   </tbody>
 </table>
 
diff --git a/qa/L0_cmdline_trace/test.sh b/qa/L0_cmdline_trace/test.sh
index 66f9a08fc0..d0f86dc2a9 100755
--- a/qa/L0_cmdline_trace/test.sh
+++ b/qa/L0_cmdline_trace/test.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -25,6 +25,19 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+# ============================= Helpers =======================================
+function assert_server_startup_failed() {
+  if [ "$SERVER_PID" != "0" ]; then
+      echo -e "\n***\n***Fail: Server start should have failed $SERVER\n***"
+      cat $SERVER_LOG
+      set -e
+      kill $SERVER_PID
+      wait $SERVER_PID
+      set +e
+      exit 1
+  fi
+}
+
 TRACE_SUMMARY=../common/trace_summary.py
 CLIENT_SCRIPT=trace_client.py
 
@@ -618,11 +631,92 @@ set -e
 kill $SERVER_PID
 wait $SERVER_PID
 
+set +e
+
+################################################################################
+# The following set of tests checks that tritonserver gracefully handles       #
+# bad OpenTelemetry BatchSpanProcessor parameters, provided through            #
+# environment variables, or tritonserver's options.                            #
+################################################################################
+export OTEL_BSP_MAX_QUEUE_SIZE="bad_value"
+
+SERVER_ARGS="--trace-config mode=opentelemetry --model-repository=$MODELSDIR"
+SERVER_LOG="./inference_server_trace_config_flag.log"
+run_server
+assert_server_startup_failed
+
+if [ `grep -c "Bad option: \"OTEL_BSP_MAX_QUEUE_SIZE\"" $SERVER_LOG` != "1" ]; then
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Test Failed\n***"
+    RET=1
+fi
+
+unset OTEL_BSP_MAX_QUEUE_SIZE
+
+export OTEL_BSP_SCHEDULE_DELAY="bad_value"
+run_server
+assert_server_startup_failed
+
+if [ `grep -c "Bad option: \"OTEL_BSP_SCHEDULE_DELAY\"" $SERVER_LOG` != "1" ]; then
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Test Failed\n***"
+    RET=1
+fi
+
+unset OTEL_BSP_SCHEDULE_DELAY
+
+export OTEL_BSP_MAX_EXPORT_BATCH_SIZE="bad_value"
+run_server
+assert_server_startup_failed
+
+if [ `grep -c "Bad option: \"OTEL_BSP_MAX_EXPORT_BATCH_SIZE\"" $SERVER_LOG` != "1" ]; then
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Test Failed\n***"
+    RET=1
+fi
+
+unset OTEL_BSP_MAX_EXPORT_BATCH_SIZE
+
+SERVER_ARGS="--model-repository=$MODELSDIR --trace-config mode=opentelemetry \
+             --trace-config opentelemetry,bsp_max_queue_size=bad_value"
+SERVER_LOG="./inference_server_trace_config_flag.log"
+run_server
+assert_server_startup_failed
+
+if [ `grep -c "Bad option: \"--trace-config opentelemetry,bsp_max_queue_size\"" $SERVER_LOG` != "1" ]; then
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Test Failed\n***"
+    RET=1
+fi
+
+SERVER_ARGS="--model-repository=$MODELSDIR --trace-config mode=opentelemetry \
+             --trace-config opentelemetry,bsp_schedule_delay=bad_value"
+SERVER_LOG="./inference_server_trace_config_flag.log"
+run_server
+assert_server_startup_failed
+
+if [ `grep -c "Bad option: \"--trace-config opentelemetry,bsp_schedule_delay\"" $SERVER_LOG` != "1" ]; then
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Test Failed\n***"
+    RET=1
+fi
+
+SERVER_ARGS="--model-repository=$MODELSDIR --trace-config mode=opentelemetry \
+             --trace-config opentelemetry,bsp_max_export_batch_size=bad_value"
+SERVER_LOG="./inference_server_trace_config_flag.log"
+run_server
+assert_server_startup_failed
+
+if [ `grep -c "Bad option: \"--trace-config opentelemetry,bsp_max_export_batch_size\"" $SERVER_LOG` != "1" ]; then
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Test Failed\n***"
+    RET=1
+fi
+
 if [ $RET -eq 0 ]; then
     echo -e "\n***\n*** Test Passed\n***"
 else
     echo -e "\n***\n*** Test FAILED\n***"
 fi
 
-
 exit $RET
diff --git a/qa/L0_trace/opentelemetry_unittest.py b/qa/L0_trace/opentelemetry_unittest.py
index 0e211b76b6..41ec1d0024 100644
--- a/qa/L0_trace/opentelemetry_unittest.py
+++ b/qa/L0_trace/opentelemetry_unittest.py
@@ -1,4 +1,4 @@
-# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -250,9 +250,9 @@ def _verify_contents(self, spans, expected_counts):
         span_names = []
         for span in spans:
             # Check that collected spans have proper events recorded
-            span_name = span[0]["name"]
+            span_name = span["name"]
             span_names.append(span_name)
-            span_events = span[0]["events"]
+            span_events = span["events"]
             event_names_only = [event["name"] for event in span_events]
             self._check_events(span_name, event_names_only)
 
@@ -283,13 +283,13 @@ def _verify_nesting(self, spans, expected_parent_span_dict):
         """
         seen_spans = {}
         for span in spans:
-            cur_span = span[0]["spanId"]
-            seen_spans[cur_span] = span[0]["name"]
+            cur_span = span["spanId"]
+            seen_spans[cur_span] = span["name"]
 
         parent_child_dict = {}
         for span in spans:
-            cur_parent = span[0]["parentSpanId"]
-            cur_span = span[0]["name"]
+            cur_parent = span["parentSpanId"]
+            cur_span = span["name"]
             if cur_parent in seen_spans.keys():
                 parent_name = seen_spans[cur_parent]
                 if parent_name not in parent_child_dict:
@@ -377,16 +377,21 @@ def _test_trace(
         """
         time.sleep(COLLECTOR_TIMEOUT)
         traces = self._parse_trace_log(self.filename)
-        self.assertEqual(len(traces), 1, "Unexpected number of traces collected")
+        expected_traces_number = 1
+        self.assertEqual(
+            len(traces),
+            expected_traces_number,
+            "Unexpected number of traces collected. Expected {}, but got {}".format(
+                expected_traces_number, len(traces)
+            ),
+        )
         self._test_resource_attributes(
             traces[0]["resourceSpans"][0]["resource"]["attributes"]
         )
 
-        parsed_spans = [
-            entry["scopeSpans"][0]["spans"] for entry in traces[0]["resourceSpans"]
-        ]
+        parsed_spans = traces[0]["resourceSpans"][0]["scopeSpans"][0]["spans"]
         root_span = [
-            entry[0] for entry in parsed_spans if entry[0]["name"] == "InferRequest"
+            entry for entry in parsed_spans if entry["name"] == "InferRequest"
         ][0]
         self.assertEqual(len(parsed_spans), expected_number_of_spans)
 
diff --git a/qa/L0_trace/test.sh b/qa/L0_trace/test.sh
index 1d553336e9..b4a17bcd95 100755
--- a/qa/L0_trace/test.sh
+++ b/qa/L0_trace/test.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -897,4 +897,132 @@ kill $SERVER_PID
 wait $SERVER_PID
 set +e
 
+################################################################################
+# Tests to make sure BatchSpanProcessor's arguments are propagated from cmd    #
+# to trace initialization step.                                                #
+################################################################################
+
+# bsp_max_queue_size = 1
+# We are sending a bls request, that results in a trace with 6 spans,
+# but because `bsp_max_queue_size` is 1, OTel should drop some of them
+# and print a warning in a log.
+EXPECTED_WARNING="BatchSpanProcessor queue is full - dropping span."
+SERVER_ARGS="--trace-config=level=TIMESTAMPS --trace-config=rate=1\
+                --trace-config=count=-1 --trace-config=mode=opentelemetry \
+                --trace-config=opentelemetry,url=localhost:$OTLP_PORT/v1/traces \
+                --trace-config opentelemetry,bsp_max_queue_size=1
+                --model-repository=$MODELSDIR --log-verbose=1"
+SERVER_LOG="./inference_server_otel_BSP_max_queue_size.log"
+
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    exit 1
+fi
+
+rm collected_traces.json
+$OTEL_COLLECTOR --config ./trace-config.yaml >> $OTEL_COLLECTOR_LOG 2>&1 & COLLECTOR_PID=$!
+
+set +e
+python -c 'import opentelemetry_unittest; \
+    opentelemetry_unittest.send_bls_request(model_name="ensemble_add_sub_int32_int32_int32")'  >> client_update.log 2>&1
+
+sleep 20
+
+if ! [[ `grep -c "$EXPECTED_WARNING" $SERVER_LOG` > 0 ]] ; then
+    echo -e "\n***\n*** $SERVER_LOG does not contain expected BSP warning.\n***"
+    cat $SERVER_LOG
+    exit 1
+fi
+
+set -e
+kill $COLLECTOR_PID
+wait $COLLECTOR_PID
+kill $SERVER_PID
+wait $SERVER_PID
+set +e
+
+# bsp_schedule_delay = 0
+# We are sending a bls request, that results in a trace with 6 spans.
+# `bsp_schedule_delay` is 0, so OTel should export traces in batches of random
+# size, that translates into random number of 'scopeSpans' field in
+# `collected_traces.json`.
+SERVER_ARGS="--trace-config=level=TIMESTAMPS --trace-config=rate=1\
+                --trace-config=count=-1 --trace-config=mode=opentelemetry \
+                --trace-config=opentelemetry,url=localhost:$OTLP_PORT/v1/traces \
+                --trace-config opentelemetry,bsp_schedule_delay=0
+                --model-repository=$MODELSDIR --log-verbose=1"
+SERVER_LOG="./inference_server_otel_BSP_schedule_delay.log"
+
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    exit 1
+fi
+
+rm collected_traces.json
+$OTEL_COLLECTOR --config ./trace-config.yaml >> $OTEL_COLLECTOR_LOG 2>&1 & COLLECTOR_PID=$!
+
+set +e
+python -c 'import opentelemetry_unittest; \
+    opentelemetry_unittest.send_bls_request(model_name="ensemble_add_sub_int32_int32_int32")'  >> client_update.log 2>&1
+
+sleep 10
+
+if ! [[ -s collected_traces.json && `grep -o "scopeSpans" ./collected_traces.json | wc -l` > 1 ]] ; then
+    echo -e "\n***\n*** collected_traces.json has unexpected number of span batches collected.\n***"
+    cat collected_traces.json
+    exit 1
+fi
+
+set -e
+kill $COLLECTOR_PID
+wait $COLLECTOR_PID
+kill $SERVER_PID
+wait $SERVER_PID
+set +e
+
+# bsp_max_export_batch_size = 1
+# We are sending a bls request, that results in a trace with 6 spans.
+# `bsp_max_export_batch_size` is 1, so OTel should export traces in batches of
+# size 1, that translates into 6 entries of 'scopeSpans' field in
+# `collected_traces.json`.
+SERVER_ARGS="--trace-config=level=TIMESTAMPS --trace-config=rate=1\
+                --trace-config=count=-1 --trace-config=mode=opentelemetry \
+                --trace-config=opentelemetry,url=localhost:$OTLP_PORT/v1/traces \
+                --trace-config opentelemetry,bsp_max_export_batch_size=1
+                --model-repository=$MODELSDIR --log-verbose=1"
+SERVER_LOG="./inference_server_otel_BSP_max_export_batch_size.log"
+
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    exit 1
+fi
+
+rm collected_traces.json
+$OTEL_COLLECTOR --config ./trace-config.yaml >> $OTEL_COLLECTOR_LOG 2>&1 & COLLECTOR_PID=$!
+
+set +e
+python -c 'import opentelemetry_unittest; \
+    opentelemetry_unittest.send_bls_request(model_name="ensemble_add_sub_int32_int32_int32")'  >> client_update.log 2>&1
+
+sleep 10
+
+if ! [[ -s collected_traces.json && `grep -o "scopeSpans" ./collected_traces.json | wc -l` == 6 ]] ; then
+    echo -e "\n***\n*** collected_traces.json has unexpected number of span batches collected.\n***"
+    cat collected_traces.json
+    exit 1
+fi
+
+set -e
+kill $COLLECTOR_PID
+wait $COLLECTOR_PID
+kill $SERVER_PID
+wait $SERVER_PID
+set +e
+
 exit $RET
diff --git a/src/command_line_parser.cc b/src/command_line_parser.cc
index 321f274def..84a4213b2b 100644
--- a/src/command_line_parser.cc
+++ b/src/command_line_parser.cc
@@ -119,6 +119,13 @@ StringTo(const std::string& arg)
   return std::stoi(arg);
 }
 
+template <>
+uint32_t
+StringTo(const std::string& arg)
+{
+  return std::stoul(arg);
+}
+
 template <>
 uint64_t
 StringTo(const std::string& arg)
@@ -2136,43 +2143,44 @@ TritonParser::SetGlobalTraceArgs(
     bool trace_rate_present, bool trace_count_present,
     bool explicit_disable_trace)
 {
-  for (const auto& global_setting : lparams.trace_config_map_[""]) {
+  for (const auto& [setting, value_variant] : lparams.trace_config_map_[""]) {
+    auto value = std::get<std::string>(value_variant);
     try {
-      if (global_setting.first == "rate") {
+      if (setting == "rate") {
         if (trace_rate_present) {
           std::cerr << "Warning: Overriding deprecated '--trace-rate' "
                        "in favor of provided rate value in --trace-config!"
                     << std::endl;
         }
-        lparams.trace_rate_ = ParseOption<int>(global_setting.second);
+        lparams.trace_rate_ = ParseOption<int>(value);
       }
-      if (global_setting.first == "level") {
+      if (setting == "level") {
         if (trace_level_present) {
           std::cerr << "Warning: Overriding deprecated '--trace-level' "
                        "in favor of provided level in --trace-config!"
                     << std::endl;
         }
-        auto parsed_level_config = ParseTraceLevelOption(global_setting.second);
+        auto parsed_level_config = ParseTraceLevelOption(value);
         explicit_disable_trace |=
             (parsed_level_config == TRITONSERVER_TRACE_LEVEL_DISABLED);
         lparams.trace_level_ = static_cast<TRITONSERVER_InferenceTraceLevel>(
             lparams.trace_level_ | parsed_level_config);
       }
-      if (global_setting.first == "mode") {
-        lparams.trace_mode_ = ParseTraceModeOption(global_setting.second);
+      if (setting == "mode") {
+        lparams.trace_mode_ = ParseTraceModeOption(value);
       }
-      if (global_setting.first == "count") {
+      if (setting == "count") {
         if (trace_count_present) {
           std::cerr << "Warning: Overriding deprecated '--trace-count' "
                        "in favor of provided count in --trace-config!"
                     << std::endl;
         }
-        lparams.trace_count_ = ParseOption<int>(global_setting.second);
+        lparams.trace_count_ = ParseOption<int>(value);
       }
     }
     catch (const ParseException& pe) {
       std::stringstream ss;
-      ss << "Bad option: \"--trace-config " << global_setting.first << "\".\n"
+      ss << "Bad option: \"--trace-config " << setting << "\".\n"
          << pe.what() << std::endl;
       throw ParseException(ss.str());
     }
@@ -2184,29 +2192,29 @@ TritonParser::SetTritonTraceArgs(
     TritonServerParameters& lparams, bool trace_filepath_present,
     bool trace_log_frequency_present)
 {
-  for (const auto& mode_setting :
+  for (const auto& [setting, value_variant] :
        lparams.trace_config_map_[std::to_string(TRACE_MODE_TRITON)]) {
+    auto value = std::get<std::string>(value_variant);
     try {
-      if (mode_setting.first == "file") {
+      if (setting == "file") {
         if (trace_filepath_present) {
           std::cerr << "Warning: Overriding deprecated '--trace-file' "
                        "in favor of provided file in --trace-config!"
                     << std::endl;
         }
-        lparams.trace_filepath_ = mode_setting.second;
-      } else if (mode_setting.first == "log-frequency") {
+        lparams.trace_filepath_ = value;
+      } else if (setting == "log-frequency") {
         if (trace_log_frequency_present) {
           std::cerr << "Warning: Overriding deprecated '--trace-file' "
                        "in favor of provided file in --trace-config!"
                     << std::endl;
         }
-        lparams.trace_log_frequency_ = ParseOption<int>(mode_setting.second);
+        lparams.trace_log_frequency_ = ParseOption<int>(value);
       }
     }
     catch (const ParseException& pe) {
       std::stringstream ss;
-      ss << "Bad option: \"--trace-config triton," << mode_setting.first
-         << "\".\n"
+      ss << "Bad option: \"--trace-config triton," << setting << "\".\n"
          << pe.what() << std::endl;
       throw ParseException(ss.str());
     }
@@ -2214,8 +2222,9 @@ TritonParser::SetTritonTraceArgs(
 }
 
 void
-TritonParser::VerifyOpentelemetryTraceArgs(
-    bool trace_filepath_present, bool trace_log_frequency_present)
+TritonParser::SetOpenTelemetryTraceArgs(
+    TritonServerParameters& lparams, bool trace_filepath_present,
+    bool trace_log_frequency_present)
 {
   if (trace_filepath_present) {
     std::cerr << "Warning: '--trace-file' is deprecated and will "
@@ -2227,6 +2236,74 @@ TritonParser::VerifyOpentelemetryTraceArgs(
                  "and will be ignored with opentelemetry tracing mode."
               << std::endl;
   }
+  triton::server::TraceConfig& otel_trace_settings =
+      lparams.trace_config_map_[std::to_string(TRACE_MODE_OPENTELEMETRY)];
+  ProcessOpenTelemetryBatchSpanProcessorArgs(otel_trace_settings);
+}
+
+void
+TritonParser::ProcessOpenTelemetryBatchSpanProcessorArgs(
+    TraceConfig& otel_trace_settings)
+{
+  std::unordered_map<std::string, std::string> otel_bsp_default_settings = {};
+  // Set up default BatchSpanProcessor parameters, or use
+  // parameters, specified by environment variables
+  auto env_bsp_max_queue_size = triton::server::GetEnvironmentVariableOrDefault(
+      "OTEL_BSP_MAX_QUEUE_SIZE", "2048");
+  otel_bsp_default_settings.insert(std::make_pair(
+      std::string("bsp_max_queue_size"), env_bsp_max_queue_size));
+  auto env_bsp_schedule_delay = triton::server::GetEnvironmentVariableOrDefault(
+      "OTEL_BSP_SCHEDULE_DELAY", "5000");
+  otel_bsp_default_settings.insert(std::make_pair(
+      std::string("bsp_schedule_delay"), env_bsp_schedule_delay));
+  auto env_bsp_max_export_batch_size =
+      triton::server::GetEnvironmentVariableOrDefault(
+          "OTEL_BSP_MAX_EXPORT_BATCH_SIZE", "512");
+  otel_bsp_default_settings.insert(std::make_pair(
+      std::string("bsp_max_export_batch_size"), env_bsp_max_export_batch_size));
+
+  // Process cmd args and convert string arguments to integers.
+  // Throw a ParseException for invalid arguments
+  for (auto& [setting, value_variant] : otel_trace_settings) {
+    try {
+      auto value = std::get<std::string>(value_variant);
+      if (setting == "bsp_max_queue_size") {
+        value_variant = ParseOption<uint32_t>(value);
+        otel_bsp_default_settings.erase("bsp_max_queue_size");
+      } else if (setting == "bsp_schedule_delay") {
+        value_variant = ParseOption<uint32_t>(value);
+        otel_bsp_default_settings.erase("bsp_schedule_delay");
+      } else if (setting == "bsp_max_export_batch_size") {
+        value_variant = ParseOption<uint32_t>(value);
+        otel_bsp_default_settings.erase("bsp_max_export_batch_size");
+      }
+    }
+    catch (const ParseException& pe) {
+      std::stringstream ss;
+      ss << "Bad option: \"--trace-config opentelemetry," << setting << "\".\n"
+         << pe.what() << std::endl;
+      throw ParseException(ss.str());
+    }
+  }
+  // If not all BSP settings were provided through cmd,
+  // populate OpenTelemetry's trace settings with the default value.
+  if (!otel_bsp_default_settings.empty()) {
+    for (const auto& [setting, value] : otel_bsp_default_settings) {
+      try {
+        otel_trace_settings.push_back(
+            std::make_pair(setting, ParseOption<uint32_t>(value)));
+      }
+      catch (const ParseException& pe) {
+        std::stringstream ss;
+        ss << "Bad option: \"OTEL_";
+        for (auto& ch : setting) {
+          ss << static_cast<char>(std::toupper(ch));
+        }
+        ss << "\".\n" << pe.what() << std::endl;
+        throw ParseException(ss.str());
+      }
+    }
+  }
 }
 
 void
@@ -2241,8 +2318,8 @@ TritonParser::PostProcessTraceArgs(
       explicit_disable_trace);
 
   if (lparams.trace_mode_ == TRACE_MODE_OPENTELEMETRY) {
-    VerifyOpentelemetryTraceArgs(
-        trace_filepath_present, trace_log_frequency_present);
+    SetOpenTelemetryTraceArgs(
+        lparams, trace_filepath_present, trace_log_frequency_present);
   } else if (lparams.trace_mode_ == TRACE_MODE_TRITON) {
     SetTritonTraceArgs(
         lparams, trace_filepath_present, trace_log_frequency_present);
diff --git a/src/command_line_parser.h b/src/command_line_parser.h
index 707b0703dd..0131c99ff2 100644
--- a/src/command_line_parser.h
+++ b/src/command_line_parser.h
@@ -302,13 +302,16 @@ class TritonParser {
   void SetTritonTraceArgs(
       TritonServerParameters& lparams, bool trace_filepath_present,
       bool trace_log_frequency_present);
-  void VerifyOpentelemetryTraceArgs(
-      bool trace_filepath_present, bool trace_log_frequency_present);
+  void SetOpenTelemetryTraceArgs(
+      TritonServerParameters& lparams, bool trace_filepath_present,
+      bool trace_log_frequency_present);
   void PostProcessTraceArgs(
       TritonServerParameters& lparams, bool trace_level_present,
       bool trace_rate_present, bool trace_count_present,
       bool trace_filepath_present, bool trace_log_frequency_present,
       bool explicit_disable_trace);
+  void ProcessOpenTelemetryBatchSpanProcessorArgs(
+      TraceConfig& otel_trace_settings);
 #endif  // TRITON_ENABLE_TRACING
   // Helper function to parse option in
   // "<string>[1st_delim]<string>[2nd_delim]<string>" format
diff --git a/src/tracer.cc b/src/tracer.cc
index 8c633ee3ab..6d7110a9f3 100644
--- a/src/tracer.cc
+++ b/src/tracer.cc
@@ -1,4 +1,4 @@
-// Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
@@ -36,13 +36,9 @@
 #include <cuda_runtime_api.h>
 #endif  // TRITON_ENABLE_GPU
 #ifndef _WIN32
-#include "opentelemetry/exporters/otlp/otlp_http_exporter_factory.h"
 #include "opentelemetry/sdk/resource/semantic_conventions.h"
-namespace otlp = opentelemetry::exporter::otlp;
-namespace otel_trace_sdk = opentelemetry::sdk::trace;
-namespace otel_trace_api = opentelemetry::trace;
+#include "opentelemetry/sdk/trace/batch_span_processor_factory.h"
 namespace otel_common = opentelemetry::common;
-namespace otel_resource = opentelemetry::sdk::resource;
 #endif
 
 namespace triton { namespace server {
@@ -409,29 +405,16 @@ TraceManager::InitTracer(const triton::server::TraceConfigMap& config_map)
   switch (global_setting_->mode_) {
     case TRACE_MODE_OPENTELEMETRY: {
 #ifndef _WIN32
-      otlp::OtlpHttpExporterOptions opts;
+      otlp::OtlpHttpExporterOptions exporter_options;
       otel_resource::ResourceAttributes attributes = {};
-      attributes[otel_resource::SemanticConventions::kServiceName] =
-          std::string("triton-inference-server");
-      auto mode_key = std::to_string(TRACE_MODE_OPENTELEMETRY);
-      auto otel_options_it = config_map.find(mode_key);
-      if (otel_options_it != config_map.end()) {
-        for (const auto& setting : otel_options_it->second) {
-          // FIXME add more configuration options of OTLP HTTP Exporter
-          if (setting.first == "url") {
-            opts.url = setting.second;
-          }
-          if (setting.first == "resource") {
-            auto pos = setting.second.find('=');
-            auto key = setting.second.substr(0, pos);
-            auto value = setting.second.substr(pos + 1);
-            attributes[key] = value;
-          }
-        }
-      }
-      auto exporter = otlp::OtlpHttpExporterFactory::Create(opts);
-      auto processor = otel_trace_sdk::SimpleSpanProcessorFactory::Create(
-          std::move(exporter));
+      otel_trace_sdk::BatchSpanProcessorOptions processor_options;
+
+      ProcessOpenTelemetryParameters(
+          config_map, exporter_options, attributes, processor_options);
+
+      auto exporter = otlp::OtlpHttpExporterFactory::Create(exporter_options);
+      auto processor = otel_trace_sdk::BatchSpanProcessorFactory::Create(
+          std::move(exporter), processor_options);
       auto resource = otel_resource::Resource::Create(attributes);
       std::shared_ptr<otel_trace_api::TracerProvider> provider =
           otel_trace_sdk::TracerProviderFactory::Create(
@@ -477,6 +460,45 @@ TraceManager::CleanupTracer()
 }
 
 #ifndef _WIN32
+void
+TraceManager::ProcessOpenTelemetryParameters(
+    const triton::server::TraceConfigMap& config_map,
+    otlp::OtlpHttpExporterOptions& exporter_options,
+    otel_resource::ResourceAttributes& attributes,
+    otel_trace_sdk::BatchSpanProcessorOptions& processor_options)
+{
+  attributes[otel_resource::SemanticConventions::kServiceName] =
+      std::string("triton-inference-server");
+  auto mode_key = std::to_string(TRACE_MODE_OPENTELEMETRY);
+  auto otel_options_it = config_map.find(mode_key);
+  if (otel_options_it == config_map.end()) {
+    return;
+  }
+  for (const auto& [setting, value] : otel_options_it->second) {
+    // FIXME add more configuration options of OTLP HTTP Exporter
+    if (setting == "url") {
+      exporter_options.url = std::get<std::string>(value);
+    }
+    if (setting == "resource") {
+      auto user_setting = std::get<std::string>(value);
+      auto pos = user_setting.find('=');
+      auto key = user_setting.substr(0, pos);
+      auto value = user_setting.substr(pos + 1);
+      attributes[key] = value;
+    }
+    if (setting == "bsp_max_queue_size") {
+      processor_options.max_queue_size = std::get<uint32_t>(value);
+    }
+    if (setting == "bsp_schedule_delay") {
+      processor_options.schedule_delay_millis =
+          std::chrono::milliseconds(std::get<uint32_t>(value));
+    }
+    if (setting == "bsp_max_export_batch_size") {
+      processor_options.max_export_batch_size = std::get<uint32_t>(value);
+    }
+  }
+}
+
 void
 TraceManager::Trace::StartSpan(
     std::string span_key, TRITONSERVER_InferenceTrace* trace,
diff --git a/src/tracer.h b/src/tracer.h
index 2af7f28956..a591f7bd98 100644
--- a/src/tracer.h
+++ b/src/tracer.h
@@ -1,4 +1,4 @@
-// Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
@@ -34,26 +34,31 @@
 #include <sstream>
 #include <string>
 #include <unordered_map>
+#include <variant>
 
 #if !defined(_WIN32) && defined(TRITON_ENABLE_TRACING)
 #include "opentelemetry/context/propagation/global_propagator.h"
+#include "opentelemetry/exporters/otlp/otlp_http_exporter_factory.h"
 #include "opentelemetry/nostd/shared_ptr.h"
 #include "opentelemetry/sdk/resource/resource.h"
+#include "opentelemetry/sdk/trace/batch_span_processor_options.h"
 #include "opentelemetry/sdk/trace/processor.h"
-#include "opentelemetry/sdk/trace/simple_processor_factory.h"
 #include "opentelemetry/sdk/trace/tracer_provider_factory.h"
 #include "opentelemetry/trace/context.h"
 #include "opentelemetry/trace/propagation/http_trace_context.h"
 #include "opentelemetry/trace/provider.h"
+namespace otlp = opentelemetry::exporter::otlp;
 namespace otel_trace_sdk = opentelemetry::sdk::trace;
 namespace otel_trace_api = opentelemetry::trace;
 namespace otel_cntxt = opentelemetry::context;
+namespace otel_resource = opentelemetry::sdk::resource;
 #endif
 #include "triton/core/tritonserver.h"
 
 namespace triton { namespace server {
 
-using TraceConfig = std::vector<std::pair<std::string, std::string>>;
+using TraceConfig = std::vector<
+    std::pair<std::string, std::variant<std::string, int, uint32_t>>>;
 using TraceConfigMap = std::unordered_map<std::string, TraceConfig>;
 #if !defined(_WIN32) && defined(TRITON_ENABLE_TRACING)
 using AbstractCarrier = otel_cntxt::propagation::TextMapCarrier;
@@ -189,6 +194,13 @@ class TraceManager {
   /// set by InitTracer.
   /// In Triton trace mode is a no-op.
   void CleanupTracer();
+#if !defined(_WIN32) && defined(TRITON_ENABLE_TRACING)
+  void ProcessOpenTelemetryParameters(
+      const triton::server::TraceConfigMap& config_map,
+      otlp::OtlpHttpExporterOptions& exporter_options,
+      otel_resource::ResourceAttributes& attributes,
+      otel_trace_sdk::BatchSpanProcessorOptions& processor_options);
+#endif
 
   struct Trace {
     Trace() : trace_(nullptr), trace_id_(0) {}
@@ -223,8 +235,8 @@ class TraceManager {
     /// it starts a new request or compute span. For the request span it
     /// adds some triton related attributes, and adds this span to
     /// `otel_context_`. Alternatively, if activity is
-    /// `TRITONSERVER_TRACE_REQUEST_END` or `TRITONSERVER_TRACE_COMPUTE_END`,
-    /// it ends the corresponding span.
+    /// `TRITONSERVER_TRACE_REQUEST_END` or
+    /// `TRITONSERVER_TRACE_COMPUTE_END`, it ends the corresponding span.
     ///
     /// \param trace TRITONSERVER_InferenceTrace instance.
     /// \param activity  Trace activity.