Merge branch 'main' of https://github.com/triton-inference-server/pyt…

…hon_backend into spolisetty_corr_id_5900
triton-inference-server · Feb 9, 2024 · cd52562 · cd52562
2 parents eea0e33 + ba616e2
commit cd52562
Show file tree

Hide file tree

Showing 16 changed files with 633 additions and 265 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -41,6 +41,12 @@ option(TRITON_ENABLE_GPU "Enable GPU support in backend" ON)
 option(TRITON_ENABLE_STATS "Include statistics collections in backend" ON)
 option(TRITON_ENABLE_NVTX "Include nvtx markers collection in backend." OFF)
 
+# FIXME: CI needs to enable the GPU flag. Python for window currently does not
+# support GPU tensors. For simplicity, we will override this option here.
+if(WIN32)
+  set(TRITON_ENABLE_GPU OFF CACHE BOOL "GPU disabled" FORCE)
+endif()
+
 set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo")
 set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo")
 set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo")
@@ -96,6 +102,9 @@ FetchContent_Declare(
   GIT_TAG "v0.8"
   GIT_SHALLOW ON
 )
+# Option must be set off so WIN32 build does not break
+set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
+set(BUILD_MOCK OFF)
 FetchContent_MakeAvailable(dlpack)
 
 #
@@ -129,7 +138,10 @@ if(${TRITON_ENABLE_NVTX})
 endif() # TRITON_ENABLE_NVTX
 
 find_package(ZLIB REQUIRED)
-find_package(Threads REQUIRED)
+
+if(NOT WIN32)
+  find_package(Threads REQUIRED)
+endif()
 
 include_directories(${CMAKE_BINARY_DIR})
 configure_file(src/libtriton_python.ldscript libtriton_python.ldscript COPYONLY)
@@ -174,21 +186,21 @@ set(
 )
 
 set(
-  PYTHON_BACKEND_SRCS
-  src/python_be.cc
-  src/python_be.h
-  src/pb_env.cc
-  src/pb_env.h
-  src/pb_metric_reporter.cc
-  src/pb_metric_reporter.h
-  src/memory_manager.cc
-  src/memory_manager.h
-  src/request_executor.cc
-  src/request_executor.h
-  src/stub_launcher.h
-  src/stub_launcher.cc
-  src/infer_payload.h
-  src/infer_payload.cc
+    PYTHON_BACKEND_SRCS
+    src/python_be.cc
+    src/python_be.h
+    src/pb_env.cc
+    src/pb_env.h
+    src/pb_metric_reporter.cc
+    src/pb_metric_reporter.h
+    src/memory_manager.cc
+    src/memory_manager.h
+    src/request_executor.cc
+    src/request_executor.h
+    src/stub_launcher.h
+    src/stub_launcher.cc
+    src/infer_payload.h
+    src/infer_payload.cc
 )
 
 list(APPEND
@@ -239,48 +251,82 @@ target_compile_options(
   triton-python-backend PRIVATE
   $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
     -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror>
+  $<$<CXX_COMPILER_ID:MSVC>:/Wall /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor>
 )
 
 target_compile_features(triton-python-backend-stub PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD})
 target_compile_options(
   triton-python-backend-stub PRIVATE
   $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
-  -fvisibility=hidden -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror>
+    -fvisibility=hidden -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror>
+  $<$<CXX_COMPILER_ID:MSVC>:/Wall /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor>
 )
 target_compile_definitions(triton-python-backend-stub PRIVATE TRITON_PB_STUB)
 
-target_link_libraries(
-  triton-python-backend
-  PRIVATE
+# For WIN32 do not link Threads and DL_LIBS
+if(WIN32)
+  target_link_libraries(
+    triton-python-backend
+    PRIVATE
+      dlpack
+      triton-backend-utils          # from repo-backend
+      -lrt                          # shared memory
+      triton-core-serverstub        # from repo-core
+      ZLIB::ZLIB
+      -larchive
+  )
+
+  target_link_libraries(
+    triton-python-backend-stub
+    PRIVATE
+    dlpack
+    triton-backend-utils           # from repo-backend
+    pybind11::embed
+    -lrt                           # shared memory
+    -larchive                      # libarchive
+  )
+else()
+  target_link_libraries(
+    triton-python-backend
+    PRIVATE
+      dlpack
+      Threads::Threads
+      triton-backend-utils          # from repo-backend
+      ${CMAKE_DL_LIBS}              # dlopen and dlclose
+      -lrt                          # shared memory
+      triton-core-serverstub        # from repo-core
+      ZLIB::ZLIB
+      -larchive
+  )
+
+  target_link_libraries(
+    triton-python-backend-stub
+    PRIVATE
     dlpack
     Threads::Threads
-    triton-backend-utils          # from repo-backend
-    ${CMAKE_DL_LIBS}              # dlopen and dlclose
-    -lrt                          # shared memory
-    triton-core-serverstub        # from repo-core
-    ZLIB::ZLIB
-    -larchive
-)
-
-target_link_libraries(
-  triton-python-backend-stub
-  PRIVATE
-   dlpack
-   Threads::Threads
-   triton-backend-utils           # from repo-backend
-   ${CMAKE_DL_LIBS}               # dlopen and dlclose
-   pybind11::embed
-   -lrt                           # shared memory
-   -larchive                      # libarchive
-)
+    triton-backend-utils           # from repo-backend
+    ${CMAKE_DL_LIBS}               # dlopen and dlclose
+    pybind11::embed
+    -lrt                           # shared memory
+    -larchive                      # libarchive
+  )
+endif()
 
-set_target_properties(
-  triton-python-backend PROPERTIES
-  POSITION_INDEPENDENT_CODE ON
-  OUTPUT_NAME triton_python
-  LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_python.ldscript
-  LINK_FLAGS "-Wl,--version-script libtriton_python.ldscript"
-)
+if(WIN32)
+  set_target_properties(
+    triton-python-backend PROPERTIES
+    POSITION_INDEPENDENT_CODE ON
+    OUTPUT_NAME triton_python
+  )
+else()
+  set_target_properties(
+    triton-python-backend PROPERTIES
+    POSITION_INDEPENDENT_CODE ON
+    OUTPUT_NAME triton_python
+    LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_python.ldscript
+    LINK_FLAGS "-Wl,--version-script libtriton_python.ldscript"
+  )
+endif()
 
 add_subdirectory(./src/shm_monitor)
 

diff --git a/src/infer_request.h b/src/infer_request.h
@@ -88,7 +88,7 @@ class InferRequest {
       const uint64_t timeout = 0, const intptr_t response_factory_address = 0,
       const intptr_t request_address = 0,
       const PreferredMemory& preferred_memory =
-          PreferredMemory(PreferredMemory::DEFAULT, 0),
+          PreferredMemory(PreferredMemory::kDefault, 0),
       const InferenceTrace& trace = InferenceTrace());
 
   const std::vector<std::shared_ptr<PbTensor>>& Inputs();

diff --git a/src/metric_family.cc b/src/metric_family.cc
@@ -201,9 +201,9 @@ TRITONSERVER_MetricKind
 MetricFamily::ToTritonServerMetricKind(const MetricKind& kind)
 {
   switch (kind) {
-    case COUNTER:
+    case kCounter:
       return TRITONSERVER_METRIC_KIND_COUNTER;
-    case GAUGE:
+    case kGauge:
       return TRITONSERVER_METRIC_KIND_GAUGE;
     default:
       throw PythonBackendException("Unknown metric kind");

diff --git a/src/pb_env.cc b/src/pb_env.cc
@@ -26,9 +26,11 @@
 
 #include "pb_env.h"
 
+#ifndef _WIN32
 #include <archive.h>
 #include <archive_entry.h>
 #include <fts.h>
+#endif
 #include <sys/stat.h>
 
 #include <cstdlib>
@@ -40,6 +42,29 @@
 
 namespace triton { namespace backend { namespace python {
 
+bool
+FileExists(std::string& path)
+{
+  struct stat buffer;
+  return stat(path.c_str(), &buffer) == 0;
+}
+
+void
+LastModifiedTime(const std::string& path, time_t* last_modified_time)
+{
+  struct stat result;
+  if (stat(path.c_str(), &result) == 0) {
+    *last_modified_time = result.st_mtime;
+  } else {
+    throw PythonBackendException(std::string(
+        "LastModifiedTime() failed as file \'" + path +
+        std::string("\' does not exists.")));
+  }
+}
+
+// FIXME: [DLIS-5969]: Develop platforom-agnostic functions
+// to support custom python environments.
+#ifndef _WIN32
 void
 CopySingleArchiveEntry(archive* input_archive, archive* output_archive)
 {
@@ -73,7 +98,6 @@ CopySingleArchiveEntry(archive* input_archive, archive* output_archive)
   }
 }
 
-
 void
 ExtractTarFile(std::string& archive_path, std::string& dst_path)
 {
@@ -153,27 +177,6 @@ ExtractTarFile(std::string& archive_path, std::string& dst_path)
   }
 }
 
-bool
-FileExists(std::string& path)
-{
-  struct stat buffer;
-  return stat(path.c_str(), &buffer) == 0;
-}
-
-void
-LastModifiedTime(const std::string& path, time_t* last_modified_time)
-{
-  struct stat result;
-  if (stat(path.c_str(), &result) == 0) {
-    *last_modified_time = result.st_mtime;
-  } else {
-    throw PythonBackendException(std::string(
-        "LastModifiedTime() failed as file \'" + path +
-        std::string("\' does not exists.")));
-  }
-}
-
-
 void
 RecursiveDirectoryDelete(const char* dir)
 {
@@ -326,5 +329,6 @@ EnvironmentManager::~EnvironmentManager()
 {
   RecursiveDirectoryDelete(base_path_);
 }
+#endif
 
 }}}  // namespace triton::backend::python
diff --git a/src/pb_env.h b/src/pb_env.h
@@ -30,6 +30,11 @@
 #include <mutex>
 #include <string>
 
+#ifdef WIN32
+#include <windows.h>
+#undef PATH_MAX
+#define PATH_MAX MAX_PATH
+#endif
 namespace triton { namespace backend { namespace python {
 
 void ExtractTarFile(std::string& archive_path, std::string& dst_path);
@@ -39,6 +44,7 @@ bool FileExists(std::string& path);
 //
 // A class that manages Python environments
 //
+#ifndef _WIN32
 class EnvironmentManager {
   std::map<std::string, std::pair<std::string, time_t>> env_map_;
   char base_path_[PATH_MAX + 1];
@@ -52,5 +58,6 @@ class EnvironmentManager {
   std::string ExtractIfNotExtracted(std::string env_path);
   ~EnvironmentManager();
 };
+#endif
 
 }}}  // namespace triton::backend::python
diff --git a/src/pb_preferred_memory.h b/src/pb_preferred_memory.h
@@ -30,10 +30,10 @@ namespace triton { namespace backend { namespace python {
 
 class PreferredMemory {
  public:
-  enum MemoryType { GPU, CPU, DEFAULT };
+  enum MemoryType { kGPU, kCPU, kDefault };
 
   PreferredMemory()
-      : preferred_memory_type_(MemoryType::DEFAULT), preferred_device_id_(0)
+      : preferred_memory_type_(MemoryType::kDefault), preferred_device_id_(0)
   {
   }