Skip to content

Commit

Permalink
Runtime may be left empty
Browse files Browse the repository at this point in the history
  • Loading branch information
@kthui
kthui committed Nov 7, 2023
1 parent 86d6b78 commit 50932e1
Show file tree
Hide file tree
Showing 4 changed files with 98 additions and 74 deletions.
106 changes: 75 additions & 31 deletions src/backend_model.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,15 +74,6 @@ TritonModel::Create(
"must specify 'backend' for '" + model_config.name() + "'");
}

// The model configuration must specify a runtime library.
const std::string& backend_libname = model_config.runtime();
if (backend_libname.empty()) {
return Status(
Status::Code::INVALID_ARG,
"must specify 'runtime' for '" + model_config.name() + "'");
}
bool is_python_backend_based_backend = IsPythonRuntime(backend_libname);

// Localize the content of the model repository corresponding to
// 'model_path'. This model holds a handle to the localized content
// so that it persists as long as the model is loaded.
Expand Down Expand Up @@ -111,14 +102,15 @@ TritonModel::Create(
RETURN_IF_ERROR(BackendConfigurationSpecializeBackendName(
backend_cmdline_config_map, backend_name, &specialized_backend_name));

bool is_python_backend_based_backend = false;
std::vector<std::string> search_paths = GetBackendLibrarySearchPaths(
model_path, version, backend_dir, backend_name);

std::string backend_libdir, backend_libpath;
RETURN_IF_ERROR(GetBackendLibraryDirectoryAndPath(
RETURN_IF_ERROR(GetBackendLibraryProperties(
model_config.name(), localized_model_dir->Path(), version, backend_dir,
specialized_backend_name, backend_libname,
is_python_backend_based_backend, &search_paths, &backend_libdir,
specialized_backend_name, model_config.runtime(),
&is_python_backend_based_backend, &search_paths, &backend_libdir,
&backend_libpath));

// Resolve the global backend configuration with the specific backend
Expand Down Expand Up @@ -306,27 +298,40 @@ TritonModel::GetBackendLibrarySearchPaths(
}

Status
TritonModel::GetBackendLibraryDirectoryAndPath(
TritonModel::GetBackendLibraryProperties(
const std::string& model_name, const std::string& model_path,
int64_t version, const std::string& backend_dir,
const std::string& backend_name, const std::string& backend_libname,
bool is_python_backend_based_backend,
const std::string& backend_name, const std::string& runtime,
bool* is_python_backend_based_backend,
std::vector<std::string>* search_paths, std::string* backend_libdir,
std::string* backend_libpath)
{
std::string python_backend_based_backend_libdir;
std::string backend_libname = runtime;
if (backend_libname.empty()) {
RETURN_IF_ERROR(GetBackendRuntimeLibraryName(
backend_name, *search_paths, &backend_libname, backend_libdir,
backend_libpath, is_python_backend_based_backend));
if (!is_python_backend_based_backend) {
// All variables are correctly set for C++ backends on initial search.
return Status::Success;
}
python_backend_based_backend_libdir = *backend_libdir;
} else {
*is_python_backend_based_backend = backend_libname == kPythonFilename;
}

std::string cpp_backend_libname = backend_libname;
if (is_python_backend_based_backend) {
// Use Python backend C++ runtime library.
// Set C++ library name to Python backend.
cpp_backend_libname = AssembleCPPRuntimeLibraryName(kPythonBackend);
// The search paths only contain locations related to the Python backend
// based backend, the global Python backend location needs to be added.
// based backend, the global Python backend location has to be added.
search_paths->emplace_back(JoinPath({backend_dir, kPythonBackend}));
}

backend_libpath->clear();
RETURN_IF_ERROR(FindBackendLibraryPath(
*search_paths, cpp_backend_libname, backend_libdir, backend_libpath));

if (backend_libpath->empty()) {
std::string search_paths_str = "";
for (const auto& path : *search_paths) {
Expand All @@ -338,31 +343,70 @@ TritonModel::GetBackendLibraryDirectoryAndPath(
"' for model '" + model_name + "', searched: " + search_paths_str);
}

// Both 'backend_libdir' and 'backend_libpath' are now pointing to the C++
// backend library, 'backend_libdir' needs adjustment for Python backend based
// backend.
if (is_python_backend_based_backend) {
// 'backend_libdir' point to Python backend based backend runtime directory.
*backend_libdir = JoinPath({backend_dir, backend_name});
// Make sure the runtime (and its directory) exists.
std::string runtime_libpath = JoinPath({*backend_libdir, kPythonFilename});
bool runtime_libpath_exist;
RETURN_IF_ERROR(FileExists(runtime_libpath, &runtime_libpath_exist));
if (!runtime_libpath_exist) {
return Status(
Status::Code::INVALID_ARG,
"unable to find Python backend based backend library '" +
backend_libname + "' for model '" + model_name +
"', searched: '" + runtime_libpath + "'");
if (python_backend_based_backend_libdir.empty()) {
python_backend_based_backend_libdir =
JoinPath({backend_dir, backend_name});
// Make sure the library and its directory exist.
std::string path =
JoinPath({python_backend_based_backend_libdir, kPythonFilename});
bool path_exist;
RETURN_IF_ERROR(FileExists(path, &path_exist));
if (!path_exist) {
return Status(
Status::Code::INVALID_ARG,
"unable to find Python backend based backend library '" +
backend_libname + "' for model '" + model_name +
"', searched: '" + path + "'");
}
}
*backend_libdir = python_backend_based_backend_libdir;
}

return Status::Success;
}

Status
TritonModel::GetBackendRuntimeLibraryName(
const std::string& backend_name,
const std::vector<std::string>& search_paths, std::string* backend_libname,
std::string* backend_libdir, std::string* backend_libpath,
bool* is_python_backend_based_backend)
{
// Try C++ runtime
*backend_libname = AssembleCPPRuntimeLibraryName(backend_name);
RETURN_IF_ERROR(FindBackendLibraryPath(
search_paths, *backend_libname, backend_libdir, backend_libpath));
if (!backend_libpath->empty()) {
*is_python_backend_based_backend = false;
return Status::Success;
}
// Try Python runtime
*backend_libname = kPythonFilename;
RETURN_IF_ERROR(FindBackendLibraryPath(
search_paths, *backend_libname, backend_libdir, backend_libpath));
if (!backend_libpath->empty()) {
*is_python_backend_based_backend = true;
return Status::Success;
}
// Cannot find runtime
return Status(
Status::Code::INVALID_ARG,
"unable to find backend library for backend '" + backend_name +
"', try specifying runtime on the model configuration.");
}

Status
TritonModel::FindBackendLibraryPath(
const std::vector<std::string>& search_paths,
const std::string& backend_libname, std::string* backend_libdir,
std::string* backend_libpath)
{
backend_libpath->clear();

for (const auto& path : search_paths) {
const auto full_path = JoinPath({path, backend_libname});
bool exists = false;
Expand Down
23 changes: 17 additions & 6 deletions src/backend_model.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,18 +200,29 @@ class TritonModel : public Model {
const std::string& model_path, int64_t version,
const std::string& backend_dir, const std::string& backend_name);

// Get backend library directory and path.
static Status GetBackendLibraryDirectoryAndPath(
// Get backend library directory and path, and search paths for the library
// and whether the backend is based on Python backend.
static Status GetBackendLibraryProperties(
const std::string& model_name, const std::string& model_path,
int64_t version, const std::string& backend_dir,
const std::string& backend_name, const std::string& backend_libname,
bool is_python_backend_based_backend,
const std::string& backend_name, const std::string& runtime,
bool* is_python_backend_based_backend,
std::vector<std::string>* search_paths, std::string* backend_libdir,
std::string* backend_libpath);

// Search for 'backend_libname' from 'search_paths'. If found, the matching
// Get 'backend_libname', 'backend_libdir', 'backend_libpath' and
// 'is_python_backend_based_backend' by searching for different possible
// backend library names on 'search_paths'
static Status GetBackendRuntimeLibraryName(
const std::string& backend_name,
const std::vector<std::string>& search_paths,
std::string* backend_libname, std::string* backend_libdir,
std::string* backend_libpath, bool* is_python_backend_based_backend);

// Search for 'backend_libname' on 'search_paths'. If found, the matching
// search path will be stored in 'backend_libdir' and the backend library path
// will be stored in 'backend_libpath'.
// will be stored in 'backend_libpath'. If not found, 'backend_libpath' will
// be set to empty.
static Status FindBackendLibraryPath(
const std::vector<std::string>& search_paths,
const std::string& backend_libname, std::string* backend_libdir,
Expand Down
37 changes: 6 additions & 31 deletions src/model_config_utils.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1140,8 +1140,11 @@ AutoCompleteBackendFields(
if (config->default_model_filename().empty()) {
config->set_default_model_filename(kPyTorchLibTorchFilename);
}
RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
RuntimeType::RUNTIME_TYPE_UNKNOWN, config));
auto runtime_type = RuntimeType::RUNTIME_TYPE_CPP;
if (config->default_model_filename() == kPythonFilename) {
runtime_type = RuntimeType::RUNTIME_TYPE_PYTHON;
}
RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(runtime_type, config));
return Status::Success;
}

Expand Down Expand Up @@ -1202,13 +1205,9 @@ AutoCompleteBackendFields(
config->set_backend(backend_name);
config->set_default_model_filename(
(std::string("model.") + backend_name).c_str());
RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
RuntimeType::RUNTIME_TYPE_UNKNOWN, config));
return Status::Success;
}

RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
RuntimeType::RUNTIME_TYPE_UNKNOWN, config));
return Status::Success;
}

Expand All @@ -1221,14 +1220,6 @@ AutoCompleteBackendRuntimeField(
fill_runtime = fill_runtime && config->platform() != kEnsemblePlatform;
#endif // TRITON_ENABLE_ENSEMBLE
if (fill_runtime) {
// auto detect C++ vs Python runtime if unknown
if (runtime_type == RuntimeType::RUNTIME_TYPE_UNKNOWN) {
if (IsPythonRuntime(config->default_model_filename())) {
runtime_type = RuntimeType::RUNTIME_TYPE_PYTHON;
} else {
runtime_type = RuntimeType::RUNTIME_TYPE_CPP;
}
}
// set runtime library from runtime type
if (runtime_type == RuntimeType::RUNTIME_TYPE_CPP) {
if (config->backend().empty()) {
Expand All @@ -1241,7 +1232,7 @@ AutoCompleteBackendRuntimeField(
} else if (runtime_type == RuntimeType::RUNTIME_TYPE_PYTHON) {
config->set_runtime(kPythonFilename);
} else {
return Status(Status::Code::INTERNAL, "Unimplemented runtime type.");
return Status(Status::Code::INTERNAL, "Unexpected runtime type.");
}
}
return Status::Success;
Expand Down Expand Up @@ -2465,22 +2456,6 @@ InstanceConfigSignature(const inference::ModelInstanceGroup& instance_config)
return config.SerializeAsString();
}

bool
IsPythonRuntime(const std::string& library_name)
{
if (library_name == kPythonFilename) {
return true;
}
const static std::string py_extension = ".py";
if (library_name.length() >= py_extension.length()) {
auto start_pos = library_name.length() - py_extension.length();
if (library_name.substr(start_pos) == py_extension) {
return true;
}
}
return false;
}

std::string
AssembleCPPRuntimeLibraryName(const std::string& backend_name)
{
Expand Down
6 changes: 0 additions & 6 deletions src/model_config_utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,12 +322,6 @@ bool EquivalentInInstanceConfig(
std::string InstanceConfigSignature(
const inference::ModelInstanceGroup& instance_config);

/// Check if the library name belongs to Python runtime.
/// \param library_name The library name to check.
/// \return True if the library name matches the 'kPythonFilename' constant or
/// ends with a '.py' extension. False if otherwise.
bool IsPythonRuntime(const std::string& library_name);

/// Assemble the C++ runtime library name.
/// \param backend_name The backend name.
/// \return The assembled C++ runtime library name.
Expand Down

0 comments on commit 50932e1

Please sign in to comment.