diff --git a/ODLA/platforms/odla_popart/odla_compute.cc b/ODLA/platforms/odla_popart/odla_compute.cc index 7ed3d0e0e..24ef24494 100644 --- a/ODLA/platforms/odla_popart/odla_compute.cc +++ b/ODLA/platforms/odla_popart/odla_compute.cc @@ -240,6 +240,7 @@ odla_status odla_DestroyComputation(odla_computation comp) { } popart::logging::warn("reset config state, comp: {}", comp); PopartConfig::instance()->reset_init_state(); + popart::logging::warn("reset config state DONE, comp: {}", comp); return ODLA_SUCCESS; } diff --git a/ODLA/platforms/odla_popart/odla_popart.cc b/ODLA/platforms/odla_popart/odla_popart.cc index 3c2ce3e74..73078a49e 100644 --- a/ODLA/platforms/odla_popart/odla_popart.cc +++ b/ODLA/platforms/odla_popart/odla_popart.cc @@ -121,6 +121,27 @@ void compute_loop(odla_computation comp) { #undef RETURN_ERROR #define RETURN_ERROR(ERR_CODE) return ERR_CODE; +void _odla_computation::release_session() { + if (nullptr == session) + popart::logging::warn("session is nullptr when try to release it"); + else if (session->getDevice().getDeviceInfo() == nullptr) + popart::logging::warn( + "session->getDevice().getDeviceInfo() is nullptr when try to release " + "it"); + else { + popart::logging::warn( + "Calling session->getDevice().getDeviceInfo()->detach() to detach the " + "device when QManager Status is {}", + QManager::instance()->get_status()); + session->getDevice().getDeviceInfo()->detach(); + popart::logging::warn("The computation:{} session:{} detached from device", + this, session.get()); + } + if (session != nullptr) session.reset(); + assert(session == nullptr); + popart::logging::warn("The computation:{} session has been reset", this); +} + odla_status _odla_computation::compile_and_export() { odla_status ret_value = ODLA_SUCCESS; POPLAR_TRY @@ -213,7 +234,7 @@ odla_status _odla_computation::init(bool is_compile) { popart::AnchorReturnType("All")); // Acquire IPU if (opts.use_ipu_model) { - popart::logging::info("Using IPU Model to run."); + popart::logging::warn("Using IPU Model to run."); std::map deviceOpts{ {"numIPUs", std::to_string(opts.ipu_num)}, {"tilesPerIPU", "1216"}}; device = @@ -230,6 +251,7 @@ odla_status _odla_computation::init(bool is_compile) { throw std::runtime_error( "Failed to get a device when initializing odla_computation"); } + popart::logging::warn("Device acquired to run model"); // Create and config SessionOptions set_session_opts(); @@ -255,6 +277,9 @@ odla_status _odla_computation::init(bool is_compile) { // Create InferenceSession new_session = std::move(popart::InferenceSession::createFromOnnxModel( proto, data_flow, device, popart::InputShapeInfo(), session_opts_)); + popart::logging::warn( + "New session: {} has been created for computation: {}", + new_session.get(), this); if (!is_compile) { if (PopartConfig::instance()->load_or_save_cache()) { @@ -297,7 +322,9 @@ odla_status _odla_computation::init(bool is_compile) { is_compile_only_ = true; } // set session after all initialization done. + popart::logging::warn("Moving new_session to session: {}", session.get()); session = std::move(new_session); + popart::logging::warn("Moved new_session to session: {}", session.get()); // Thread must be started after all initialization done if (!is_compile) { ExecutionMode mode = PopartConfig::instance()->execution_mode(); @@ -404,7 +431,7 @@ bool _odla_computation::hold() { } else { std::stringstream ss_holder; ss_holder << thread_id_of_holder; - popart::logging::warn( + popart::logging::info( "The odla_computation {} has been held by thread: {}" ", when thread {} try to hold it.", this, thread_id_of_holder, this_thread_id); diff --git a/ODLA/platforms/odla_popart/odla_popart.h b/ODLA/platforms/odla_popart/odla_popart.h index 331a82dfa..37bcfbf9b 100644 --- a/ODLA/platforms/odla_popart/odla_popart.h +++ b/ODLA/platforms/odla_popart/odla_popart.h @@ -151,17 +151,7 @@ struct _odla_computation { inline Execution* executor() { return executor_; } inline bool is_done() { return thread_state_ != RUNNING; } inline bool is_compile_only() { return is_compile_only_; } - inline void release_session() { - if (session != nullptr) { - session->getDevice().getDeviceInfo()->detach(); - popart::logging::warn( - "The computation:{} session:{} detached from device", this, - session.get()); - session.reset(); - assert(session == nullptr); - popart::logging::warn("The computation:{} session has been reset", this); - } - } + void release_session(); inline void set_thread_run() { std::unique_lock lock(thread_done_mutex_); thread_state_ = RUNNING; diff --git a/ODLA/platforms/odla_popart/popart_config.h b/ODLA/platforms/odla_popart/popart_config.h index 1a676ccaf..512705f99 100644 --- a/ODLA/platforms/odla_popart/popart_config.h +++ b/ODLA/platforms/odla_popart/popart_config.h @@ -115,7 +115,7 @@ class PopartConfig { std::lock_guard guard(config_mutex_); if (inited_) { inited_ = false; - if (cache_fs->is_open()) { + if (cache_fs && cache_fs->is_open()) { cache_fs->close(); cache_fs->clear(); }