Make all resources used in thread intialized before start the thread

Author: jackzipu

ODLA/platforms/odla_popart/odla_compute.cc

@@ -167,7 +167,8 @@ odla_status odla_CreateComputation(odla_computation* comp) {
 
   static void* custom_op_handle = nullptr;
   *comp = _odla_computation::instance();
-  popart::logging::info("computation created");
+  popart::logging::warn("computation created: {}",
+                        _odla_computation::instance());
   if (custom_op_handle == nullptr) {
     custom_op_handle = dlopen("libcustom_ops.so", RTLD_NOW | RTLD_GLOBAL);
     if (custom_op_handle == nullptr) {
@@ -228,7 +229,7 @@ odla_status odla_DestroyContext(odla_context ctx) {
 
 odla_status odla_DestroyComputation(odla_computation comp) {
   std::lock_guard<std::mutex> guard(g_computation_mutex);
-  popart::logging::info("call odla_destroyComputation");
+  popart::logging::warn("call odla_destroyComputation comp: {}", comp);
   if (comp != nullptr) {
     if (!comp->is_compile_only()) {
       comp->mark_done();
@@ -237,7 +238,7 @@ odla_status odla_DestroyComputation(odla_computation comp) {
     comp->release_session();
     _odla_computation::destruct(); // release the real computation
   }
-  popart::logging::info("reset config state");
+  popart::logging::warn("reset config state, comp: {}", comp);
   PopartConfig::instance()->reset_init_state();
 
   return ODLA_SUCCESS;

ODLA/platforms/odla_popart/odla_popart.cc

@@ -81,15 +81,23 @@ void compute_loop(odla_computation comp) {
   popart::StepIOCallback stepio(input_callback, input_complete_callback,
                                 output_callback, output_complete_callback);
   int i = 0;
+  bool info_printed = false;
   POPLAR_TRY
+  comp->set_thread_run(); // set the state to RUNNING
   while (!comp->is_done()) {
     auto start = std::chrono::steady_clock::now();
     popart::logging::info("This is the {} time for the inference", i++);
     if (i == INT_MAX) i = 0;
+    if (!info_printed) {
+      popart::logging::warn(
+          "Start to run the stepio with comp:{}, session:{}, device:{}", comp,
+          comp->session.get(), comp->session->getDevice().getDeviceInfo());
+      info_printed = true;
+    }
     comp->session->run(stepio);
     auto end = std::chrono::steady_clock::now();
     std::chrono::duration<double> elapsed_seconds = end - start;
-    popart::logging::warn(
+    popart::logging::info(
         "[ {} ] ONE_STEP takes {} s. Check whether more inference tasks "
         "wating.",
         i, elapsed_seconds.count());
@@ -99,11 +107,12 @@ void compute_loop(odla_computation comp) {
       std::this_thread::sleep_for(std::chrono::milliseconds(1));
     end = std::chrono::steady_clock::now();
     std::chrono::duration<double, std::milli> elapsed_ms = end - start;
-    popart::logging::warn("Found new tasks in {} ms.", elapsed_ms.count());
+    popart::logging::info("Found new tasks in {} ms.", elapsed_ms.count());
   }
   POPLAR_CATCH
 
-  popart::logging::info("The pipeline loop finished");
+  popart::logging::warn(
+      "The computation: {} pipeline loop finished after {} steps run", comp, i);
   comp->thread_done();
 }
 
@@ -184,6 +193,7 @@ odla_status _odla_computation::compile_and_export() {
 
 odla_status _odla_computation::init(bool is_compile) {
   if (!session) {
+    popart::logging::warn("The computation:{} start to init", this);
     std::lock_guard<std::mutex> guard(init_mutex_);
     if (!session) {
       POPLAR_TRY
@@ -281,22 +291,26 @@ odla_status _odla_computation::init(bool is_compile) {
         new_session->prepareDevice();
         new_session->setRandomSeed(0);  // Init seed
         new_session->weightsFromHost(); // Copy weights from host to IPU
-        // If in parallel mode, start the thread
+      } else {
+        is_compile_only_ = true;
+      }
+      // set session after all initialization done.
+      session = std::move(new_session);
+      // Thread must be started after all initialization done
+      if (!is_compile) {
         ExecutionMode mode = PopartConfig::instance()->execution_mode();
         if (PIPELINE == mode || PARALLEL == mode || PIPELINE_ASYNC == mode) {
           std::thread parallel_thread(compute_loop, this);
-          thread_state_ = RUNNING;
-          popart::logging::warn("Parallel loop has been started");
+          popart::logging::warn(
+              "The computation: {}, parallel loop has been started", this);
           parallel_thread.detach();
         }
-      } else {
-        is_compile_only_ = true;
       }
-
-      session =
-          std::move(new_session); // set session after all initialization done.
       POPLAR_CATCH
     }
+    popart::logging::warn(
+        "The computation:{} has been initialised with session:{}", this,
+        session.get());
   }
   return ODLA_SUCCESS;
 }

ODLA/platforms/odla_popart/odla_popart.h

@@ -101,6 +101,8 @@ struct _odla_computation {
     if (instance_ == nullptr) {
       std::lock_guard<std::mutex> guard(comp_mutex_);
       if (instance_ == nullptr) instance_ = new _odla_computation();
+      popart::logging::warn("The computation:{} has been firstly created",
+                            instance_);
     }
     if (hold_it) instance_->hold();
     return instance_;
@@ -110,6 +112,8 @@ struct _odla_computation {
       std::lock_guard<std::mutex> guard(comp_mutex_);
       if (instance_ != nullptr) {
         delete instance_;
+        popart::logging::warn("The computation:{} has been destructed",
+                              instance_);
         instance_ = nullptr;
       }
     }
@@ -150,19 +154,31 @@ struct _odla_computation {
   inline void release_session() {
     if (session != nullptr) {
       session->getDevice().getDeviceInfo()->detach();
+      popart::logging::warn(
+          "The computation:{} session:{} detached from device", this,
+          session.get());
       session.reset();
       assert(session == nullptr);
+      popart::logging::warn("The computation:{} session has been reset", this);
     }
   }
-
+  inline void set_thread_run() {
+    std::unique_lock<std::mutex> lock(thread_done_mutex_);
+    thread_state_ = RUNNING;
+  }
   inline void mark_done() {
     while (thread_state_ != DONE) {
       std::unique_lock<std::mutex> lock(thread_done_mutex_);
       if (thread_state_ != DONE) {
         thread_state_ = MARK_DONE;
+        popart::logging::warn(
+            "The computation:{} thread now is MARK_DONE, waiting for DONE",
+            this);
         thread_done_cv_.wait_for(lock, std::chrono::milliseconds(5));
       } else
-        popart::logging::warn("Alread DONE when try to mark_done");
+        popart::logging::warn(
+            "The computation {} thread already DONE when try to mark_done",
+            this);
     }
     // Once get notified, only detach the device once
     std::lock_guard<std::mutex> guard(init_mutex_);
@@ -171,6 +187,7 @@ struct _odla_computation {
   inline void thread_done() {
     std::unique_lock<std::mutex> lock(thread_done_mutex_);
     thread_state_ = DONE;
+    popart::logging::warn("The computation:{} thread is DONE.", this);
     thread_done_cv_.notify_all();
   }
 };