update log

ODLA/platforms/odla_popart/odla_compute.cc

@@ -18,6 +18,7 @@
 
 #include <ODLA/odla.h>
 #include <dlfcn.h>
+#include <stdlib.h>
 
 #include <cstdlib>
 #include <fstream>
@@ -28,6 +29,9 @@
 #include <popart/session.hpp>
 #include <popart/tensorinfo.hpp>
 #include <popart/voiddata.hpp>
+#include <poplar/exceptions.hpp>
+#include <random>
+#include <stdexcept>
 #include <string>
 
 #include "ODLA/odla_common.h"
@@ -86,9 +90,14 @@ odla_status odla_CreateExecutable(odla_executable* executable,
       return comp->compile_and_export();
     } else {
       popart::logging::info("Computation is not initialized. init it first");
-      _odla_computation::instance()->init(true); // set is_compile to true
-                                                 // this comp init will create
-                                                 // executable
+      odla_status ret =
+          _odla_computation::instance()->init(true); // set is_compile to true
+                                                     // this comp init will
+                                                     // create executable
+      if (ret != ODLA_SUCCESS) {
+        popart::logging::err("Failed to init computation when compiling.");
+        return ODLA_FAILURE;
+      }
       _odla_computation::instance()->compile_and_export();
     }
   }
@@ -100,7 +109,7 @@ odla_status odla_StoreExecutable(const odla_char* file_name,
   return ODLA_SUCCESS;
 }
 
-odla_status odla_LoadExecutable(const odla_char* file_name, odla_device device,
+odla_status odla_LoadExecutable(const odla_char* file_name,
                                 odla_executable* executable,
                                 odla_context* context,
                                 odla_computation* computation) {
@@ -119,20 +128,13 @@ odla_status odla_CreateComputation(odla_computation* comp) {
     }
   }
   // Read the config file
+  popart::logging::info("loading config");
   if (!PopartConfig::instance()->inited()) {
-    if (PopartConfig::instance()->load_cache()) {
-      odla_status ret = PopartConfig::instance()->extract_config_from_cache();
-      if (ret == ODLA_FAILURE) {
-        popart::logging::err("load config from cache failed");
-        return ret;
-      }
-    } else {
-      auto ret = PopartConfig::instance()->load_config(
-          std::getenv("ODLA_POPART_CONFIG"));
-      if (ret != ODLA_SUCCESS) {
-        popart::logging::err("error load config");
-        return ret;
-      }
+    auto ret = PopartConfig::instance()->load_config(
+        std::getenv("ODLA_POPART_CONFIG"));
+    if (ret != ODLA_SUCCESS) {
+      popart::logging::err("error load config");
+      return ret;
     }
   }
   odla_status status = _odla_computation::instance()->set_executor();
@@ -169,6 +171,7 @@ odla_status odla_DestroyContext(odla_context ctx) {
 }
 
 odla_status odla_DestroyComputation(odla_computation comp) {
+  popart::logging::info("call odla_destroyComputation");
   if (comp != nullptr) {
     if (!comp->is_compile_only()) {
       comp->mark_done();
@@ -177,6 +180,8 @@ odla_status odla_DestroyComputation(odla_computation comp) {
     comp->release_session();
     _odla_computation::destruct(); // release the real computation
   }
+  popart::logging::info("reset config state");
+  PopartConfig::instance()->reset_init_state();
 
   return ODLA_SUCCESS;
 }

ODLA/platforms/odla_popart/odla_popart.cc

@@ -79,14 +79,14 @@ void compute_loop(odla_computation comp) {
     popart::logging::err("Poplar unrecoverable_runtime_error exception caught");
     QManager::instance()->set_status(ODLA_UNRECOVERABLE_ERR);
   } catch (poplar::unknown_runtime_error& e) {
-    popart::logging::info("Poplar unknown runtime exception caught}");
+    popart::logging::err("Poplar unknown runtime exception caught");
     QManager::instance()->set_status(ODLA_UNRECOVERABLE_ERR);
   } catch (...) {
-    popart::logging::info("Poplar unknown exception caught");
+    popart::logging::err("Poplar unknown exception caught");
     QManager::instance()->set_status(ODLA_UNRECOVERABLE_ERR);
   }
 
-  popart::logging::warn("The pipeline loop finished");
+  popart::logging::info("The pipeline loop finished");
   comp->thread_done();
 }
 
@@ -99,16 +99,18 @@ odla_status _odla_computation::compile_and_export() {
   int file_prefix = cache_file_name.rfind(file_suffix);
   if (file_prefix == std::string::npos ||
       file_prefix + file_suffix.size() < cache_file_name.size()) {
-    popart::logging::err("Bad cache file name");
+    popart::logging::err(
+        "Bad cache file name. File name should end with '.popart'");
     return ODLA_FAILURE;
   }
   if (file_prefix == std::string::npos) {
     file_prefix = cache_file_name.size() - 1;
   }
   std::string config_file_name(cache_file_name.substr(0, file_prefix) +
                                ".json");
-  std::fstream cache_fs(cache_file_name,
-                        std::ios_base::out | std::ifstream::binary);
+  std::fstream cache_fs(cache_file_name, std::ios_base::out |
+                                             std::ifstream::binary |
+                                             std::ios_base::trunc);
   if (!cache_fs.is_open()) {
     popart::logging::err("Open or create cache file falied");
     return ODLA_FAILURE;
@@ -119,7 +121,7 @@ odla_status _odla_computation::compile_and_export() {
     config_fs.open(config_file_name, std::ios_base::in | std::ifstream::binary);
     if (!config_fs.is_open()) {
       popart::logging::warn(
-          "invalid config file name:[ {} ] will use default config",
+          "Open config file failed:[ {} ] will use default config",
           config_file_name);
       PopartConfig::instance()->use_default();
       config_string = PopartConfig::instance()->get_default_config_string();
@@ -132,10 +134,12 @@ odla_status _odla_computation::compile_and_export() {
     config_string = PopartConfig::instance()->get_default_config_string();
   }
   // add sdk_version in the file content
-  std::string version_string(popart::core::versionString());
+  std::string version_string(popart::core::packageHash());
   popart::logging::info("the popart version is: {}", version_string);
-  version_string = "\n\"sdk_version\":\"" + version_string + "\",";
-  config_string.insert(1, version_string);
+  if (config_string.find("sdk_version") == std::string::npos) {
+    std::string item_string = "\n\"sdk_version\":\"" + version_string + "\",";
+    config_string.insert(1, item_string);
+  }
   popart::logging::info("the config_string with sdk_version is: {}",
                         config_string);
   // added the sdk_version information to the file content
@@ -148,6 +152,9 @@ odla_status _odla_computation::compile_and_export() {
   } catch (std::exception& e) {
     popart::logging::err("compileAndExport Falied: {}", e.what());
     ret_value = ODLA_FAILURE;
+  } catch (...) {
+    popart::logging::err("compileAndExport Falied");
+    ret_value = ODLA_FAILURE;
   }
   cache_fs.flush();
   cache_fs.close();
@@ -191,6 +198,10 @@ odla_status _odla_computation::init(bool is_compile) {
         try {
           builder = popart::Builder::createFromOnnxModel(set_pipeline_stage());
         } catch (std::exception& e) {
+          popart::logging::err("create builder from onnx model failed:{}",
+                               e.what());
+          return ODLA_FAILURE;
+        } catch (...) {
           popart::logging::err("create builder from onnx model failed.");
           return ODLA_FAILURE;
         }
@@ -219,12 +230,15 @@ odla_status _odla_computation::init(bool is_compile) {
         popart::logging::err("Session::createFromOnnxModel failed:{}",
                              e.what());
         return ODLA_FAILURE;
+      } catch (...) {
+        popart::logging::err("Session::createFromOnnxModel failed");
+        return ODLA_FAILURE;
       }
 
       if (!is_compile) {
-        if (PopartConfig::instance()->load_cache()) {
+        if (PopartConfig::instance()->load_or_save_cache()) {
           popart::logging::info("Load cachefile from existing stream");
-          std::string version_string(popart::core::versionString());
+          std::string version_string(popart::core::packageHash());
           if (!PopartConfig::instance()->sdk_version_match(version_string)) {
             popart::logging::err("The sdk version of cache does not match {}",
                                  version_string);
@@ -233,9 +247,18 @@ odla_status _odla_computation::init(bool is_compile) {
           auto cache_fs = PopartConfig::instance()->get_cache_fs();
           if (cache_fs->is_open()) {
             try {
+              cache_fs->seekg(0, std::ios::beg);
+              int config_len = 0;
+              cache_fs->read((char*)&config_len, sizeof(config_len));
+              cache_fs->seekg(config_len + sizeof(config_len), std::ios::beg);
               new_session->loadExecutableFromStream(*(cache_fs.get()));
             } catch (std::exception& e) {
+              popart::logging::err("bad cache file, will compile the graph:{}",
+                                   e.what());
+              return ODLA_FAILURE;
+            } catch (...) {
               popart::logging::err("bad cache file, will compile the graph");
+              return ODLA_FAILURE;
             }
           }
         }
@@ -287,6 +310,7 @@ odla_status _odla_computation::init(bool is_compile) {
           std::move(new_session); // set session after all initialization done.
     }
   }
+  return ODLA_SUCCESS;
 }
 
 // Now we set this by config file, should set by the caller?
@@ -522,10 +546,10 @@ odla_status Sequence::compute(odla_computation comp, odla_context context,
     popart::logging::err("Poplar unrecoverable_runtime_error exception caught");
     return ODLA_UNRECOVERABLE_ERR;
   } catch (poplar::unknown_runtime_error& e) {
-    popart::logging::info("Poplar unknown runtime exception caught}");
+    popart::logging::err("Poplar unknown runtime exception caught.");
     return ODLA_UNRECOVERABLE_ERR;
   } catch (...) {
-    popart::logging::info("Poplar unknown exception caught");
+    popart::logging::err("Poplar unknown exception caught");
     return ODLA_UNRECOVERABLE_ERR;
   }
   return ODLA_SUCCESS;

ODLA/platforms/odla_popart/popart_config.cc

@@ -27,9 +27,27 @@
 #include "json.hpp"
 
 PopartConfig* PopartConfig::instance_ = new PopartConfig();
+std::vector<std::string> PopartConfig::mode = {"unknown", "pipeline",
+                                               "parallel", "sequence"};
+
+const char* bool_to_str(const bool& value) { return value ? "true" : "false"; }
+
+const std::string& get_config_path_from_cache_file(
+    const std::string& cache_path) {
+  std::string file_suffix(".popart");
+  int file_prefix = cache_path.rfind(file_suffix);
+  if (file_prefix == std::string::npos ||
+      file_prefix + file_suffix.size() < cache_path.size()) {
+    popart::logging::err(
+        "Bad cache file name. File name should end with '.popart'");
+    return std::move(std::string(""));
+  }
+  return std::move(std::string(cache_path.substr(0, file_prefix) + ".json"));
+}
 
 void PopartConfig::use_default() {
   amp_ = 0.6;
+  sdk_version_ = popart::core::packageHash();
   version_ = "1.0.0";
   batches_per_step_ = 1;
   ipu_num_ = 1;
@@ -41,31 +59,58 @@ void PopartConfig::use_default() {
   queue_type_ = "LockFreeQueue";
   queue_capacity_ = 1024 * 1024;
   debug_ = false;
-  default_config_string_ =
+  char config_base[] =
       "{\n\
-      \"version\":\"1.0.0\",\n\
-      \"amp\":0.6,\n\
-      \"batches_per_step\":1,\n\
-      \"execution_mode\":\"sequence\",\n\
-      \"ipu_num\":1,\n\
-      \"load_onnx\":false, \n\
-      \"load_onnx_path\":\"test-load-time.onnx\",\n\
-      \"queue_type\":\"LockFreeQueue\",\n\
-      \"queue_capacity\":1048576,\n\
-      \"debug\": false\n\
+      \"sdk_version\":\"%s\",\n\
+      \"version\":\"%s\",\n\
+      \"amp\":%f,\n\
+      \"batches_per_step\":%d,\n\
+      \"execution_mode\":\"%s\",\n\
+      \"ipu_num\":%d,\n\
+      \"load_onnx\":%s, \n\
+      \"load_onnx_path\":\"%s\",\n\
+      \"queue_type\":\"%s\",\n\
+      \"queue_capacity\":%d,\n\
+      \"debug\":%s\n\
       }\n";
+  char raw_default_config[1024] = {0};
+  snprintf(raw_default_config, 1024, config_base, sdk_version_.c_str(),
+           version_.c_str(), amp_, batches_per_step_,
+           PopartConfig::mode[(int)execution_mode_].c_str(), ipu_num_,
+           bool_to_str(load_onnx_), load_onnx_path_.c_str(),
+           queue_type_.c_str(), queue_capacity_, bool_to_str(debug_));
+  default_config_string_.assign(raw_default_config);
 }
 
-odla_status PopartConfig::load_config(const char* file_path) {
+odla_status PopartConfig::load_config(const char* env_file_path) {
   if (inited_) {
     popart::logging::info("config already inited");
     return ODLA_SUCCESS;
   }
-  use_default();
-  if (file_path != nullptr) {
-    load_from_file(file_path);
-  } else {
-    popart::logging::info("use default config");
+  odla_status ret = ODLA_FAILURE;
+  popart::logging::info("load or save check {}", load_or_save_cache_);
+  if (load_or_save_cache()) {
+    ret = extract_config_from_cache();
+    if (ret != ODLA_SUCCESS) {
+      popart::logging::warn("load config from cache failed");
+      std::string config_file_path =
+          get_config_path_from_cache_file(std::string(cache_path_));
+      if (!config_file_path.empty()) {
+        popart::logging::info("try load from file: {}", config_file_path);
+        ret = load_from_file(config_file_path);
+      }
+    }
+  }
+  if (ret != ODLA_SUCCESS) {
+    use_default();
+    if (env_file_path != nullptr) {
+      ret = load_from_file(env_file_path);
+      if (ret != ODLA_SUCCESS) {
+        popart::logging::info("use default config");
+      }
+    } else {
+      popart::logging::info("use default config");
+    }
   }
   return ODLA_SUCCESS;
 }
@@ -139,6 +184,9 @@ odla_status PopartConfig::load_from_string(const std::string& config_string) {
   } catch (std::exception& e) {
     popart::logging::err("parse config falied:{}", e.what());
     return ODLA_FAILURE;
+  } catch (...) {
+    popart::logging::err("parse config falied");
+    return ODLA_FAILURE;
   }
   parse_from_json(jf);
   return ODLA_SUCCESS;
@@ -149,7 +197,7 @@ odla_status PopartConfig::load_from_file(const std::string& file_path) {
     return ODLA_SUCCESS;
   }
   using json = nlohmann::json;
-  std::ifstream ifs(file_path);
+  std::ifstream ifs(file_path, std::ios_base::in);
   if (!ifs.good()) {
     popart::logging::err("config file {} not found", file_path);
     return ODLA_FAILURE;
@@ -162,12 +210,12 @@ odla_status PopartConfig::load_from_file(const std::string& file_path) {
 void PopartConfig::print() {
   std::string line(80, '=');
   popart::logging::info(line);
+  popart::logging::info("sdk_version: {}", sdk_version_);
   popart::logging::info("version: {}", version_);
   popart::logging::info("amp: {}", amp_);
   popart::logging::info("batch_per_step: {}", batches_per_step_);
-  std::string mode[] = {"UNKNOWN", "PIPELINE", "PARALLEL", "SEQUENCE"};
   popart::logging::info("execution_mode: {}",
-                        mode[(long unsigned int)execution_mode_]);
+                        PopartConfig::mode[(long unsigned int)execution_mode_]);
   popart::logging::info("ipu_num: {}", ipu_num_);
   std::string bool_value[] = {"false", "true"};
   popart::logging::info("load_onnx: {}",