Merge branch 'master' into sarkars/unify_pooling_translate_functions

Author: sayantan-nervana

Diff for: CMakeLists.txt

@@ -239,7 +239,7 @@ if (NOT USE_PRE_BUILT_NGRAPH)
     ExternalProject_Add(
         ext_ngraph
         GIT_REPOSITORY https://github.com/NervanaSystems/ngraph
-        GIT_TAG v0.12.0
+        GIT_TAG v0.14.0-rc.0  
         CMAKE_ARGS 
             -DNGRAPH_DISTRIBUTED_ENABLE=${NGRAPH_DISTRIBUTED_ENABLE}
             -DNGRAPH_INSTALL_PREFIX=${NGRAPH_ARTIFACTS_DIR}

Diff for: build_ngtf.py

@@ -432,7 +432,7 @@ def main():
     #-------------------------------
 
     # Component versions
-    ngraph_version = "v0.12.0"
+    ngraph_version = "v0.14.0-rc.0" 
     tf_version = "v1.12.0"
 
     # Default directories
@@ -521,9 +521,9 @@ def main():
             ngraph_cmake_flags.extend(["-DCMAKE_BUILD_TYPE=Debug"])
 
         if (arguments.distributed_build): 
-            ngraph_cmake_flags.extend(["-DNGRAPH_DISTRIBUTED_ENABLE=TRUE"])
+            ngraph_cmake_flags.extend(["-DNGRAPH_DISTRIBUTED_OMPI_ENABLE=TRUE"])
         else:
-            ngraph_cmake_flags.extend(["-DNGRAPH_DISTRIBUTED_ENABLE=FALSE"])
+            ngraph_cmake_flags.extend(["-DNGRAPH_DISTRIBUTED_OMPI_ENABLE=FALSE"])
 
         build_ngraph("./ngraph", ngraph_cmake_flags, verbosity)
 

Diff for: src/ngraph_encapsulate_op.cc

@@ -34,6 +34,7 @@
 #include "ngraph_mark_for_clustering.h"
 #include "ngraph_utils.h"
 
+#include "ngraph/runtime/backend.hpp"
 #include "ngraph/runtime/interpreter/int_backend.hpp"
 
 #if defined NGRAPH_DISTRIBUTED
@@ -262,11 +263,18 @@ class NGraphEncapsulateOp : public OpKernel {
     //
     // TODO(amprocte): Investigate performance of the compilation cache.
     if (it == m_ng_functions.end()) {
+      // Measure the total memory here first
+      long vm, rss, vm0, rss0;
+
+      MemoryProfile(vm0, rss0);
+
       NGRAPH_VLOG(1) << "Compilation cache miss: " << ctx->op_kernel().name();
       OP_REQUIRES_OK(
           ctx, Builder::TranslateGraph(input_shapes, static_input_map, &m_graph,
                                        ng_function));
 
+      auto function_size = ng_function->get_graph_size() / 1024;  // kb unit
+
       // Serialize to nGraph if needed
       if (std::getenv("NGRAPH_ENABLE_SERIALIZE") != nullptr) {
         std::string file_name =
@@ -282,9 +290,39 @@ class NGraphEncapsulateOp : public OpKernel {
                         ng_function);
 #endif
       }
-
+      // Evict the cache if the number of elements exceeds 16
+      const char* m_cache_depth_specified =
+          std::getenv("NGRAPH_TF_FUNCTION_CACHE_ITEM_DEPTH");
+      if (m_cache_depth_specified != nullptr) {
+        NGRAPH_TF_FUNCTION_CACHE_ITEM_DEPTH = atoi(m_cache_depth_specified);
+      }
+      if (m_ng_functions.size() >= NGRAPH_TF_FUNCTION_CACHE_ITEM_DEPTH) {
+        op_backend->remove_compiled_function(m_ng_functions[m_lru.back()]);
+        m_ng_functions.erase(m_lru.back());
+        m_lru.pop_back();
+      }
       m_ng_functions[signature] = ng_function;
+      m_lru.push_front(signature);
+      // Memory after
+      MemoryProfile(vm, rss);
+      auto delta_vm_mem = vm - vm0;
+      auto delta_res_mem = rss - rss0;
+
+      NGRAPH_VLOG(1) << "NGRAPH_TF_CACHE_PROFILE: " << ctx->op_kernel().name()
+                     << "  Step_ID: " << ctx->step_id()
+                     << "  Delta VM: " << delta_vm_mem
+                     << "  Delta RSS: " << delta_res_mem
+                     << "  Function name:  " << ng_function->get_name()
+                     << "  Function Memory Measurment:  " << function_size
+                     << "  Total RSS in KB:  " << rss
+                     << "  Total VM in KB:  " << vm
+                     << "  Cache length: " << m_ng_functions.size() << endl;
     } else {
+      // Update the lru list
+      if (signature != m_lru.front()) {
+        m_lru.remove(signature);
+        m_lru.push_front(signature);
+      }
       ng_function = it->second;
     }
 
@@ -465,6 +503,7 @@ class NGraphEncapsulateOp : public OpKernel {
       try {
         op_backend->call(op_backend->compile(ng_function), ng_outputs,
                          ng_inputs);
+
       } catch (const std::exception& exp) {
         BackendManager::UnlockBackend(m_op_backend_name);
         NgraphSerialize(
@@ -541,6 +580,8 @@ class NGraphEncapsulateOp : public OpKernel {
   std::vector<bool> m_input_is_static;
   std::mutex m_compute_lock;
   string m_op_backend_name;
+  std::list<std::string> m_lru;
+  int NGRAPH_TF_FUNCTION_CACHE_ITEM_DEPTH = 16;
 };
 
 }  // namespace ngraph_bridge

Diff for: src/ngraph_mark_for_clustering.cc

@@ -287,6 +287,7 @@ Status MarkForClustering(Graph* graph) {
       confirmation_function_map["PreventGradient"] =
           SimpleConfirmationFunction();
       confirmation_function_map["Prod"] = SimpleConfirmationFunction();
+      confirmation_function_map["Rank"] = SimpleConfirmationFunction();
       confirmation_function_map["QuantizeAndDequantizeV2"] = [](Node* n,
                                                                 bool* result) {
         // accept only when num_bits == 8 and range is given

Diff for: src/ngraph_utils.cc

@@ -242,6 +242,27 @@ void NgraphSerialize(const std::string& file_name,
                    << std::endl;
     NGRAPH_VLOG(0) << e.what() << std::endl;
   }
+}
+
+void MemoryProfile(long& vm_usage, long& resident_set) {
+  vm_usage = 0;
+  resident_set = 0;
+
+  // Get the two fields we want
+  long vsize;
+  long rss;
+
+  std::ifstream ifs("/proc/self/stat", std::ios_base::in);
+  std::string mem_in;
+  getline(ifs, mem_in);
+  vector<string> mem_str = ng::split(mem_in, ' ');
+  vsize = std::stol(mem_str[22]);
+  rss = std::stol(mem_str[23]);
+
+  long page_size_kb = sysconf(_SC_PAGE_SIZE) /
+                      1024;  // in case x86-64 is configured to use 2MB pages
+  vm_usage = vsize / 1024;   // unit kb
+  resident_set = rss * page_size_kb;
 };
 
 }  // namespace ngraph_bridge

Diff for: src/ngraph_utils.h

@@ -202,6 +202,9 @@ Status CheckAxisDimInRange(std::vector<int64> axes, size_t rank);
 void NgraphSerialize(const std::string&,
                      const std::shared_ptr<ngraph::Function>&);
 
+// Collect the total memory usage through /proc/self/stat
+void MemoryProfile(long&, long&);
+
 }  // namespace ngraph_bridge
 
 }  // namespace tensorflow