mca/threads: fix tsd management

To suppress Valgrind warnings, opal_tsd_keys_destruct() needs to explicitly
release TSD values of the main thread.  However, they were not freed if keys are
created by non-main threads.  This patch fixes it.

This patch also optimizes allocation of opal_tsd_key_values by doubling its size
when count >= length instead of increasing the size by one.

Signed-off-by: Shintaro Iwasaki <siwasaki@anl.gov>

Author: shintaro-iwasaki

opal/mca/threads/argobots/threads_argobots_module.c

@@ -39,9 +39,10 @@ struct opal_tsd_key_value {
     opal_tsd_destructor_t destructor;
 };
 
-static ABT_thread opal_main_thread;
+static opal_mutex_t opal_tsd_lock = OPAL_MUTEX_STATIC_INIT;
 static struct opal_tsd_key_value *opal_tsd_key_values = NULL;
 static int opal_tsd_key_values_count = 0;
+static int opal_tsd_key_values_size = 0;
 
 /*
  * Constructor
@@ -95,8 +96,6 @@ int opal_thread_join(opal_thread_t *t, void **thr_return)
 
 void opal_thread_set_main()
 {
-    opal_threads_argobots_ensure_init();
-    opal_main_thread = opal_thread_get_argobots_self();
 }
 
 int opal_thread_start(opal_thread_t *t)
@@ -125,14 +124,19 @@ int opal_tsd_key_create(opal_tsd_key_t *key, opal_tsd_destructor_t destructor)
     opal_threads_argobots_ensure_init();
     int rc;
     rc = ABT_key_create(destructor, key);
-    if ((ABT_SUCCESS == rc) &&
-        (opal_thread_get_argobots_self() == opal_main_thread)) {
-        opal_tsd_key_values = (struct opal_tsd_key_value *)
-            realloc(opal_tsd_key_values, (opal_tsd_key_values_count + 1) *
+    if (ABT_SUCCESS == rc) {
+        opal_mutex_lock(&opal_tsd_lock);
+        if (opal_tsd_key_values_size <= opal_tsd_key_values_count) {
+            opal_tsd_key_values_size = opal_tsd_key_values_size == 0
+                                       ? 1 : opal_tsd_key_values_size * 2;
+            opal_tsd_key_values = (struct opal_tsd_key_value *)
+                realloc(opal_tsd_key_values, opal_tsd_key_values_size *
                                              sizeof(struct opal_tsd_key_value));
+        }
         opal_tsd_key_values[opal_tsd_key_values_count].key = *key;
         opal_tsd_key_values[opal_tsd_key_values_count].destructor = destructor;
         opal_tsd_key_values_count++;
+        opal_mutex_unlock(&opal_tsd_lock);
     }
     return (ABT_SUCCESS == rc) ? OPAL_SUCCESS : OPAL_ERROR;
 }
@@ -141,6 +145,7 @@ int opal_tsd_keys_destruct(void)
 {
     int i;
     void *ptr;
+    opal_mutex_lock(&opal_tsd_lock);
     for (i = 0; i < opal_tsd_key_values_count; i++) {
         if (OPAL_SUCCESS ==
             opal_tsd_getspecific(opal_tsd_key_values[i].key, &ptr)) {
@@ -152,7 +157,10 @@ int opal_tsd_keys_destruct(void)
     }
     if (0 < opal_tsd_key_values_count) {
         free(opal_tsd_key_values);
+        opal_tsd_key_values = NULL;
         opal_tsd_key_values_count = 0;
+        opal_tsd_key_values_size = 0;
     }
+    opal_mutex_unlock(&opal_tsd_lock);
     return OPAL_SUCCESS;
 }

opal/mca/threads/pthreads/threads_pthreads_module.c

@@ -32,15 +32,15 @@
 #include "opal/mca/threads/threads.h"
 #include "opal/mca/threads/tsd.h"
 
-static pthread_t opal_main_thread;
-
 struct opal_tsd_key_value {
     opal_tsd_key_t key;
     opal_tsd_destructor_t destructor;
 };
 
+static opal_mutex_t opal_tsd_lock = OPAL_MUTEX_STATIC_INIT;
 static struct opal_tsd_key_value *opal_tsd_key_values = NULL;
 static int opal_tsd_key_values_count = 0;
+static int opal_tsd_key_values_size = 0;
 
 /*
  * Constructor
@@ -93,13 +93,19 @@ int opal_tsd_key_create(opal_tsd_key_t *key, opal_tsd_destructor_t destructor)
 {
     int rc;
     rc = pthread_key_create(key, destructor);
-    if ((0 == rc) && (pthread_self() == opal_main_thread)) {
-        opal_tsd_key_values = (struct opal_tsd_key_value *)
-            realloc(opal_tsd_key_values, (opal_tsd_key_values_count + 1) *
+    if (0 == rc) {
+        opal_mutex_lock(&opal_tsd_lock);
+        if (opal_tsd_key_values_size <= opal_tsd_key_values_count) {
+            opal_tsd_key_values_size = opal_tsd_key_values_size == 0
+                                       ? 1 : opal_tsd_key_values_size * 2;
+            opal_tsd_key_values = (struct opal_tsd_key_value *)
+                realloc(opal_tsd_key_values, opal_tsd_key_values_size *
                                              sizeof(struct opal_tsd_key_value));
+        }
         opal_tsd_key_values[opal_tsd_key_values_count].key = *key;
         opal_tsd_key_values[opal_tsd_key_values_count].destructor = destructor;
         opal_tsd_key_values_count++;
+        opal_mutex_unlock(&opal_tsd_lock);
     }
     return 0 == rc ? OPAL_SUCCESS : OPAL_ERR_IN_ERRNO;
 }
@@ -108,6 +114,7 @@ int opal_tsd_keys_destruct(void)
 {
     int i;
     void *ptr;
+    opal_mutex_lock(&opal_tsd_lock);
     for (i = 0; i < opal_tsd_key_values_count; i++) {
         if (OPAL_SUCCESS ==
             opal_tsd_getspecific(opal_tsd_key_values[i].key, &ptr)) {
@@ -119,14 +126,16 @@ int opal_tsd_keys_destruct(void)
     }
     if (0 < opal_tsd_key_values_count) {
         free(opal_tsd_key_values);
+        opal_tsd_key_values = NULL;
         opal_tsd_key_values_count = 0;
+        opal_tsd_key_values_size = 0;
     }
+    opal_mutex_unlock(&opal_tsd_lock);
     return OPAL_SUCCESS;
 }
 
 void opal_thread_set_main(void)
 {
-    opal_main_thread = pthread_self();
 }
 
 void opal_event_use_threads(void)