Add pthread mutex deadlock detection for debug builds

This would really have helped in at least detecting the deadlock that
was found in #24570 / #24565

Author: sbc100

system/lib/libc/musl/src/thread/pthread_mutex_lock.c

@@ -2,9 +2,12 @@
 
 int __pthread_mutex_lock(pthread_mutex_t *m)
 {
+#if !defined(__EMSCRIPTEN__) || defined(NDEBUG)
+	/* XXX EMSCRIPTEN always take the slow path in debug builds so we can trap rather than deadlock */
 	if ((m->_m_type&15) == PTHREAD_MUTEX_NORMAL
 	    && !a_cas(&m->_m_lock, 0, EBUSY))
 		return 0;
+#endif
 
 	return __pthread_mutex_timedlock(m, 0);
 }

system/lib/libc/musl/src/thread/pthread_mutex_timedlock.c

@@ -1,5 +1,9 @@
 #include "pthread_impl.h"
 
+#ifdef __EMSCRIPTEN__
+#include <assert.h>
+#endif
+
 #ifndef __EMSCRIPTEN__
 #define IS32BIT(x) !((x)+0x80000000ULL>>32)
 #define CLAMP(x) (int)(IS32BIT(x) ? (x) : 0x7fffffffU+((0ULL+(x))>>63))
@@ -57,9 +61,12 @@ static int pthread_mutex_timedlock_pi(pthread_mutex_t *restrict m, const struct
 
 int __pthread_mutex_timedlock(pthread_mutex_t *restrict m, const struct timespec *restrict at)
 {
+#if !defined(__EMSCRIPTEN__) || defined(NDEBUG)
+	/* XXX EMSCRIPTEN always take the slow path in debug builds so we can trap rather than deadlock */
 	if ((m->_m_type&15) == PTHREAD_MUTEX_NORMAL
 	    && !a_cas(&m->_m_lock, 0, EBUSY))
 		return 0;
+#endif
 
 	int type = m->_m_type;
 	int r, t, priv = (type & 128) ^ 128;
@@ -79,6 +86,9 @@ int __pthread_mutex_timedlock(pthread_mutex_t *restrict m, const struct timespec
 		int own = r & 0x3fffffff;
 		if (!own && (!r || (type&4)))
 			continue;
+#if defined(__EMSCRIPTEN__) && !defined(NDEBUG)
+		assert(own != __pthread_self()->tid && "pthread mutex deadlock detected");
+#endif
 		if ((type&3) == PTHREAD_MUTEX_ERRORCHECK
 		    && own == __pthread_self()->tid)
 			return EDEADLK;

system/lib/libc/musl/src/thread/pthread_mutex_trylock.c

@@ -71,8 +71,11 @@ int __pthread_mutex_trylock_owner(pthread_mutex_t *m)
 
 int __pthread_mutex_trylock(pthread_mutex_t *m)
 {
+#if !defined(__EMSCRIPTEN__) || defined(NDEBUG)
+	/* XXX EMSCRIPTEN always take the slow path in debug builds so we can trap rather than deadlock */
 	if ((m->_m_type&15) == PTHREAD_MUTEX_NORMAL)
 		return a_cas(&m->_m_lock, 0, EBUSY) & EBUSY;
+#endif
 	return __pthread_mutex_trylock_owner(m);
 }
 

test/other/test_pthread_mutex_deadlock.c

@@ -0,0 +1,21 @@
+#include <assert.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdio.h>
+
+pthread_mutex_t m = PTHREAD_MUTEX_INITIALIZER;
+
+int main() {
+  printf("in main\n");
+  int rtn = pthread_mutex_lock(&m);
+  assert(rtn == 0);
+
+  // Attempt to lock a second time.  In debug builds this should
+  // hit an assertion.  In release builds this will deadlock and
+  // never return.
+  pthread_mutex_lock(&m);
+  printf("should never get here\n");
+  assert(false);
+  return 0;
+}
+

test/test_other.py

@@ -12946,6 +12946,12 @@ def test_pthread_reuse(self):
   def test_pthread_hello(self, args):
     self.do_other_test('test_pthread_hello.c', args)
 
+  @crossplatform
+  @node_pthreads
+  def test_pthread_mutex_deadlock(self):
+    self.do_runf('other/test_pthread_mutex_deadlock.c', 'pthread mutex deadlock detected',
+                 cflags=['-g'], assert_returncode=NON_ZERO)
+
   @node_pthreads
   def test_pthread_relocatable(self):
     self.do_run_in_out_file_test('hello_world.c', cflags=['-sRELOCATABLE'])