[core] Proposed solution for m_ConnectionLock/m_LSLock inversion

srtcore/api.cpp

@@ -2783,7 +2783,7 @@ void srt::CUDTUnited::removeSocket(const SRTSOCKET u)
     m_ClosedSockets.erase(i);
 
     HLOGC(smlog.Debug, log << "GC/removeSocket: closing associated UDT @" << u);
-    s->core().closeInternal();
+    s->core().closeInternal(false /* DO NOT lock m_ConnectionLock inside */);
     HLOGC(smlog.Debug, log << "GC/removeSocket: DELETING SOCKET @" << u);
     delete s;
     HLOGC(smlog.Debug, log << "GC/removeSocket: socket @" << u << " DELETED. Checking muxer.");

srtcore/core.cpp

@@ -976,23 +976,28 @@ void srt::CUDT::open()
 
 void srt::CUDT::setListenState()
 {
-    ScopedLock cg(m_ConnectionLock);
+    {
+        ScopedLock cg(m_ConnectionLock);
 
-    if (!m_bOpened)
-        throw CUDTException(MJ_NOTSUP, MN_NONE, 0);
+        if (!m_bOpened)
+            throw CUDTException(MJ_NOTSUP, MN_NONE, 0);
 
-    if (m_bConnecting || m_bConnected)
-        throw CUDTException(MJ_NOTSUP, MN_ISCONNECTED, 0);
+        if (m_bConnecting || m_bConnected)
+            throw CUDTException(MJ_NOTSUP, MN_ISCONNECTED, 0);
 
-    // listen can be called more than once
-    if (m_bListening)
-        return;
+        // listen can be called more than once
+        if (m_bListening)
+            return;
+
+        m_bListening = true;
+    }
 
     // if there is already another socket listening on the same port
     if (m_pRcvQueue->setListener(this) < 0)
+    {
+        m_bListening = false;
         throw CUDTException(MJ_NOTSUP, MN_BUSY, 0);
-
-    m_bListening = true;
+    }
 }
 
 size_t srt::CUDT::fillSrtHandshake(uint32_t *aw_srtdata, size_t srtlen, int msgtype, int hs_version)
@@ -6052,7 +6057,7 @@ void srt::CUDT::addressAndSend(CPacket& w_pkt)
 }
 
 // [[using maybe_locked(m_GlobControlLock, if called from GC)]]
-bool srt::CUDT::closeInternal()
+bool srt::CUDT::closeInternal(bool lock_connectionLock)
 {
     // NOTE: this function is called from within the garbage collector thread.
 
@@ -6164,7 +6169,8 @@ bool srt::CUDT::closeInternal()
 
     HLOGC(smlog.Debug, log << CONID() << "CLOSING STATE. Acquiring connection lock");
 
-    ScopedLock connectguard(m_ConnectionLock);
+    if (lock_connectionLock)
+        enterCS(m_ConnectionLock);
 
     // Signal the sender and recver if they are waiting for data.
     releaseSynch();
@@ -6205,6 +6211,9 @@ bool srt::CUDT::closeInternal()
         m_bConnected = false;
     }
 
+    if (lock_connectionLock)
+        leaveCS(m_ConnectionLock);
+
     HLOGC(smlog.Debug, log << CONID() << "CLOSING, joining send/receive threads");
 
     // waiting all send and recv calls to stop

srtcore/core.h

@@ -406,7 +406,9 @@ class CUDT
     // immediately to free the socket
     void notListening()
     {
-        sync::ScopedLock cg(m_ConnectionLock);
+        // XXX This is done when closing the socket after setting m_bBroken
+        // Likely m_ConnectionLock is not necessary in this situation.
+        //sync::ScopedLock cg(m_ConnectionLock);
         m_bListening = false;
         m_pRcvQueue->removeListener(this);
     }
@@ -573,7 +575,7 @@ class CUDT
 
     /// Close the opened UDT entity.
 
-    bool closeInternal();
+    bool closeInternal(bool lock_connectionLock = true);
     void updateBrokenConnection();
     void completeBrokenConnectionDependencies(int errorcode);