Add support for both Qtip and UDP traffic for Server Listeners.

src/core/connection.c

@@ -6648,6 +6648,21 @@
         break;
     }
 
+    case QUIC_PARAM_CONN_QTIP: {
+        if (BufferLength != sizeof(BOOLEAN) || Buffer == NULL) {
+            Status = QUIC_STATUS_INVALID_PARAMETER;
+            break;
+        }
+        if (Connection->State.Started) {
+            Status = QUIC_STATUS_INVALID_STATE;
+            break;
+        }
+        Connection->State.UseQTIP = *(BOOLEAN*)Buffer;
+        Connection->State.AppDidSetQTIP = TRUE;
+        Status = QUIC_STATUS_SUCCESS;
+        break;
+    }
+
     //
     // Private
     //
@@ -7284,6 +7299,24 @@
         Status = QUIC_STATUS_SUCCESS;
         break;
 
+    case QUIC_PARAM_CONN_QTIP:
+        if (*BufferLength < sizeof(BOOLEAN)) {
+            *BufferLength = sizeof(BOOLEAN);
+            Status = QUIC_STATUS_BUFFER_TOO_SMALL;
+            break;
+        }
+
+        if (Buffer == NULL) {
+            Status = QUIC_STATUS_INVALID_PARAMETER;
+            break;
+        }
+
+        *BufferLength = sizeof(BOOLEAN);
+        *(BOOLEAN*)Buffer = Connection->State.UseQTIP;
+
+        Status = QUIC_STATUS_SUCCESS;
+        break;
+
     default:
         Status = QUIC_STATUS_INVALID_PARAMETER;
         break;

src/core/connection.h

@@ -208,6 +208,16 @@ typedef union QUIC_CONNECTION_STATE {
         //
         BOOLEAN DisableVneTp : 1;
 #endif
+
+        //
+        // Whether to use QTIP on sends for this connection.
+        //
+        BOOLEAN UseQTIP : 1;
+
+        //
+        // Whether or not the app explicitly set the UseQTIP flag.
+        //
+        BOOLEAN AppDidSetQTIP : 1;
     };
 } QUIC_CONNECTION_STATE;
 

src/core/listener.c

@@ -740,35 +740,49 @@
         const void* Buffer
     )
 {
-    if (Param == QUIC_PARAM_LISTENER_CIBIR_ID) {
-        if (BufferLength > QUIC_MAX_CIBIR_LENGTH + 1) {
-            return QUIC_STATUS_INVALID_PARAMETER;
-        }
-        if (BufferLength == 0) {
-            CxPlatZeroMemory(Listener->CibirId, sizeof(Listener->CibirId));
+    switch (Param) {
+        case QUIC_PARAM_LISTENER_CIBIR_ID: {
+            if (BufferLength > QUIC_MAX_CIBIR_LENGTH + 1) {
+                return QUIC_STATUS_INVALID_PARAMETER;
+            }
+            if (BufferLength == 0) {
+                CxPlatZeroMemory(Listener->CibirId, sizeof(Listener->CibirId));
+                return QUIC_STATUS_SUCCESS;
+            }
+            if (BufferLength < 2) { // Must have at least the offset and 1 byte of payload.
+                return QUIC_STATUS_INVALID_PARAMETER;
+            }
+
+            if (((uint8_t*)Buffer)[0] != 0) {
+                return QUIC_STATUS_NOT_SUPPORTED; // Not yet supproted.
+            }
+
+            Listener->CibirId[0] = (uint8_t)BufferLength - 1;
+            memcpy(Listener->CibirId + 1, Buffer, BufferLength);
+
+            QuicTraceLogVerbose(
+                ListenerCibirIdSet,
+                "[list][%p] CIBIR ID set (len %hhu, offset %hhu)",
+                Listener,
+                Listener->CibirId[0],
+                Listener->CibirId[1]);
+
             return QUIC_STATUS_SUCCESS;
         }
-        if (BufferLength < 2) { // Must have at least the offset and 1 byte of payload.
-            return QUIC_STATUS_INVALID_PARAMETER;
-        }
-
-        if (((uint8_t*)Buffer)[0] != 0) {
-            return QUIC_STATUS_NOT_SUPPORTED; // Not yet supproted.
+        case QUIC_PARAM_LISTENER_QTIP: {
+            if (BufferLength > sizeof(uint8_t)) {
+                return QUIC_STATUS_INVALID_PARAMETER;
+            }
+            if (Buffer == NULL) {
+                return QUIC_STATUS_INVALID_PARAMETER;
+            }
+            Listener->UseQTIP = *(uint8_t*)Buffer;
+            return QUIC_STATUS_SUCCESS;
         }
 
-        Listener->CibirId[0] = (uint8_t)BufferLength - 1;
-        memcpy(Listener->CibirId + 1, Buffer, BufferLength);
-
-        QuicTraceLogVerbose(
-            ListenerCibirIdSet,
-            "[list][%p] CIBIR ID set (len %hhu, offset %hhu)",
-            Listener,
-            Listener->CibirId[0],
-            Listener->CibirId[1]);
-
-        return QUIC_STATUS_SUCCESS;
+        default:
+            break;
     }
-
     return QUIC_STATUS_INVALID_PARAMETER;
 }
 
@@ -855,6 +869,25 @@
         Status = QUIC_STATUS_SUCCESS;
         break;
 
+    case QUIC_PARAM_LISTENER_QTIP:
+
+        if (*BufferLength < sizeof(uint8_t)) {
+            *BufferLength = sizeof(uint8_t);
+            Status = QUIC_STATUS_BUFFER_TOO_SMALL;
+            break;
+        }
+
+        if (Buffer == NULL) {
+            Status = QUIC_STATUS_INVALID_PARAMETER;
+            break;
+        }
+
+        *BufferLength = sizeof(uint8_t);
+        *(uint8_t*)Buffer = Listener->UseQTIP;
+
+        Status = QUIC_STATUS_SUCCESS;
+        break;
+
     default:
         Status = QUIC_STATUS_INVALID_PARAMETER;
         break;

src/core/listener.h

@@ -110,6 +110,11 @@ typedef struct QUIC_LISTENER {
     //
     uint8_t CibirId[2 + QUIC_MAX_CIBIR_LENGTH];
 
+    //
+    // Whether or not this listener accepts QTIP traffic (BOOLEAN)
+    //
+    BOOLEAN UseQTIP : 1;
+
 } QUIC_LISTENER;
 
 #ifdef QUIC_SILO

src/core/send.c

@@ -1061,7 +1061,7 @@ CxPlatIsRouteReady(
         QuicConnAddRef(Connection, QUIC_CONN_REF_ROUTE);
         QUIC_STATUS Status =
             CxPlatResolveRoute(
-                Path->Binding->Socket, &Path->Route, Path->ID, (void*)Connection, QuicConnQueueRouteCompletion);
+                Path->Binding->Socket, &Path->Route, Path->ID, (void*)Connection, QuicConnQueueRouteCompletion, Connection->State.UseQTIP, Connection->State.AppDidSetQTIP);
         if (Status == QUIC_STATUS_SUCCESS) {
             QuicConnRelease(Connection, QUIC_CONN_REF_ROUTE);
             return TRUE;

src/cs/lib/msquic_generated.cs

@@ -3372,6 +3372,9 @@ internal static unsafe partial class MsQuic
         [NativeTypeName("#define QUIC_PARAM_LISTENER_CIBIR_ID 0x04000002")]
         internal const uint QUIC_PARAM_LISTENER_CIBIR_ID = 0x04000002;
 
+        [NativeTypeName("#define QUIC_PARAM_LISTENER_QTIP 0x04000004")]
+        internal const uint QUIC_PARAM_LISTENER_QTIP = 0x04000004;
+
         [NativeTypeName("#define QUIC_PARAM_CONN_QUIC_VERSION 0x05000000")]
         internal const uint QUIC_PARAM_CONN_QUIC_VERSION = 0x05000000;
 
@@ -3450,6 +3453,9 @@ internal static unsafe partial class MsQuic
         [NativeTypeName("#define QUIC_PARAM_CONN_SEND_DSCP 0x05000019")]
         internal const uint QUIC_PARAM_CONN_SEND_DSCP = 0x05000019;
 
+        [NativeTypeName("#define QUIC_PARAM_CONN_QTIP 0x0500001A")]
+        internal const uint QUIC_PARAM_CONN_QTIP = 0x0500001A;
+
         [NativeTypeName("#define QUIC_PARAM_TLS_HANDSHAKE_INFO 0x06000000")]
         internal const uint QUIC_PARAM_TLS_HANDSHAKE_INFO = 0x06000000;
 

src/generated/linux/listener.c.clog.h

@@ -71,11 +71,11 @@ tracepoint(CLOG_LISTENER_C, ListenerIndicateNewConnection , arg2, arg3);\
 // Decoder Ring for ListenerCibirIdSet
 // [list][%p] CIBIR ID set (len %hhu, offset %hhu)
 // QuicTraceLogVerbose(
-            ListenerCibirIdSet,
-            "[list][%p] CIBIR ID set (len %hhu, offset %hhu)",
-            Listener,
-            Listener->CibirId[0],
-            Listener->CibirId[1]);
+                ListenerCibirIdSet,
+                "[list][%p] CIBIR ID set (len %hhu, offset %hhu)",
+                Listener,
+                Listener->CibirId[0],
+                Listener->CibirId[1]);
 // arg2 = arg2 = Listener = arg2
 // arg3 = arg3 = Listener->CibirId[0] = arg3
 // arg4 = arg4 = Listener->CibirId[1] = arg4

src/generated/linux/listener.c.clog.h.lttng.h

@@ -47,11 +47,11 @@ TRACEPOINT_EVENT(CLOG_LISTENER_C, ListenerIndicateNewConnection,
 // Decoder Ring for ListenerCibirIdSet
 // [list][%p] CIBIR ID set (len %hhu, offset %hhu)
 // QuicTraceLogVerbose(
-            ListenerCibirIdSet,
-            "[list][%p] CIBIR ID set (len %hhu, offset %hhu)",
-            Listener,
-            Listener->CibirId[0],
-            Listener->CibirId[1]);
+                ListenerCibirIdSet,
+                "[list][%p] CIBIR ID set (len %hhu, offset %hhu)",
+                Listener,
+                Listener->CibirId[0],
+                Listener->CibirId[1]);
 // arg2 = arg2 = Listener = arg2
 // arg3 = arg3 = Listener->CibirId[0] = arg3
 // arg4 = arg4 = Listener->CibirId[1] = arg4

src/inc/msquic.h

@@ -889,6 +889,7 @@ typedef struct QUIC_SCHANNEL_CREDENTIAL_ATTRIBUTE_W {
 #define QUIC_PARAM_LISTENER_STATS                       0x04000001  // QUIC_LISTENER_STATISTICS
 #ifdef QUIC_API_ENABLE_PREVIEW_FEATURES
 #define QUIC_PARAM_LISTENER_CIBIR_ID                    0x04000002  // uint8_t[] {offset, id[]}
+#define QUIC_PARAM_LISTENER_QTIP                        0x04000004  // uint8_t (BOOLEAN)
 #endif
 
 //
@@ -925,6 +926,10 @@ typedef struct QUIC_SCHANNEL_CREDENTIAL_ATTRIBUTE_W {
 #define QUIC_PARAM_CONN_ORIG_DEST_CID                   0x05000018  // uint8_t[]
 #define QUIC_PARAM_CONN_SEND_DSCP                       0x05000019  // uint8_t
 
+#ifdef QUIC_API_ENABLE_PREVIEW_FEATURES
+#define QUIC_PARAM_CONN_QTIP                            0x0500001A  // uint8_t (BOOLEAN)
+#endif
+
 //
 // Parameters for TLS.
 //

src/inc/quic_datapath.h

@@ -202,14 +202,15 @@ typedef struct CXPLAT_ROUTE {
     uint8_t NextHopLinkLayerAddress[6];
 
     uint16_t DatapathType; // CXPLAT_DATAPATH_TYPE
+    uint8_t UseQTIP : 1;         // TRUE if the route is using QTIP
+    uint8_t AppDidSetQTIP : 1;   // TRUE if the app explicitly set the UseQTIP flag
 
     //
     // QuicCopyRouteInfo copies memory up to this point (not including State).
     //
 
     CXPLAT_ROUTE_STATE State;
     CXPLAT_RAW_TCP_STATE TcpState;
-
 } CXPLAT_ROUTE;
 
 //
@@ -848,7 +849,9 @@ CxPlatResolveRoute(
     _Inout_ CXPLAT_ROUTE* Route,
     _In_ uint8_t PathId,
     _In_ void* Context,
-    _In_ CXPLAT_ROUTE_RESOLUTION_CALLBACK_HANDLER Callback
+    _In_ CXPLAT_ROUTE_RESOLUTION_CALLBACK_HANDLER Callback,
+    _In_ uint8_t UseQTIP,
+    _In_ uint8_t OverrideGlobalQTIPSettings
     );
 
 _IRQL_requires_max_(PASSIVE_LEVEL)

src/platform/datapath_kqueue.c

@@ -2156,13 +2156,17 @@ CxPlatResolveRoute(
     _Inout_ CXPLAT_ROUTE* Route,
     _In_ uint8_t PathId,
     _In_ void* Context,
-    _In_ CXPLAT_ROUTE_RESOLUTION_CALLBACK_HANDLER Callback
+    _In_ CXPLAT_ROUTE_RESOLUTION_CALLBACK_HANDLER Callback,
+    _In_ uint8_t UseQTIP,
+    _In_ uint8_t OverrideGlobalQTIPSettings
     )
 {
     UNREFERENCED_PARAMETER(Socket);
     UNREFERENCED_PARAMETER(PathId);
     UNREFERENCED_PARAMETER(Context);
     UNREFERENCED_PARAMETER(Callback);
+    UNREFERENCED_PARAMETER(UseQTIP);
+    UNREFERENCED_PARAMETER(OverrideGlobalQTIPSettings);
     Route->State = RouteResolved;
     return QUIC_STATUS_SUCCESS;
 }

src/platform/datapath_raw.c

@@ -234,7 +234,12 @@ CxPlatDpRawRxEthernet(
 
         if (Socket) {
             if (PacketChain->Reserved == L4_TYPE_UDP || PacketChain->Reserved == L4_TYPE_TCP) {
-                uint8_t SocketType = Socket->UseTcp ? L4_TYPE_TCP : L4_TYPE_UDP;
+                //
+                // If we have UseTcp enabled, we should still support UDP type of packets.
+                //
+                // uint8_t SocketType = (Socket->UseTcp && PacketChain->Reserved == L4_TYPE_TCP) ? L4_TYPE_TCP : L4_TYPE_UDP;
+
+                uint8_t SocketType = (Socket->UseTcp) ? L4_TYPE_TCP : L4_TYPE_UDP; // This is the old code
 
                 //
                 // Found a match. Chain and deliver contiguous packets with the same 4-tuple.

src/platform/datapath_raw_socket.c

@@ -1139,4 +1139,4 @@ CxPlatTryAddSocket(
     }
 
     return Status;
-}
+}

src/platform/datapath_winuser.c

@@ -1490,7 +1490,7 @@ SocketCreateUdp(
     Socket->HasFixedRemoteAddress = (Config->RemoteAddress != NULL);
     Socket->Type = CXPLAT_SOCKET_UDP;
     Socket->UseRio = Datapath->UseRio;
-    Socket->UseTcp = Datapath->UseTcp;
+    Socket->UseTcp = Datapath->UseTcp; // TODO: If we ever decide to remove the global execution param, this needs to be updated.
     if (Config->LocalAddress) {
         CxPlatConvertToMappedV6(Config->LocalAddress, &Socket->LocalAddress);
     } else {

src/platform/datapath_xplat.c

@@ -409,9 +409,16 @@ CxPlatResolveRoute(
     _Inout_ CXPLAT_ROUTE* Route,
     _In_ uint8_t PathId,
     _In_ void* Context,
-    _In_ CXPLAT_ROUTE_RESOLUTION_CALLBACK_HANDLER Callback
+    _In_ CXPLAT_ROUTE_RESOLUTION_CALLBACK_HANDLER Callback,
+    _In_ uint8_t UseQTIP,
+    _In_ uint8_t OverrideGlobalQTIPSettings
     )
 {
+    if (OverrideGlobalQTIPSettings) {
+        Socket->UseTcp = UseQTIP;
+        Route->UseQTIP = UseQTIP;
+        Route->AppDidSetQTIP = TRUE;
+    }
     if (Socket->UseTcp || Route->DatapathType == CXPLAT_DATAPATH_TYPE_RAW ||
         (Route->DatapathType == CXPLAT_DATAPATH_TYPE_UNKNOWN &&
         Socket->RawSocketAvailable && !IS_LOOPBACK(Route->RemoteAddress))) {

src/platform/unittest/DataPathTest.cpp

@@ -569,7 +569,7 @@ struct CxPlatSocket {
                 // This is a connected socket and its route must be resolved
                 // to be able to send traffic.
                 //
-                InitStatus = CxPlatResolveRoute(Socket, &Route, 0, &Route, ResolveRouteComplete);
+                InitStatus = CxPlatResolveRoute(Socket, &Route, 0, &Route, ResolveRouteComplete, FALSE, FALSE);
                 //
                 // Duonic sets up static neighbor entries, so CxPlatResolveRoute should
                 // complete synchronously. If this changes, we will need to add code to

src/test/MsQuicTests.h

@@ -407,7 +407,8 @@ QuicTestConnectAndPing(
     _In_ bool UseSendBuffer,
     _In_ bool UnidirectionalStreams,
     _In_ bool ServerInitiatedStreams,
-    _In_ bool FifoScheduling
+    _In_ bool FifoScheduling,
+    _In_ bool SendUdpToQtipListener
     );
 
 //
@@ -820,6 +821,7 @@ typedef struct {
     uint8_t UnidirectionalStreams;
     uint8_t ServerInitiatedStreams;
     uint8_t FifoScheduling;
+    uint8_t SendUdpToQtipListener;
 } QUIC_RUN_CONNECT_AND_PING_PARAMS;
 
 #pragma pack(pop)