Intra-vendor model routing feature enhancements

all-in-one-apim/modules/distribution/resources/api_templates/ai_api_endpoints_seq_template.xml

@@ -0,0 +1,87 @@
+<!--
+ ~ Copyright (c) 2025, WSO2 LLC (http://www.wso2.org) All Rights Reserved.
+ ~
+ ~ WSO2 LLC licenses this file to you under the Apache License,
+ ~ Version 2.0 (the "License"); you may not use this file except
+ ~ in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~    http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied.  See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+        #####################  Define the macros ######################
+
+        #macro ( print_string_if_exist $field $default)
+        #if($field && "$field" !="")
+        "$field"
+        #else
+        "$default"
+        #end
+        #end
+
+        #macro ( draw_endpoint $type $endpoint )
+        #if($endpoint.endpointSecurityEnabled)
+<class name="org.wso2.carbon.apimgt.gateway.mediators.ApiKeyMediator">
+    <property name="apiKeyIdentifier" value="${endpoint.apiKeyIdentifier}" type="STRING"/>
+    <property name="apiKeyIdentifierType" value="${endpoint.apiKeyIdentifierType}" type="STRING"/>
+    <property name="apiKeyValue" value="${endpoint.apiKeyValue}" type="STRING"/>
+</class>
+        #end
+
+        #if( $endpoint && $endpoint.endpointUuid )
+        #set( $ep_key = "${endpointKey}_API_LLMEndpoint_${endpoint.endpointUuid}" )
+        #else
+        #set( $ep_key = "${endpointKey}_API${deploymentStage}Endpoint")
+        #end
+<call>
+<endpoint xmlns="http://ws.apache.org/ns/synapse" key="$ep_key"/>
+</call>
+        #end
+
+        #macro ( fail_scenario )
+<property name="ERROR_CODE" value="900808"/>
+<property name="ERROR_MESSAGE" value="No active endpoints available. Please try again later."/>
+<property name="CUSTOM_HTTP_SC" value="503"/>
+<sequence key="fault"/>
+        #end
+
+<sequence xmlns="http://ws.apache.org/ns/synapse" name="${endpointKey}_EndpointsSeq${deploymentStage}">
+#if( $defaultEndpoint )
+<filter xpath="$ctx:retryCount > 10">
+<then>
+    #fail_scenario
+</then>
+<else>
+    <property name="retryCount" expression="get-property('default','retryCount') + 1" scope="default"/>
+<switch source="get-property('TARGET_ENDPOINT')">
+    #foreach( $endpoint in $!endpoints )
+    <case regex="$endpoint.endpointUuid">
+        #draw_endpoint($type, $endpoint)
+    </case>
+    #end
+    <case regex="DEFAULT">
+        #draw_endpoint($type, $defaultEndpoint)
+    </case>
+    <case regex="REJECT">
+        #fail_scenario
+    </case>
+    <case regex="EXIT">
+        <loopback/>
+    </case>
+</switch>
+<property name="retryCount" expression="get-property('default','retryCount') + 1" scope="default" action="set"/>
+<class name="org.wso2.carbon.apimgt.gateway.mediators.AIAPIMediator">
+    <property name="llmProviderId" expression="get-property('default','llmProviderId')" type="STRING"/>
+    <property name="direction" value="OUT" type="STRING"/>
+</class>
+<sequence key="${endpointKey}_EndpointsSeq${deploymentStage}" />
+</else>
+</filter>
+#end
+</sequence>

all-in-one-apim/modules/distribution/resources/api_templates/ai_api_template.xml

@@ -1,7 +1,7 @@
 <!--
- ~ Copyright (c) 2005-2010, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
+ ~ Copyright (c) 2025, WSO2 LLC (http://www.wso2.org) All Rights Reserved.
  ~
- ~ WSO2 Inc. licenses this file to you under the Apache License,
+ ~ WSO2 LLC licenses this file to you under the Apache License,
  ~ Version 2.0 (the "License"); you may not use this file except
  ~ in compliance with the License.
  ~ You may obtain a copy of the License at
@@ -26,26 +26,6 @@
         #end
         #end
 
-        #macro ( draw_endpoint $type $endpoint )
-        #if($endpoint.endpointSecurityEnabled)
-<class name="org.wso2.carbon.apimgt.gateway.mediators.ApiKeyMediator">
-    <property name="apiKeyIdentifier" value="${endpoint.apiKeyIdentifier}" type="STRING"/>
-    <property name="apiKeyIdentifierType" value="${endpoint.apiKeyIdentifierType}" type="STRING"/>
-    <property name="apiKeyValue" value="${endpoint.apiKeyValue}" type="STRING"/>
-</class>
-        #end
-
-        #if( $endpoint && $endpoint.endpointUuid )
-            #set( $ep_key = "${endpointKey}_API_LLMEndpoint_${endpoint.endpointUuid}" )
-        #else
-            #set( $ep_key = "${endpointKey}_API${type}Endpoint")
-        #end
-<call>
-    <endpoint xmlns="http://ws.apache.org/ns/synapse" key="$ep_key"/>
-</call>
-<loopback/>
-        #end
-
         ######################## End of Macros ######################################
 
 
@@ -102,8 +82,13 @@
 
         faultSequence=#print_string_if_exist($faultSequence "fault")>
 <inSequence>
+<header name="Accept-Encoding" scope="transport" action="remove"/>
 <property name="llmProviderId" value="$util.escapeXml($!llmProviderId)" scope="default" type="STRING"/>
-<sequence key="_ai_api_request_seq_"/>
+<class name="org.wso2.carbon.apimgt.gateway.mediators.AIAPIMediator">
+    <property name="llmProviderId" expression="get-property('default','llmProviderId')" type="STRING"/>
+    <property name="direction" value="IN" type="STRING"/>
+</class>
+<property name="retryCount" value="0" scope="default"/>
 <property name="api.ut.backendRequestTime" expression="get-property('SYSTEM_TIME')"/>
 ##############  define the filter based on environment type production only, sandbox only , hybrid ############
 
@@ -139,64 +124,19 @@
     <then>
         #if( ($environmentType == 'sandbox') || ($environmentType == 'hybrid' && !$defaultProductionEndpoint)) )
         #if( $defaultSandboxEndpoint )
-        <switch source="get-property('targetEndpoint')">
-            #foreach( $endpoint in $!sandboxEndpoints )
-            <case regex="$endpoint.endpointUuid">
-                #draw_endpoint("sandbox", $endpoint)
-            </case>
-            #end
-            <case regex="DEFAULT">
-                #draw_endpoint("sandbox", $defaultSandboxEndpoint)
-            </case>
-            <case regex="REJECT">
-                <property name="ERROR_CODE" value="900808"/>
-                <property name="ERROR_MESSAGE" value="No active endpoints available. Please try again later."/>
-                <property name="CUSTOM_HTTP_SC" value="503"/>
-                <sequence key="fault"/>
-            </case>
-        </switch>
+        <sequence key="${endpointKey}_EndpointsSeqSANDBOX" />
         #end
         #else
         #if( $defaultProductionEndpoint )
-        <switch source="get-property('targetEndpoint')">
-            #foreach( $endpoint in $!productionEndpoints )
-            <case regex="$endpoint.endpointUuid">
-                #draw_endpoint("production", $endpoint)
-            </case>
-            #end
-            <case regex="DEFAULT">
-                #draw_endpoint("production", $defaultProductionEndpoint)
-            </case>
-            <case regex="REJECT">
-                <property name="ERROR_CODE" value="900808"/>
-                <property name="ERROR_MESSAGE" value="No active endpoints available. Please try again later."/>
-                <property name="CUSTOM_HTTP_SC" value="503"/>
-                <sequence key="fault"/>
-            </case>
-        </switch>
+        <sequence key="${endpointKey}_EndpointsSeqPRODUCTION" />
         #end
         #end
     </then>
     <else>
         #if( $environmentType !='hybrid' )
         #if( $resource.getAuthType() == 'None' && $environmentType == 'sandbox' && $defaultSandboxEndpoint)) )
         #if( $defaultSandboxEndpoint )
-        <switch source="get-property('targetEndpoint')">
-            #foreach( $endpoint in $!sandboxEndpoints )
-            <case regex="$endpoint.endpointUuid">
-                #draw_endpoint("sandbox", $endpoint)
-            </case>
-            #end
-            <case regex="DEFAULT">
-                #draw_endpoint("sandbox", $defaultSandboxEndpoint)
-            </case>
-            <case regex="REJECT">
-                <property name="ERROR_CODE" value="900808"/>
-                <property name="ERROR_MESSAGE" value="No active endpoints available. Please try again later."/>
-                <property name="CUSTOM_HTTP_SC" value="503"/>
-                <sequence key="fault"/>
-            </case>
-        </switch>
+        <sequence key="${endpointKey}_EndpointsSeqSANDBOX" />
         #end
         #else
         <payloadFactory>
@@ -219,42 +159,12 @@
         #end
         #else
         #if( $defaultSandboxEndpoint && $defaultProductionEndpoint )
-        <switch source="get-property('targetEndpoint')">
-            #foreach( $endpoint in $!sandboxEndpoints )
-            <case regex="$endpoint.endpointUuid">
-                #draw_endpoint("sandbox", $endpoint)
-            </case>
-            #end
-            <case regex="DEFAULT">
-                #draw_endpoint("sandbox", $defaultSandboxEndpoint)
-            </case>
-            <case regex="REJECT">
-                <property name="ERROR_CODE" value="900808"/>
-                <property name="ERROR_MESSAGE" value="No active endpoints available. Please try again later."/>
-                <property name="CUSTOM_HTTP_SC" value="503"/>
-                <sequence key="fault"/>
-            </case>
-        </switch>
+        <sequence key="${endpointKey}_EndpointsSeqSANDBOX" />
         #elseif( $defaultProductionEndpoint )
         <sequence key="_sandbox_key_error_"/>
         #elseif( $defaultSandboxEndpoint )
         #if( $resource.getAuthType() == 'None' )
-        <switch source="get-property('targetEndpoint')">
-            #foreach( $endpoint in $!sandboxEndpoints )
-            <case regex="$endpoint.endpointUuid">
-                #draw_endpoint("sandbox", $endpoint)
-            </case>
-            #end
-            <case regex="DEFAULT">
-                #draw_endpoint("sandbox", $defaultSandboxEndpoint)
-            </case>>
-            <case regex="REJECT">
-                <property name="ERROR_CODE" value="900808"/>
-                <property name="ERROR_MESSAGE" value="No active endpoints available. Please try again later."/>
-                <property name="CUSTOM_HTTP_SC" value="503"/>
-                <sequence key="fault"/>
-            </case>
-        </switch>
+        <sequence key="${endpointKey}_EndpointsSeqSANDBOX" />
         #else
         <sequence key="_production_key_error_"/>
         #end
@@ -265,7 +175,6 @@
 #end
 </inSequence>
 <outSequence>
-<sequence key="_ai_api_response_seq_"/>
 <send/>
 </outSequence>
         </resource>

all-in-one-apim/modules/distribution/resources/operation_policies/definitions/modelFailover_v1.j2

@@ -1 +1,3 @@
-<property action="set" name="failoverConfigs" value="{{failoverConfigs}}" scope="transport" />
+<class name="org.wso2.carbon.apimgt.gateway.mediators.FailoverMediator">
+    <property action="set" name="failoverConfigs" value="{{failoverConfigs}}" scope="transport" />
+</class>

all-in-one-apim/modules/distribution/resources/operation_policies/specifications/modelFailover_v1.json

@@ -3,7 +3,7 @@
   "name": "modelFailover",
   "version": "v1",
   "displayName": "Model Failover",
-  "description": "This policy allows you to route traffic to a different model(s) when the primary model fails",
+  "description": "This policy enables traffic rerouting to one or more fallback models if the target model fails, ensuring continued availability",
   "policyAttributes": [
     {
       "name": "failoverConfigs",

all-in-one-apim/modules/integration/tests-common/clients/publisher/docs/OperationPolicyDataDTO.md

@@ -14,7 +14,7 @@ Name | Type | Description | Notes
 **description** | **String** |  |  [optional]
 **applicableFlows** | **List<String>** |  |  [optional]
 **supportedGateways** | **List<String>** |  |  [optional]
-**supportedApiTypes** | **List<Object>** | Supported API types as an array of strings, or an array of maps [HTTP, SOAP]  [{apiType: HTTP, subType: AI}, {apiType: SOAP}]  |  [optional]
+**supportedApiTypes** | **List<Object>** | Supported API types as an array of strings, or an array of maps [HTTP, SOAP] [{apiType: HTTP, subType: AI}, {apiType: SOAP}]  |  [optional]
 **isAPISpecific** | **Boolean** |  |  [optional]
 **md5** | **String** |  |  [optional]
 **policyAttributes** | [**List<OperationPolicySpecAttributeDTO>**](OperationPolicySpecAttributeDTO.md) |  |  [optional]

all-in-one-apim/modules/integration/tests-common/clients/publisher/src/gen/java/org/wso2/am/integration/clients/publisher/api/v1/dto/OperationPolicyDataDTO.java

@@ -272,11 +272,11 @@ public OperationPolicyDataDTO supportedApiTypes(List<Object> supportedApiTypes)
         }
 
     /**
-        * Supported API types as an array of strings, or an array of maps [HTTP, SOAP]  [{apiType: HTTP, subType: AI}, {apiType: SOAP}]
+        * Supported API types as an array of strings, or an array of maps [HTTP, SOAP] [{apiType: HTTP, subType: AI}, {apiType: SOAP}] 
     * @return supportedApiTypes
     **/
         @javax.annotation.Nullable
-      @ApiModelProperty(example = "[\"HTTP\",\"SOAP\"]", value = "Supported API types as an array of strings, or an array of maps [HTTP, SOAP]  [{apiType: HTTP, subType: AI}, {apiType: SOAP}] ")
+      @ApiModelProperty(example = "[\"HTTP\",\"SOAP\"]", value = "Supported API types as an array of strings, or an array of maps [HTTP, SOAP] [{apiType: HTTP, subType: AI}, {apiType: SOAP}] ")
 
     public List<Object> getSupportedApiTypes() {
         return supportedApiTypes;