vllm serve /home/jovyan/qwen3-8b --reasoning-parser qwen3 --max-num-seqs 4 --tool-call-parser hermes --enable-auto-tool-choice --served-model-name model --max-model-len 32648 --guided-decoding-backend xgrammar --max-model-len 32648   -tp 2

from lmformatenforcer.external.jsonschemaobject import JsonSchemaObject

from openai import OpenAI

openai_api_key = "EMPTY"
openai_api_base = "http://localhost:8000/v1"

client = OpenAI(
    api_key=openai_api_key,
    base_url=openai_api_base,
)
tools = [
    {
        "type": "function",
        "function": {
            "name": "get_current_weather",
            "description": "Get the current weather in a given location",
            "parameters": {
                "type": "object",
                "properties": {
                    "city": {
                        "type": "string",
                        "description":
                        "The city to find the weather for, e.g. 'Vienna'",
                        "default": "Vienna",
                    },
                    "country": {
                        "type":
                        "string",
                        "description":
                        "The country that the city is in, e.g. 'Austria'",
                    },
                    "unit": {
                        "type": "string",
                        "description":
                        "The unit to fetch the temperature in",
                        "enum": ["celsius", "fahrenheit"],
                    },
                },
                "required": ["country", "unit"],
            },
        },
    },
    {
        "type": "function",
        "function": {
            "name": "get_forecast",
            "description": "Get the weather forecast for a given location",
            "parameters": {
                "type": "object",
                "properties": {
                    "city": {
                        "type": "string",
                        "description":
                        "The city to get the forecast for, e.g. 'Vienna'",
                        "default": "Vienna",
                    },
                    "country": {
                        "type":
                        "string",
                        "description":
                        "The country that the city is in, e.g. 'Austria'",
                    },
                    "days": {
                        "type":
                        "integer",
                        "description":
                        "Number of days to get the forecast for (1-7)",
                    },
                    "unit": {
                        "type": "string",
                        "description":
                        "The unit to fetch the temperature in",
                        "enum": ["celsius", "fahrenheit"],
                    },
                },
                "required": ["country", "days", "unit"],
            },
        },
    },
]

messages = [
    {
        "role": "user",
        "content": "Hi! How are you doing today?"
    },
    {
        "role": "assistant",
        "content": "I'm doing well! How can I help you?"
    },
    {
        "role":
        "user",
        "content":
        "Can you tell me what the current weather is in Berlin and the "\
        "forecast for the next 5 days, in fahrenheit?",
    },
]


chat_completion = client.chat.completions.create(
    messages=messages,
    model='',
    tools=tools,
    tool_choice="required"
)
print("Chat completion response:")
print(f"Chat completion: {chat_completion}")
for choice in chat_completion.choices:
    if choice.message.tool_calls:
        print(
            f"Tool calls: {choice.message.tool_calls}")
    else:
        print("No tool calls found.")
assert chat_completion.choices[0].message.tool_calls is not None
assert len(chat_completion.choices[0].message.tool_calls) > 0

Chat completion response:
Chat completion: ChatCompletion(id='chatcmpl-13503360686a458ea55e3a1f93897ea0', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content='', refusal=None, role='assistant', annotations=None, audio=None, function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='chatcmpl-tool-387aad93bc9345ac846534e05aa43b1d', function=Function(arguments='{"city": "Berlin", "country": "Germany", "unit": "fahrenheit"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='chatcmpl-tool-363045d0bf3d40cfa386329ec357aec2', function=Function(arguments='{"city": "Berlin", "country": "Germany", "days": 5, "unit": "fahrenheit"}', name='get_forecast'), type='function')], reasoning_content=None), stop_reason=None)], created=1748949549, model='model', object='chat.completion', service_tier=None, system_fingerprint=None, usage=CompletionUsage(completion_tokens=530, prompt_tokens=474, total_tokens=1004, completion_tokens_details=None, prompt_tokens_details=None), prompt_logprobs=None, kv_transfer_params=None)
Tool calls: [ChatCompletionMessageToolCall(id='chatcmpl-tool-387aad93bc9345ac846534e05aa43b1d', function=Function(arguments='{"city": "Berlin", "country": "Germany", "unit": "fahrenheit"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='chatcmpl-tool-363045d0bf3d40cfa386329ec357aec2', function=Function(arguments='{"city": "Berlin", "country": "Germany", "days": 5, "unit": "fahrenheit"}', name='get_forecast'), type='function')]