From 2eabd3b4baac3c2dbfce0ae5de738a537aa666f4 Mon Sep 17 00:00:00 2001 From: Kegan Hollern Date: Fri, 6 Dec 2024 20:38:03 -0600 Subject: [PATCH] feat(openai): max_completion_tokens now used over max_tokens for chat_completeions endpoint. --- src/lib/server/endpoints/openai/endpointOai.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/server/endpoints/openai/endpointOai.ts b/src/lib/server/endpoints/openai/endpointOai.ts index 1699545dee3..27df26a8b0c 100644 --- a/src/lib/server/endpoints/openai/endpointOai.ts +++ b/src/lib/server/endpoints/openai/endpointOai.ts @@ -111,6 +111,8 @@ export const endpointOAIParametersSchema = z.object({ }), }) .default({}), + /* enable use of max_completion_tokens in place of max_tokens */ + useCompletionTokens: z.boolean().default(true), }); export async function endpointOai( @@ -125,6 +127,7 @@ export async function endpointOai( defaultQuery, multimodal, extraBody, + useCompletionTokens, } = endpointOAIParametersSchema.parse(input); let OpenAI; @@ -246,7 +249,7 @@ export async function endpointOai( model: model.id ?? model.name, messages: messagesOpenAI, stream: true, - max_tokens: parameters?.max_new_tokens, + ...(useCompletionTokens) ? {max_completion_tokens: parameters?.max_new_tokens} : {max_tokens: parameters?.max_new_tokens}, stop: parameters?.stop, temperature: parameters?.temperature, top_p: parameters?.top_p,