feat(openai): max_completion_tokens now used over max_tokens for chat…

…_completeions endpoint.
huggingface · Dec 7, 2024 · 2eabd3b · 2eabd3b
1 parent 4c2de79
commit 2eabd3b
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/src/lib/server/endpoints/openai/endpointOai.ts b/src/lib/server/endpoints/openai/endpointOai.ts
@@ -111,6 +111,8 @@ export const endpointOAIParametersSchema = z.object({
 			}),
 		})
 		.default({}),
+	/* enable use of max_completion_tokens in place of max_tokens */
+	useCompletionTokens: z.boolean().default(true),
 });
 
 export async function endpointOai(
@@ -125,6 +127,7 @@ export async function endpointOai(
 		defaultQuery,
 		multimodal,
 		extraBody,
+		useCompletionTokens,
 	} = endpointOAIParametersSchema.parse(input);
 
 	let OpenAI;
@@ -246,7 +249,7 @@ export async function endpointOai(
 				model: model.id ?? model.name,
 				messages: messagesOpenAI,
 				stream: true,
-				max_tokens: parameters?.max_new_tokens,
+				...(useCompletionTokens) ? {max_completion_tokens: parameters?.max_new_tokens} : {max_tokens: parameters?.max_new_tokens},
 				stop: parameters?.stop,
 				temperature: parameters?.temperature,
 				top_p: parameters?.top_p,