fix(endpoints): fix for tool calling on hf inference with openai endpoint type (#1754)

* fix(endpoints): fix for tool calling on hf inference with openai endpoint type

* moar fix

* fix: typechecks

src/lib/server/endpoints/openai/openAIChatToTextGenerationStream.ts

@@ -49,10 +49,47 @@ export async function* openAIChatToTextGenerationStream(
 	let generatedText = "";
 	let tokenId = 0;
 	const toolCalls: ToolCallWithParameters[] = [];
+	let toolBuffer = ""; // XXX: hack because tools seem broken on tgi openai endpoints?
+
 	for await (const completion of completionStream) {
 		const { choices } = completion;
 		const content = choices[0]?.delta?.content ?? "";
 		const last = choices[0]?.finish_reason === "stop" || choices[0]?.finish_reason === "length";
+
+		// if the last token is a stop and the tool buffer is not empty, yield it as a generated_text
+		if (choices[0]?.finish_reason === "stop" && toolBuffer.length > 0) {
+			yield {
+				token: {
+					id: tokenId++,
+					special: true,
+					logprob: 0,
+					text: "",
+				},
+				generated_text: toolBuffer,
+				details: null,
+			} as TextGenerationStreamOutput;
+			break;
+		}
+
+		// weird bug where the parameters are streamed in like this
+		if (choices[0]?.delta?.tool_calls) {
+			const calls = Array.isArray(choices[0].delta.tool_calls)
+				? choices[0].delta.tool_calls
+				: [choices[0].delta.tool_calls];
+
+			if (
+				calls.length === 1 &&
+				calls[0].index === 0 &&
+				calls[0].id === "" &&
+				calls[0].type === "function" &&
+				!!calls[0].function &&
+				calls[0].function.name === null
+			) {
+				toolBuffer += calls[0].function.arguments;
+				continue;
+			}
+		}
+
 		if (content) {
 			generatedText = generatedText + content;
 		}

src/lib/server/textGeneration/tools.ts

@@ -314,6 +314,22 @@ function isValidCallObject(call: unknown): call is Record<string, unknown> {
 }
 
 function parseExternalCall(callObj: Record<string, unknown>) {
+	let toolCall = callObj;
+	if (
+		isValidCallObject(callObj) &&
+		"function" in callObj &&
+		isValidCallObject(callObj.function) &&
+		"_name" in callObj.function
+	) {
+		toolCall = {
+			tool_name: callObj["function"]["_name"],
+			parameters: {
+				...callObj["function"],
+				_name: undefined,
+			},
+		};
+	}
+
 	const nameFields = ["tool_name", "name"] as const;
 	const parametersFields = ["parameters", "arguments", "parameter_definitions"] as const;
 
@@ -323,14 +339,14 @@ function parseExternalCall(callObj: Record<string, unknown>) {
 	};
 
 	for (const name of nameFields) {
-		if (callObj[name]) {
-			groupedCall.tool_name = callObj[name] as string;
+		if (toolCall[name]) {
+			groupedCall.tool_name = toolCall[name] as string;
 		}
 	}
 
 	for (const name of parametersFields) {
-		if (callObj[name]) {
-			groupedCall.parameters = callObj[name] as Record<string, string>;
+		if (toolCall[name]) {
+			groupedCall.parameters = toolCall[name] as Record<string, string>;
 		}
 	}