System message as part of Conversation

src/lib/components/InferencePlayground/InferencePlayground.svelte

@@ -20,16 +20,17 @@
 
 	export let models: ModelEntryWithTokenizer[];
 
-	const startMessage: ChatCompletionInputMessage = { role: "user", content: "" };
+	const startMessageUser: ChatCompletionInputMessage = { role: "user", content: "" };
+	const startMessageSystem: ChatCompletionInputMessage = { role: "system", content: "" };
 
 	let conversation: Conversation = {
 		model: models[0],
 		config: defaultGenerationConfig,
-		messages: [{ ...startMessage }],
+		messages: [{ ...startMessageUser }],
+		systemMessage: startMessageSystem,
 		streaming: true,
 	};
 
-	let systemMessage: ChatCompletionInputMessage = { role: "system", content: "" };
 	let hfToken: string | undefined = import.meta.env.VITE_HF_TOKEN;
 	let viewCode = false;
 	let showTokenModal = false;
@@ -41,7 +42,7 @@
 	$: systemPromptSupported = isSystemPromptSupported(conversation.model);
 	$: {
 		if (!systemPromptSupported) {
-			systemMessage = { role: "system", content: "" };
+			conversation.systemMessage = { role: "system", content: "" };
 		}
 	}
 
@@ -61,8 +62,8 @@
 	}
 
 	function reset() {
-		systemMessage.content = "";
-		conversation.messages = [{ ...startMessage }];
+		conversation.systemMessage.content = "";
+		conversation.messages = [{ ...startMessageUser }];
 	}
 
 	function abort() {
@@ -98,12 +99,11 @@
 							conversation.messages = [...conversation.messages];
 						}
 					},
-					abortController,
-					systemMessage
+					abortController
 				);
 			} else {
 				waitForNonStreaming = true;
-				const newMessage = await handleNonStreamingResponse(hf, conversation, systemMessage);
+				const newMessage = await handleNonStreamingResponse(hf, conversation);
 				// check if the user did not abort the request
 				if (waitForNonStreaming) {
 					conversation.messages = [...conversation.messages, newMessage];
@@ -162,7 +162,7 @@
 				placeholder={systemPromptSupported
 					? "Enter a custom prompt"
 					: "System prompt is not supported with the chosen model."}
-				bind:value={systemMessage.content}
+				bind:value={conversation.systemMessage.content}
 				class="absolute inset-x-0 bottom-0 h-full resize-none bg-transparent px-3 pt-10 text-sm outline-none"
 			></textarea>
 		</div>

src/lib/components/InferencePlayground/inferencePlaygroundUtils.ts

@@ -11,17 +11,17 @@ export async function handleStreamingResponse(
 	hf: HfInference,
 	conversation: Conversation,
 	onChunk: (content: string) => void,
-	abortController: AbortController,
-	systemMessage?: ChatCompletionInputMessage
+	abortController: AbortController
 ): Promise<void> {
+	const { model, systemMessage } = conversation;
 	const messages = [
-		...(isSystemPromptSupported(conversation.model) && systemMessage?.content?.length ? [systemMessage] : []),
+		...(isSystemPromptSupported(model) && systemMessage.content?.length ? [systemMessage] : []),
 		...conversation.messages,
 	];
 	let out = "";
 	for await (const chunk of hf.chatCompletionStream(
 		{
-			model: conversation.model.id,
+			model: model.id,
 			messages,
 			temperature: conversation.config.temperature,
 			max_tokens: conversation.config.maxTokens,
@@ -37,16 +37,16 @@ export async function handleStreamingResponse(
 
 export async function handleNonStreamingResponse(
 	hf: HfInference,
-	conversation: Conversation,
-	systemMessage?: ChatCompletionInputMessage
+	conversation: Conversation
 ): Promise<ChatCompletionInputMessage> {
+	const { model, systemMessage } = conversation;
 	const messages = [
-		...(isSystemPromptSupported(conversation.model) && systemMessage?.content?.length ? [systemMessage] : []),
+		...(isSystemPromptSupported(model) && systemMessage.content?.length ? [systemMessage] : []),
 		...conversation.messages,
 	];
 
 	const response = await hf.chatCompletion({
-		model: conversation.model,
+		model: model.id,
 		messages,
 		temperature: conversation.config.temperature,
 		max_tokens: conversation.config.maxTokens,

src/lib/components/InferencePlayground/types.ts

@@ -6,6 +6,7 @@ export type Conversation = {
 	model: ModelEntryWithTokenizer;
 	config: GenerationConfig;
 	messages: ChatCompletionInputMessage[];
+	systemMessage: ChatCompletionInputMessage;
 	streaming: boolean;
 };