Refactor summarization so it gets called from backend (#456)

* Refactor summarization

* get rid of debug log

* remove old todo

src/lib/server/summarize.ts

@@ -0,0 +1,22 @@
+import { buildPrompt } from "$lib/buildPrompt";
+import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndpoint";
+import { defaultModel } from "$lib/server/models";
+
+export async function summarize(prompt: string) {
+	const userPrompt = `Please summarize the following message: \n` + prompt;
+
+	const summaryPrompt = await buildPrompt({
+		messages: [{ from: "user", content: userPrompt }],
+		preprompt:
+			"You are a summarization AI. Your task is to summarize user requests, in a single sentence of less than 5 words. Do not try to answer questions, just summarize the user's request.",
+		model: defaultModel,
+	});
+
+	const generated_text = await generateFromDefaultEndpoint(summaryPrompt);
+
+	if (generated_text) {
+		return generated_text;
+	}
+
+	return null;
+}

src/routes/conversation/[id]/+page.svelte

@@ -34,12 +34,6 @@
 	let pending = false;
 	let loginRequired = false;
 
-	async function summarizeTitle(id: string) {
-		await fetch(`${base}/conversation/${id}/summarize`, {
-			method: "POST",
-		});
-	}
-
 	// this function is used to send new message to the backends
 	async function writeMessage(message: string, messageId = randomUUID()) {
 		if (!message.trim()) return;
@@ -146,15 +140,7 @@
 			// reset the websearchmessages
 			webSearchMessages = [];
 
-			// do title summarization
-			// TODO: we should change this to wait until there is an assistant response.
-			if (messages.filter((m) => m.from === "user").length === 1) {
-				summarizeTitle($page.params.id)
-					.then(() => invalidate(UrlDependency.ConversationList))
-					.catch(console.error);
-			} else {
-				await invalidate(UrlDependency.ConversationList);
-			}
+			await invalidate(UrlDependency.ConversationList);
 		} catch (err) {
 			if (err instanceof Error && err.message.includes("overloaded")) {
 				$error = "Too much traffic, please try again.";

src/routes/conversation/[id]/+server.ts

@@ -17,7 +17,8 @@ import { AwsClient } from "aws4fetch";
 import type { MessageUpdate } from "$lib/types/MessageUpdate";
 import { runWebSearch } from "$lib/server/websearch/runWebSearch";
 import type { WebSearch } from "$lib/types/WebSearch";
-import { abortedGenerations } from "$lib/server/abortedGenerations.js";
+import { abortedGenerations } from "$lib/server/abortedGenerations";
+import { summarize } from "$lib/server/summarize";
 
 export async function POST({ request, fetch, locals, params, getClientAddress }) {
 	const id = z.string().parse(params.id);
@@ -167,6 +168,10 @@ export async function POST({ request, fetch, locals, params, getClientAddress })
 			}
 
 			async function saveLast(generated_text: string) {
+				if (!conv) {
+					throw new Error("Conversation not found");
+				}
+
 				const lastMessage = messages[messages.length - 1];
 
 				if (lastMessage) {
@@ -195,6 +200,7 @@ export async function POST({ request, fetch, locals, params, getClientAddress })
 						{
 							$set: {
 								messages,
+								title: (await summarize(newPrompt)) ?? conv.title,
 								updatedAt: new Date(),
 							},
 						}
@@ -277,6 +283,7 @@ export async function POST({ request, fetch, locals, params, getClientAddress })
 				{
 					$set: {
 						messages,
+						title: (await summarize(newPrompt)) ?? conv.title,
 						updatedAt: new Date(),
 					},
 				}

src/routes/conversation/[id]/summarize/+server.ts

@@ -1,74 +0,0 @@
-import { RATE_LIMIT } from "$env/static/private";
-import { buildPrompt } from "$lib/buildPrompt";
-import { authCondition } from "$lib/server/auth";
-import { collections } from "$lib/server/database";
-import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndpoint";
-import { defaultModel } from "$lib/server/models";
-import { ERROR_MESSAGES } from "$lib/stores/errors";
-import { error } from "@sveltejs/kit";
-import { ObjectId } from "mongodb";
-
-export async function POST({ params, locals, getClientAddress }) {
-	const convId = new ObjectId(params.id);
-
-	const conversation = await collections.conversations.findOne({
-		_id: convId,
-		...authCondition(locals),
-	});
-
-	if (!conversation) {
-		throw error(404, "Conversation not found");
-	}
-
-	const userId = locals.user?._id ?? locals.sessionId;
-
-	await collections.messageEvents.insertOne({
-		userId: userId,
-		createdAt: new Date(),
-		ip: getClientAddress(),
-	});
-
-	const nEvents = Math.max(
-		await collections.messageEvents.countDocuments({ userId }),
-		await collections.messageEvents.countDocuments({ ip: getClientAddress() })
-	);
-
-	if (RATE_LIMIT != "" && nEvents > parseInt(RATE_LIMIT)) {
-		throw error(429, ERROR_MESSAGES.rateLimited);
-	}
-
-	const firstMessage = conversation.messages.find((m) => m.from === "user");
-
-	const userPrompt =
-		`Please summarize the following message as a single sentence of less than 5 words:\n` +
-		firstMessage?.content;
-
-	const prompt = await buildPrompt({
-		messages: [{ from: "user", content: userPrompt }],
-		model: defaultModel,
-	});
-	const generated_text = await generateFromDefaultEndpoint(prompt);
-
-	if (generated_text) {
-		await collections.conversations.updateOne(
-			{
-				_id: convId,
-				...authCondition(locals),
-			},
-			{
-				$set: { title: generated_text },
-			}
-		);
-	}
-
-	return new Response(
-		JSON.stringify(
-			generated_text
-				? {
-						title: generated_text,
-				  }
-				: {}
-		),
-		{ headers: { "Content-Type": "application/json" } }
-	);
-}