Configurable websearch timeout (#1427)

Made playwright websearch timeout value configurable through an env variable

.env

@@ -33,6 +33,7 @@ PLAYWRIGHT_ADBLOCKER=true
 WEBSEARCH_ALLOWLIST=`[]` # if it's defined, allow websites from only this list.
 WEBSEARCH_BLOCKLIST=`[]` # if it's defined, block websites from this list.
 WEBSEARCH_JAVASCRIPT=true # CPU usage reduces by 60% on average by disabling javascript. Enable to improve website compatibility
+WEBSEARCH_TIMEOUT = 3500 # in milliseconds, determines how long to wait to load a page before timing out
 
 # Parameters to enable open id login
 OPENID_CONFIG=`{

src/lib/server/websearch/scrape/playwright.ts

@@ -70,9 +70,13 @@ export async function withPage<T>(
 		const page = await ctx.newPage();
 		env.PLAYWRIGHT_ADBLOCKER === "true" && (await blocker.enableBlockingInPage(page));
 
-		const res = await page.goto(url, { waitUntil: "load", timeout: 3500 }).catch(() => {
-			logger.warn(`Failed to load page within 2s: ${url}`);
-		});
+		const res = await page
+			.goto(url, { waitUntil: "load", timeout: parseInt(env.WEBSEARCH_TIMEOUT) })
+			.catch(() => {
+				console.warn(
+					`Failed to load page within ${parseInt(env.WEBSEARCH_TIMEOUT) / 1000}s: ${url}`
+				);
+			});
 
 		// await needed here so that we don't close the context before the callback is done
 		return await callback(page, res ?? undefined);