feat(media): support audio & video files (#1408)

src/lib/components/chat/ChatMessage.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
 	import { marked, type MarkedOptions } from "marked";
 	import markedKatex from "marked-katex-extension";
-	import type { Message, MessageFile } from "$lib/types/Message";
+	import type { Message } from "$lib/types/Message";
 	import { afterUpdate, createEventDispatcher, tick } from "svelte";
 	import { deepestChild } from "$lib/utils/deepestChild";
 	import { page } from "$app/stores";
@@ -30,7 +30,6 @@
 	} from "$lib/types/MessageUpdate";
 	import { base } from "$app/paths";
 	import { useConvTreeStore } from "$lib/stores/convTree";
-	import Modal from "../Modal.svelte";
 	import ToolUpdate from "./ToolUpdate.svelte";
 	import { useSettingsStore } from "$lib/stores/settings";
 	import DOMPurify from "isomorphic-dompurify";
@@ -209,30 +208,8 @@
 	const convTreeStore = useConvTreeStore();
 
 	$: if (message.children?.length === 0) $convTreeStore.leaf = message.id;
-
-	$: modalImageToShow = null as MessageFile | null;
 </script>
 
-{#if modalImageToShow}
-	<!-- show the image file full screen, click outside to exit -->
-	<Modal width="sm:max-w-[500px]" on:close={() => (modalImageToShow = null)}>
-		{#if modalImageToShow.type === "hash"}
-			<img
-				src={urlNotTrailing + "/output/" + modalImageToShow.value}
-				alt="input from user"
-				class="aspect-auto"
-			/>
-		{:else}
-			<!-- handle the case where this is a base64 encoded image -->
-			<img
-				src={`data:${modalImageToShow.mime};base64,${modalImageToShow.value}`}
-				alt="input from user"
-				class="aspect-auto"
-			/>
-		{/if}
-	</Modal>
-{/if}
-
 {#if message.from === "assistant"}
 	<div
 		class="group relative -mb-4 flex items-start justify-start gap-4 pb-4 leading-relaxed"
@@ -259,23 +236,7 @@
 			{#if message.files?.length}
 				<div class="flex h-fit flex-wrap gap-x-5 gap-y-2">
 					{#each message.files as file}
-						<!-- handle the case where this is a hash that points to an image in the db, hash is always 64 char long -->
-						<button on:click={() => (modalImageToShow = file)}>
-							{#if file.type === "hash"}
-								<img
-									src={urlNotTrailing + "/output/" + file.value}
-									alt="output from assistant"
-									class="my-2 aspect-auto max-h-48 cursor-pointer rounded-lg shadow-lg xl:max-h-56"
-								/>
-							{:else}
-								<!-- handle the case where this is a base64 encoded image -->
-								<img
-									src={`data:${file.mime};base64,${file.value}`}
-									alt="output from assistant"
-									class="my-2 aspect-auto max-h-48 cursor-pointer rounded-lg shadow-lg xl:max-h-56"
-								/>
-							{/if}
-						</button>
+						<UploadedFile {file} canClose={false} isPreview={false} />
 					{/each}
 				</div>
 			{/if}
@@ -400,13 +361,7 @@
 			{#if message.files?.length}
 				<div class="flex w-fit gap-4 px-5">
 					{#each message.files as file}
-						{#if file.mime.startsWith("image/")}
-							<button on:click={() => (modalImageToShow = file)}>
-								<UploadedFile {file} canClose={false} />
-							</button>
-						{:else}
-							<UploadedFile {file} canClose={false} />
-						{/if}
+						<UploadedFile {file} canClose={false} isPreview={false} />
 					{/each}
 				</div>
 			{/if}

src/lib/components/chat/UploadedFile.svelte

@@ -5,8 +5,16 @@
 	import CarbonClose from "~icons/carbon/close";
 	import CarbonDocumentBlank from "~icons/carbon/document-blank";
 
+	import Modal from "../Modal.svelte";
+	import AudioPlayer from "../players/AudioPlayer.svelte";
+
 	export let file: MessageFile;
 	export let canClose = true;
+	export let isPreview = false;
+
+	$: showModal = false;
+	$: urlNotTrailing = $page.url.pathname.replace(/\/$/, "");
+
 	const dispatch = createEventDispatcher<{ close: void }>();
 
 	function truncateMiddle(text: string, maxLength: number): string {
@@ -20,47 +28,92 @@
 
 		return `${start}…${end}`;
 	}
+
+	const isImage = (mime: string) =>
+		mime.startsWith("image/") || mime === "webp" || mime === "jpeg" || mime === "png";
+
+	const isAudio = (mime: string) => mime.startsWith("audio/");
+	const isVideo = (mime: string) => mime.startsWith("video/");
+
+	$: isClickable = isImage(file.mime) && !isPreview;
 </script>
 
-<div
-	class="group relative flex items-center rounded-xl shadow-sm"
-	class:w-24={file.mime.startsWith("image/")}
-	class:w-72={!file.mime.startsWith("image/")}
->
-	{#if file.mime.startsWith("image/")}
-		<div class="size-24 overflow-hidden rounded-xl">
+{#if showModal && isClickable}
+	<!-- show the image file full screen, click outside to exit -->
+	<Modal width="sm:max-w-[500px]" on:close={() => (showModal = false)}>
+		{#if file.type === "hash"}
+			<img
+				src={urlNotTrailing + "/output/" + file.value}
+				alt="input from user"
+				class="aspect-auto"
+			/>
+		{:else}
+			<!-- handle the case where this is a base64 encoded image -->
 			<img
+				src={`data:${file.mime};base64,${file.value}`}
+				alt="input from user"
+				class="aspect-auto"
+			/>
+		{/if}
+	</Modal>
+{/if}
+
+<button on:click={() => (showModal = true)} disabled={!isClickable}>
+	<div class="group relative flex items-center rounded-xl shadow-sm">
+		{#if isImage(file.mime)}
+			<div class=" overflow-hidden rounded-xl" class:size-24={isPreview} class:size-48={!isPreview}>
+				<img
+					src={file.type === "base64"
+						? `data:${file.mime};base64,${file.value}`
+						: urlNotTrailing + "/output/" + file.value}
+					alt={file.name}
+					class="h-full w-full bg-gray-200 object-cover dark:bg-gray-800"
+				/>
+			</div>
+		{:else if isAudio(file.mime)}
+			<AudioPlayer
 				src={file.type === "base64"
 					? `data:${file.mime};base64,${file.value}`
-					: $page.url.pathname + "/output/" + file.value}
-				alt={file.name}
-				class="h-full w-full bg-gray-200 object-cover dark:bg-gray-800"
+					: urlNotTrailing + "/output/" + file.value}
+				name={truncateMiddle(file.name, 28)}
 			/>
-		</div>
-	{:else}
-		<div
-			class="flex h-14 w-72 items-center gap-2 overflow-hidden rounded-xl border border-gray-200 bg-white p-2 dark:border-gray-800 dark:bg-gray-900"
-		>
+		{:else if isVideo(file.mime)}
 			<div
-				class="grid size-10 flex-none place-items-center rounded-lg bg-gray-100 dark:bg-gray-800"
+				class="border-1 w-72 overflow-clip rounded-xl border-gray-200 bg-white dark:border-gray-800 dark:bg-gray-900"
 			>
-				<CarbonDocumentBlank class="text-base text-gray-700 dark:text-gray-300" />
+				<!-- svelte-ignore a11y-media-has-caption -->
+				<video
+					src={file.type === "base64"
+						? `data:${file.mime};base64,${file.value}`
+						: urlNotTrailing + "/output/" + file.value}
+					controls
+				/>
 			</div>
-			<dl class="flex flex-col truncate leading-tight">
-				<dd class="text-sm">
-					{truncateMiddle(file.name, 28)}
-				</dd>
-				<dt class="text-xs text-gray-400">{file.mime.split("/")[1].toUpperCase()}</dt>
-			</dl>
-		</div>
-	{/if}
-	<!-- add a button on top that removes the image -->
-	{#if canClose}
-		<button
-			class="invisible absolute -right-2 -top-2 grid size-6 place-items-center rounded-full border bg-black group-hover:visible dark:border-gray-700"
-			on:click={() => dispatch("close")}
-		>
-			<CarbonClose class=" text-xs  text-white" />
-		</button>
-	{/if}
-</div>
+		{:else}
+			<div
+				class="flex h-14 w-72 items-center gap-2 overflow-hidden rounded-xl border border-gray-200 bg-white p-2 dark:border-gray-800 dark:bg-gray-900"
+			>
+				<div
+					class="grid size-10 flex-none place-items-center rounded-lg bg-gray-100 dark:bg-gray-800"
+				>
+					<CarbonDocumentBlank class="text-base text-gray-700 dark:text-gray-300" />
+				</div>
+				<dl class="flex flex-col truncate leading-tight">
+					<dd class="text-sm">
+						{truncateMiddle(file.name, 28)}
+					</dd>
+					<dt class="text-xs text-gray-400">{file.mime.split("/")[1].toUpperCase()}</dt>
+				</dl>
+			</div>
+		{/if}
+		<!-- add a button on top that removes the image -->
+		{#if canClose}
+			<button
+				class="invisible absolute -right-2 -top-2 grid size-6 place-items-center rounded-full border bg-black group-hover:visible dark:border-gray-700"
+				on:click={() => dispatch("close")}
+			>
+				<CarbonClose class=" text-xs  text-white" />
+			</button>
+		{/if}
+	</div>
+</button>

src/lib/components/players/AudioPlayer.svelte

@@ -0,0 +1,79 @@
+<script lang="ts">
+	export let src: string;
+	export let name: string;
+
+	import CarbonPause from "~icons/carbon/pause";
+	import CarbonPlay from "~icons/carbon/play";
+
+	let time = 0;
+	let duration = 0;
+	let paused = true;
+
+	function format(time: number) {
+		if (isNaN(time)) return "...";
+
+		const minutes = Math.floor(time / 60);
+		const seconds = Math.floor(time % 60);
+
+		return `${minutes}:${seconds < 10 ? `0${seconds}` : seconds}`;
+	}
+
+	function seek(e: PointerEvent) {
+		if (!e.currentTarget) return;
+		const { left, width } = (e.currentTarget as HTMLElement).getBoundingClientRect();
+
+		let p = (e.clientX - left) / width;
+		if (p < 0) p = 0;
+		if (p > 1) p = 1;
+
+		time = p * duration;
+	}
+</script>
+
+<div
+	class="flex h-14 w-72 items-center gap-4 rounded-2xl border border-gray-200 bg-white p-2.5 text-gray-600 shadow-sm transition-all dark:border-gray-800 dark:bg-gray-900 dark:text-gray-300"
+>
+	<audio
+		{src}
+		bind:currentTime={time}
+		bind:duration
+		bind:paused
+		preload="metadata"
+		on:ended={() => {
+			time = 0;
+		}}
+	/>
+
+	<button
+		class="mx-auto my-auto aspect-square size-8 rounded-full border border-gray-400 bg-gray-100 dark:border-gray-800 dark:bg-gray-700"
+		aria-label={paused ? "play" : "pause"}
+		on:click={() => (paused = !paused)}
+	>
+		{#if paused}
+			<CarbonPlay class="mx-auto my-auto text-gray-600 dark:text-gray-300" />
+		{:else}
+			<CarbonPause class="mx-auto my-auto text-gray-600 dark:text-gray-300" />
+		{/if}
+	</button>
+
+	<div class="overflow-hidden">
+		<div class="truncate font-medium">{name}</div>
+
+		<div class="flex items-center gap-2">
+			<span class="text-xs">{format(time)}</span>
+			<div
+				class="relative h-2 flex-1 rounded-full bg-gray-200 dark:bg-gray-700"
+				on:pointerdown={() => {
+					paused = true;
+				}}
+				on:pointerup={seek}
+			>
+				<div
+					class="absolute inset-0 h-full bg-gray-400 dark:bg-gray-600"
+					style="width: {(time / duration) * 100}%"
+				/>
+			</div>
+			<span class="text-xs">{duration ? format(duration) : "--:--"}</span>
+		</div>
+	</div>
+</div>