Image examples cleanup

AudioPlayer.tsx

@@ -0,0 +1,80 @@
+import { useEffect, useMemo, useRef, useState } from 'react'
+import WaveSurfer from 'wavesurfer.js'
+// @ts-ignore: No types for timeline.esm.js
+// import Timeline from 'wavesurfer.js/dist/plugins/timeline.esm.js'
+import TimelinePlugin from 'wavesurfer.js/dist/plugins/timeline.esm.js'
+import API from '../API' // Correct import for the API class
+
+const AudioPlayer = ({ src }: { src: string }) => {
+  const containerRef = useRef<HTMLDivElement>(null)
+  const wavesurferRef = useRef<WaveSurfer | null>(null)
+  const [isPlaying, setIsPlaying] = useState(false)
+  // const plugins = useMemo(() => [TimelinePlugin.create()], [])
+
+  const bottomTimeline = TimelinePlugin.create({
+    height: 16,
+    timeInterval: 0.1,
+    primaryLabelInterval: 1,
+    style: {
+      fontSize: '10px',
+      // color: '#6A3274',
+    },
+  })
+
+  // Initialize WaveSurfer when component mounts
+  useEffect(() => {
+    if (!containerRef.current) return
+
+    // Get proxied URL to bypass CORS
+    const proxiedUrl = API.getProxiedUrl(src)
+
+    // Create an instance of WaveSurfer
+    wavesurferRef.current = WaveSurfer.create({
+      container: containerRef.current,
+      waveColor: 'rgb(200, 0, 200)',
+      progressColor: 'rgb(100, 0, 100)',
+      url: proxiedUrl, // Use the proxied URL
+      minPxPerSec: 100,
+      barWidth: 10,
+      barRadius: 10,
+      barGap: 2,
+      mediaControls: true,
+
+      // plugins: [bottomTimeline],
+    })
+
+    // Play on click
+    wavesurferRef.current.on('interaction', () => {
+      wavesurferRef.current?.play()
+      setIsPlaying(true)
+    })
+
+    // Rewind to the beginning on finished playing
+    wavesurferRef.current.on('finish', () => {
+      wavesurferRef.current?.setTime(0)
+      setIsPlaying(false)
+    })
+
+    // Update playing state
+    wavesurferRef.current.on('play', () => setIsPlaying(true))
+    wavesurferRef.current.on('pause', () => setIsPlaying(false))
+
+    // Cleanup on unmount
+    return () => {
+      wavesurferRef.current?.destroy()
+    }
+  }, [src])
+
+  const handlePlayPause = () => {
+    wavesurferRef.current?.playPause()
+  }
+
+  return (
+    <div className="">
+      <div ref={containerRef} />
+      {/* <button onClick={handlePlayPause}>{isPlaying ? 'Pause' : 'Play'}</button> */}
+    </div>
+  )
+}
+
+export default AudioPlayer

backend/examples.py

@@ -47,8 +47,7 @@ def build_description(
 
     if i == 0:
         fake_det = data_none["fake_det"]
-
-        return f"detected: {fake_det}"
+        return {"detected": fake_det}
     elif i == 1:
         # Fixed metrics
         det = data_none["watermark_det"]
@@ -56,30 +55,36 @@ def build_description(
         bit_acc = data_none["bit_acc"]
 
         # Dynamic metrics
-        metrics_output = []
+        metrics_output = {}
         for metric in quality_metrics:
             value = float(data_none[metric])
-            metrics_output.append(f"{metric}: {value:.2f}")
+            metrics_output[metric] = round(value, 2)
 
         # Fixed metrics output
-        fixed_metrics_output = (
-            f" detected: {det} p_value: {p_value:.2f} bit_acc: {bit_acc:.2f}"
+        metrics_output.update(
+            {
+                "detected": det,
+                "p_value": round(p_value, 2),
+                "bit_acc": round(bit_acc, 2),
+            }
         )
 
-        # Combine all outputs
-        return " ".join(metrics_output) + f"{fixed_metrics_output}"
+        return metrics_output
     elif i == 2:
         fake_det = data_attack["fake_det"]
-
-        return f"det: {fake_det}"
-    elif i == 3:
+        return {"detected": fake_det}
+    elif i >= 3:  # REVISIT THIS, it used to be == 3
         det = data_attack["watermark_det"]
-
         p_value = float(data_attack["p_value"])
         word_acc = data_attack["word_acc"]
         bit_acc = data_attack["bit_acc"]
 
-        return f"word_acc: {word_acc:.2f} detected: {det} p_value: {p_value:.2f} bit_acc: {bit_acc:.2f}"
+        return {
+            "word_acc": round(word_acc, 2),
+            "detected": det,
+            "p_value": round(p_value, 2),
+            "bit_acc": round(bit_acc, 2),
+        }
 
 
 def build_infos(abs_path: Path, datatype: str, dataset_name: str, db_key: str):
@@ -176,7 +181,10 @@ def build_infos(abs_path: Path, datatype: str, dataset_name: str, db_key: str):
                     files = [
                         {
                             "image_url": f,
-                            "description": f"{n}\n{build_description(i, data_none, data_attack, quality_metrics)}",
+                            "name": n,
+                            "metadata": build_description(
+                                i, data_none, data_attack, quality_metrics
+                            ),
                             **(
                                 {"audio_url": f.replace(".png", ".wav")}
                                 if datatype == "audio" and f.endswith(".png")

frontend/src/components/AudioGallery.tsx

@@ -0,0 +1,30 @@
+import React from 'react'
+import AudioPlayer from './AudioPlayer'
+import type { ExamplesData } from './Examples'
+
+interface GalleryProps {
+  selectedModel: string
+  selectedAttack: string
+  examples: {
+    [model: string]: {
+      [attack: string]: ExamplesData[]
+    }
+  }
+}
+
+const AudioGallery: React.FC<GalleryProps> = ({ selectedModel, selectedAttack, examples }) => {
+  const exampleItems = examples[selectedModel][selectedAttack]
+  return (
+    <div className="example-display">
+      {exampleItems.map((item, index) => (
+        <div key={index} className="example-item">
+          <p>{item.name}</p>
+          {item.audio_url && <AudioPlayer src={item.audio_url} />}
+          <img src={item.image_url} alt={item.name} className="example-image" />
+        </div>
+      ))}
+    </div>
+  )
+}
+
+export default AudioGallery

frontend/src/components/Examples.tsx

@@ -1,17 +1,23 @@
 import React, { useState, useEffect } from 'react'
 import API from '../API'
-import AudioPlayer from './AudioPlayer'
 import LoadingSpinner from './LoadingSpinner'
+import ImageGallery from './ImageGallery'
+import AudioGallery from './AudioGallery'
+import VideoGallery from './VideoGallery'
 
 interface ExamplesProps {
   fileType: 'image' | 'audio' | 'video'
 }
 
-type ExamplesData = {
+// Move ExamplesData type export to allow import in Galleries.tsx
+export type ExamplesData = {
   image_url: string
   audio_url?: string
   video_url?: string
-  description: string
+  name: string
+  metadata: {
+    [key: string]: string | boolean
+  }
 }
 
 const Examples = ({ fileType }: ExamplesProps) => {
@@ -55,45 +61,8 @@ const Examples = ({ fileType }: ExamplesProps) => {
       })
   }, [fileType])
 
-  // Define the Gallery component within this file
-  const Gallery = ({
-    selectedModel,
-    selectedAttack,
-    examples,
-    fileType,
-  }: {
-    selectedModel: string
-    selectedAttack: string
-    examples: {
-      [model: string]: {
-        [attack: string]: ExamplesData[]
-      }
-    }
-    fileType: 'image' | 'audio' | 'video'
-  }) => {
-    const exampleItems = examples[selectedModel][selectedAttack]
-
-    return (
-      <div className="example-display">
-        {exampleItems.map((item, index) => (
-          <div key={index} className="example-item">
-            <p>{item.description}</p>
-            {fileType === 'image' && (
-              <img src={item.image_url} alt={item.description} className="example-image" />
-            )}
-            {fileType === 'audio' && item.audio_url && (
-              <>
-                <AudioPlayer src={item.audio_url} />
-                <img src={item.image_url} alt={item.description} className="example-image" />
-              </>
-            )}
-            {fileType === 'video' && (
-              <video controls src={item.video_url} className="example-video" />
-            )}
-          </div>
-        ))}
-      </div>
-    )
+  if (loading) {
+    return <LoadingSpinner />
   }
 
   return (
@@ -132,15 +101,27 @@ const Examples = ({ fileType }: ExamplesProps) => {
         )}
       </div>
 
-      {loading && <LoadingSpinner />}
       {error && <p className="error">Error: {error}</p>}
 
-      {selectedModel && selectedAttack && (
-        <Gallery
+      {selectedModel && selectedAttack && fileType === 'image' && (
+        <ImageGallery
+          selectedModel={selectedModel}
+          selectedAttack={selectedAttack}
+          examples={examples}
+        />
+      )}
+      {selectedModel && selectedAttack && fileType === 'audio' && (
+        <AudioGallery
+          selectedModel={selectedModel}
+          selectedAttack={selectedAttack}
+          examples={examples}
+        />
+      )}
+      {selectedModel && selectedAttack && fileType === 'video' && (
+        <VideoGallery
           selectedModel={selectedModel}
           selectedAttack={selectedAttack}
           examples={examples}
-          fileType={fileType}
         />
       )}
     </div>

frontend/src/components/Galleries.tsx

@@ -0,0 +1,5 @@
+import ImageGallery from './ImageGallery'
+import AudioGallery from './AudioGallery'
+import VideoGallery from './VideoGallery'
+
+export { ImageGallery, AudioGallery, VideoGallery }

frontend/src/components/ImageGallery.tsx

@@ -0,0 +1,276 @@
+import React from 'react'
+import type { ExamplesData } from './Examples'
+import { groupByNameAndVariant } from './galleryUtils'
+
+interface GalleryProps {
+  selectedModel: string
+  selectedAttack: string
+  examples: {
+    [model: string]: {
+      [attack: string]: ExamplesData[]
+    }
+  }
+}
+
+const Metadata: React.FC<{ metadata: Record<string, string | number | boolean> }> = ({
+  metadata,
+}) => (
+  <fieldset className="fieldset w-full p-4 rounded border border-gray-700 bg-base-200 mb-4">
+    <legend className="fieldset-legend font-semibold">Metadata</legend>
+    <ul className="text-xs flex flex-row gap-4">
+      {(() => {
+        const entries = Object.entries(metadata || {}).filter(([k]) => k !== 'image_url')
+        const detectedIdx = entries.findIndex(([k]) => k === 'detected')
+        if (detectedIdx > -1) {
+          const [detected] = entries.splice(detectedIdx, 1)
+          entries.unshift(detected)
+        }
+        return entries.map(([k, v]) => (
+          <li key={k}>
+            <span className="font-mono">{k}</span>: {String(v)}
+          </li>
+        ))
+      })()}
+    </ul>
+  </fieldset>
+)
+
+const VariantSelector: React.FC<{
+  variantKeys: string[]
+  selectedVariant: string
+  setSelectedVariant: (v: string) => void
+}> = ({ variantKeys, selectedVariant, setSelectedVariant }) => (
+  <fieldset className="fieldset w-full p-4 rounded border border-gray-700 bg-base-200 mb-4">
+    <legend className="fieldset-legend font-semibold">Variants</legend>
+    <div className="mb-2 flex gap-4 flex-wrap">
+      {variantKeys.map((variant, idx) => (
+        <label key={variant} className="flex items-center gap-1 cursor-pointer">
+          <input
+            type="radio"
+            name={`variant-selector`}
+            value={variant}
+            checked={selectedVariant === variant}
+            onChange={() => setSelectedVariant(variant)}
+          />
+          <span className="text-xs font-semibold">
+            {variant} <span className="opacity-60">[{idx + 1}]</span>
+          </span>
+        </label>
+      ))}
+    </div>
+  </fieldset>
+)
+
+const ExampleDetailsSection: React.FC<{ children: React.ReactNode }> = ({ children }) => (
+  <fieldset className="fieldset w-full p-4 rounded border border-gray-700 bg-base-200 mt-6">
+    {children}
+  </fieldset>
+)
+
+const ImageGallery: React.FC<GalleryProps> = ({ selectedModel, selectedAttack, examples }) => {
+  const exampleItems = examples[selectedModel][selectedAttack]
+
+  // Group by image name (name after removing variant prefix), and for each, map variant to metrics
+  const grouped = groupByNameAndVariant(exampleItems)
+
+  const imageNames = Object.keys(grouped)
+  const [selectedImage, setSelectedImage] = React.useState(imageNames[0] || '')
+  const variants = grouped[selectedImage] || {}
+  const variantKeys = Object.keys(variants)
+  const [selectedVariant, setSelectedVariant] = React.useState(variantKeys[0] || '')
+
+  const [zoom, setZoom] = React.useState<{ x: number; y: number } | null>(null)
+  const [imgDims, setImgDims] = React.useState<{ width: number; height: number }>({
+    width: 0,
+    height: 0,
+  })
+  const imgRef = React.useRef<HTMLImageElement | null>(null)
+
+  // Set a fixed zoom box size for all variants
+  const ZOOM_BOX_SIZE = 300
+  const [zoomLevel, setZoomLevel] = React.useState(2) // 2x default zoom
+
+  // Keyboard shortcut for variant switching
+  React.useEffect(() => {
+    const handler = (e: KeyboardEvent) => {
+      if (document.activeElement && (document.activeElement as HTMLElement).tagName === 'INPUT')
+        return
+      const idx = parseInt(e.key, 10)
+      if (!isNaN(idx) && idx > 0 && idx <= variantKeys.length) {
+        setSelectedVariant(variantKeys[idx - 1])
+      }
+    }
+    window.addEventListener('keydown', handler)
+    return () => window.removeEventListener('keydown', handler)
+  }, [variantKeys])
+
+  // Calculate the minimum width and height across all variants for the selected image
+  const variantImages = variantKeys.map((v) => variants[v]?.image_url).filter(Boolean)
+  const [naturalDims, setNaturalDims] = React.useState<{ width: number; height: number } | null>(
+    null
+  )
+
+  React.useEffect(() => {
+    let isMounted = true
+    if (variantImages.length > 0) {
+      Promise.all(
+        variantImages.map(
+          (src) =>
+            new Promise<{ width: number; height: number }>((resolve) => {
+              const img = new window.Image()
+              img.onload = () => resolve({ width: img.naturalWidth, height: img.naturalHeight })
+              img.src = src!
+            })
+        )
+      ).then((dimsArr) => {
+        if (!isMounted) return
+        const first = dimsArr[0]
+        const allSame = dimsArr.every((d) => d.width === first.width && d.height === first.height)
+        if (!allSame) {
+          setNaturalDims(null)
+        } else {
+          setNaturalDims(first)
+        }
+      })
+    }
+    return () => {
+      isMounted = false
+    }
+  }, [variantImages.join(','), selectedImage])
+
+  // Place the zoom level slider outside of the zoom hover logic
+  const zoomSlider = (
+    <div className="flex flex-col items-center mb-4 w-full">
+      <label className="font-semibold text-xs mb-1">Zoom Level: {zoomLevel}x</label>
+      <input
+        type="range"
+        min={2}
+        max={6}
+        step={0.1}
+        value={zoomLevel}
+        onChange={(e) => setZoomLevel(Number(e.target.value))}
+        className="range range-xs w-48"
+      />
+    </div>
+  )
+
+  // If no image is selected, show a message
+  if (!variantImages.length) {
+    return (
+      <div className="w-full mt-12 flex items-center justify-center">
+        <div className="text-gray-500">
+          No images available. Please select another model and attack.
+        </div>
+      </div>
+    )
+  }
+
+  // Ensure the main container allows scrolling and images retain their natural size
+  return (
+    <div className="w-full overflow-auto" style={{ minHeight: '100vh' }}>
+      <div className="example-display">
+        <div className="mb-4">
+          <fieldset className="fieldset">
+            <legend className="fieldset-legend">Image</legend>
+            <select
+              className="select select-bordered"
+              value={selectedImage || ''}
+              onChange={(e) => {
+                setSelectedImage(e.target.value || '')
+                const newVariants = grouped[e.target.value] || {}
+                const newVariantKeys = Object.keys(newVariants)
+                setSelectedVariant(newVariantKeys[0] || '')
+              }}
+            >
+              {imageNames.map((name) => (
+                <option key={name} value={name}>
+                  {name}
+                </option>
+              ))}
+            </select>
+          </fieldset>
+        </div>
+        {selectedImage && selectedVariant && variants[selectedVariant] && (
+          <>
+            <VariantSelector
+              variantKeys={variantKeys}
+              selectedVariant={selectedVariant}
+              setSelectedVariant={setSelectedVariant}
+            />
+            <Metadata metadata={variants[selectedVariant].metadata || {}} />
+            {zoomSlider}
+            <ExampleDetailsSection>
+              <div
+                style={{
+                  width: '100%',
+                  display: 'flex',
+                  flexDirection: 'row',
+                  justifyContent: 'center',
+                }}
+              >
+                <div style={{ position: 'relative', flex: 'none', display: 'inline-block' }}>
+                  {/* Original image and overlay */}
+                  {naturalDims ? (
+                    <img
+                      ref={imgRef}
+                      src={variants[selectedVariant].image_url as string}
+                      alt={selectedImage}
+                      width={naturalDims.width}
+                      height={naturalDims.height}
+                      style={{ display: 'block', cursor: 'zoom-in' }}
+                      onMouseMove={(e) => {
+                        const rect = (e.target as HTMLImageElement).getBoundingClientRect()
+                        const x = e.clientX - rect.left
+                        const y = e.clientY - rect.top
+                        setZoom({ x, y })
+                      }}
+                      onMouseLeave={() => setZoom(null)}
+                    />
+                  ) : (
+                    <div className="text-red-500 font-bold">
+                      Image sizes do not match across variants!
+                    </div>
+                  )}
+                  {/* Zoomed box - now overlays on top of the cursor position */}
+                  {zoom &&
+                    naturalDims &&
+                    (() => {
+                      const { width, height } = naturalDims
+                      const halfBox = ZOOM_BOX_SIZE / (2 * zoomLevel)
+                      let centerX = zoom.x
+                      let centerY = zoom.y
+                      centerX = Math.max(halfBox, Math.min(width - halfBox, centerX))
+                      centerY = Math.max(halfBox, Math.min(height - halfBox, centerY))
+                      const bgX = centerX * zoomLevel - ZOOM_BOX_SIZE / 2
+                      const bgY = centerY * zoomLevel - ZOOM_BOX_SIZE / 2
+                      return (
+                        <div
+                          style={{
+                            position: 'absolute',
+                            left: zoom.x - ZOOM_BOX_SIZE / 2, // use raw cursor position for overlay
+                            top: zoom.y - ZOOM_BOX_SIZE / 2,
+                            width: ZOOM_BOX_SIZE,
+                            height: ZOOM_BOX_SIZE,
+                            boxSizing: 'border-box',
+                            backgroundImage: `url(${variants[selectedVariant]?.image_url})`,
+                            backgroundPosition: `-${bgX}px -${bgY}px`,
+                            backgroundSize: `${width * zoomLevel}px ${height * zoomLevel}px`,
+                            backgroundRepeat: 'no-repeat',
+                            zIndex: 10,
+                            pointerEvents: 'none',
+                            cursor: 'zoom-in',
+                          }}
+                        />
+                      )
+                    })()}
+                </div>
+              </div>
+            </ExampleDetailsSection>
+          </>
+        )}
+      </div>
+    </div>
+  )
+}
+
+export default ImageGallery

frontend/src/components/VideoGallery.tsx

@@ -0,0 +1,28 @@
+import React from 'react'
+import type { ExamplesData } from './Examples'
+
+interface GalleryProps {
+  selectedModel: string
+  selectedAttack: string
+  examples: {
+    [model: string]: {
+      [attack: string]: ExamplesData[]
+    }
+  }
+}
+
+const VideoGallery: React.FC<GalleryProps> = ({ selectedModel, selectedAttack, examples }) => {
+  const exampleItems = examples[selectedModel][selectedAttack]
+  return (
+    <div className="example-display">
+      {exampleItems.map((item, index) => (
+        <div key={index} className="example-item">
+          <p>{item.name}</p>
+          <video controls src={item.video_url} className="example-video" />
+        </div>
+      ))}
+    </div>
+  )
+}
+
+export default VideoGallery

frontend/src/components/galleryUtils.ts

@@ -0,0 +1,36 @@
+// Common parsing and grouping utilities for galleries
+
+export const VARIANT_PREFIX_MAP: { [variant: string]: string } = {
+  attacked_wmd: 'attacked_wmd_',
+  attacked: 'attacked_',
+  wmd: 'wmd_',
+}
+
+export function getVariant(name: string): string {
+  return (
+    Object.entries(VARIANT_PREFIX_MAP).find(([, prefix]) => name.startsWith(prefix))?.[0] ||
+    'original'
+  )
+}
+
+export function getImageName(name: string, variant: string): string {
+  if (variant !== 'original' && variant in VARIANT_PREFIX_MAP) {
+    return name.replace(VARIANT_PREFIX_MAP[variant], '')
+  }
+  return name
+}
+
+export function groupByNameAndVariant<T extends { name: string }>(
+  items: T[]
+): {
+  [name: string]: { [variant: string]: T }
+} {
+  const grouped: { [name: string]: { [variant: string]: T } } = {}
+  items.forEach((item) => {
+    const variant = getVariant(item.name)
+    const imageName = getImageName(item.name, variant)
+    if (!grouped[imageName]) grouped[imageName] = {}
+    grouped[imageName][variant] = item
+  })
+  return grouped
+}