Introduce Playwright to run the browser-based benchmark in headless mode

.claude/settings.local.json

@@ -0,0 +1,14 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(npm run bench:*)",
+      "Bash(npm install:*)",
+      "Bash(npm run bench:cli:*)",
+      "Bash(timeout 120 npm run bench:cli -- Xenova/all-MiniLM-L6-v2 feature-extraction --mode warm --repeats 2 --device wasm)",
+      "Bash(timeout 120 npm run bench:cli -- Xenova/all-MiniLM-L6-v2 feature-extraction --mode warm --repeats 2 --device webgpu)",
+      "Bash(timeout 180 npm run bench:cli -- Xenova/all-MiniLM-L6-v2 feature-extraction --mode cold --repeats 2 --device wasm)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

bench-web/README.md

@@ -1,18 +1,50 @@
-# bench-web (warm/cold, repeats, p50/p90)
+# bench-web (warm/cold, repeats, p50/p90, CPU/GPU)
 
 ## Setup
 ```bash
 cd bench-web
 npm i
+npm run bench:install  # Install Playwright browsers for CLI mode
 ```
 
-## Run (dev)
+## Run (Interactive UI)
 ```bash
 npm run dev
 # open http://localhost:5173
 ```
 
+## Run (CLI with Playwright)
+```bash
+npm run bench:cli -- <model> <task> --mode <warm|cold> --repeats <n> --device <wasm|webgpu> [--browser <chromium|firefox|webkit>] [--headed true]
+```
+
+### Examples
+```bash
+# WASM (CPU) benchmark
+npm run bench:cli -- Xenova/all-MiniLM-L6-v2 feature-extraction --mode warm --repeats 3 --device wasm
+
+# WebGPU benchmark
+npm run bench:cli -- Xenova/all-MiniLM-L6-v2 feature-extraction --mode warm --repeats 3 --device webgpu
+
+# Cold mode
+npm run bench:cli -- Xenova/all-MiniLM-L6-v2 feature-extraction --mode cold --repeats 3 --device wasm
+
+# With Firefox
+npm run bench:cli -- Xenova/all-MiniLM-L6-v2 feature-extraction --mode warm --repeats 3 --device wasm --browser firefox
+
+# Headed mode (for debugging)
+npm run bench:cli -- Xenova/all-MiniLM-L6-v2 feature-extraction --mode warm --repeats 3 --device wasm --headed true
+```
+
 ## How it works
+### Interactive UI
 - **warm**: prefetch once (non-measured) → auto-reload → measure `repeats` times with disk caches populated.
 - **cold**: clear Cache Storage & IndexedDB, then measure in the same tab
   - Note: only the 1st iteration is strictly cold within a single page session.
+
+### CLI Mode
+- Starts a Vite dev server and launches headless browser via Playwright
+- **warm**: prefetch once (non-measured) → measure `repeats` times with caches populated (no page reload)
+- **cold**: clears all caches before each run
+- **device**: `wasm` for CPU, `webgpu` for GPU acceleration
+- Supports Chromium, Firefox, and WebKit browsers

bench-web/index.html

@@ -32,12 +32,17 @@
       </select>
       <label for="repeats">Repeats</label>
       <input id="repeats" type="number" value="3" min="1" style="width: 5rem;" />
+      <label for="device">Device</label>
+      <select id="device">
+        <option value="webgpu" selected>webgpu</option>
+        <option value="wasm">wasm (CPU)</option>
+      </select>
     </div>
     <div class="row">
       <button id="run">Run benchmark</button>
       <span id="status"></span>
     </div>
-    <pre id="out">{}</pre>
+    <code><pre id="out">{}</pre></code>
     <script type="module" src="/src/main.ts"></script>
   </body>
 </html>

bench-web/package-lock.json

@@ -11,6 +11,9 @@
         "@huggingface/transformers": "^3.7.4"
       },
       "devDependencies": {
+        "@playwright/test": "^1.55.1",
+        "playwright": "^1.55.1",
+        "tsx": "^4.20.6",
         "typescript": "^5.9.3",
         "vite": "^7.1.7"
       }
@@ -927,6 +930,22 @@
         "node": ">=18.0.0"
       }
     },
+    "node_modules/@playwright/test": {
+      "version": "1.55.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.55.1.tgz",
+      "integrity": "sha512-IVAh/nOJaw6W9g+RJVlIQJ6gSiER+ae6mKQ5CX1bERzQgbC1VSeBlwdvczT7pxb0GWiyrxH4TGKbMfDb4Sq/ig==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright": "1.55.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/@protobufjs/aspromise": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
@@ -1497,6 +1516,19 @@
         "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
       }
     },
+    "node_modules/get-tsconfig": {
+      "version": "4.10.1",
+      "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.10.1.tgz",
+      "integrity": "sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "resolve-pkg-maps": "^1.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
+      }
+    },
     "node_modules/global-agent": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz",
@@ -1702,6 +1734,53 @@
       "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==",
       "license": "MIT"
     },
+    "node_modules/playwright": {
+      "version": "1.55.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.55.1.tgz",
+      "integrity": "sha512-cJW4Xd/G3v5ovXtJJ52MAOclqeac9S/aGGgRzLabuF8TnIb6xHvMzKIa6JmrRzUkeXJgfL1MhukP0NK6l39h3A==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.55.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.55.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.55.1.tgz",
+      "integrity": "sha512-Z6Mh9mkwX+zxSlHqdr5AOcJnfp+xUWLCt9uKV18fhzA8eyxUd8NUWzAjxUh55RZKSYwDGX0cfaySdhZJGMoJ+w==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/playwright/node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
     "node_modules/postcss": {
       "version": "8.5.6",
       "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz",
@@ -1755,6 +1834,16 @@
         "node": ">=12.0.0"
       }
     },
+    "node_modules/resolve-pkg-maps": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
+      "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
+      "dev": true,
+      "license": "MIT",
+      "funding": {
+        "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
+      }
+    },
     "node_modules/roarr": {
       "version": "2.15.4",
       "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz",
@@ -1945,6 +2034,26 @@
       "license": "0BSD",
       "optional": true
     },
+    "node_modules/tsx": {
+      "version": "4.20.6",
+      "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz",
+      "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "esbuild": "~0.25.0",
+        "get-tsconfig": "^4.7.5"
+      },
+      "bin": {
+        "tsx": "dist/cli.mjs"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      },
+      "optionalDependencies": {
+        "fsevents": "~2.3.3"
+      }
+    },
     "node_modules/type-fest": {
       "version": "0.13.1",
       "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz",

bench-web/package.json

@@ -6,13 +6,18 @@
   "scripts": {
     "dev": "vite",
     "build": "vite build",
-    "preview": "vite preview"
+    "preview": "vite preview",
+    "bench:cli": "tsx src/cli.ts",
+    "bench:install": "playwright install"
   },
   "dependencies": {
     "@huggingface/transformers": "^3.7.4"
   },
   "devDependencies": {
+    "@playwright/test": "^1.55.1",
+    "playwright": "^1.55.1",
+    "tsx": "^4.20.6",
     "typescript": "^5.9.3",
     "vite": "^7.1.7"
   }
-}
+}

bench-web/src/cli.ts

@@ -0,0 +1,103 @@
+import { chromium, firefox, webkit, Browser, Page } from "playwright";
+import { createServer } from "vite";
+
+// CLI for running browser benchmarks headlessly via Playwright
+
+const modelId = process.argv[2] || "Xenova/distilbert-base-uncased";
+const task = process.argv[3] || "feature-extraction";
+
+function getArg(name: string, def?: string) {
+  const i = process.argv.indexOf(`--${name}`);
+  if (i !== -1 && i + 1 < process.argv.length) return process.argv[i + 1];
+  return def;
+}
+
+const mode = getArg("mode", "warm") as "warm" | "cold";
+const repeats = Math.max(1, parseInt(getArg("repeats", "3") || "3", 10));
+const device = getArg("device", "webgpu") as "webgpu" | "wasm";
+const browserType = getArg("browser", "chromium") as "chromium" | "firefox" | "webkit";
+const headed = getArg("headed") === "true";
+
+async function main() {
+  console.log(`Model   : ${modelId}`);
+  console.log(`Task    : ${task}`);
+  console.log(`Mode    : ${mode}`);
+  console.log(`Repeats : ${repeats}`);
+  console.log(`Device  : ${device}`);
+  console.log(`Browser : ${browserType}`);
+  console.log(`Headed  : ${headed}`);
+
+  // Start Vite dev server
+  const server = await createServer({
+    server: {
+      port: 5173,
+      strictPort: false,
+    },
+    logLevel: "error",
+  });
+
+  await server.listen();
+
+  const port = server.config.server.port || 5173;
+  const url = `http://localhost:${port}`;
+
+  console.log(`Vite server started at ${url}`);
+
+  let browser: Browser;
+  const launchOptions = {
+    headless: !headed,
+    args: device === "wasm"
+      ? ["--disable-gpu", "--disable-software-rasterizer"]
+      : ["--enable-unsafe-webgpu", "--enable-features=Vulkan"]
+  };
+
+  switch (browserType) {
+    case "firefox":
+      browser = await firefox.launch(launchOptions);
+      break;
+    case "webkit":
+      browser = await webkit.launch(launchOptions);
+      break;
+    case "chromium":
+    default:
+      browser = await chromium.launch(launchOptions);
+      break;
+  }
+
+  try {
+    const context = await browser.newContext();
+    const page = await context.newPage();
+
+    // Expose console logs
+    page.on("console", (msg) => {
+      const type = msg.type();
+      if (type === "error" || type === "warning") {
+        console.log(`[browser ${type}]`, msg.text());
+      }
+    });
+
+    // Navigate to the app
+    await page.goto(url);
+
+    // Wait for the page to be ready
+    await page.waitForSelector("#run");
+
+    console.log("\nStarting benchmark...");
+
+    // Use the exposed CLI function from main.ts
+    const result = await page.evaluate(({ modelId, task, mode, repeats, device }) => {
+      return (window as any).runBenchmarkCLI({ modelId, task, mode, repeats, device });
+    }, { modelId, task, mode, repeats, device });
+
+    console.log("\n" + JSON.stringify(result, null, 2));
+
+  } finally {
+    await browser.close();
+    await server.close();
+  }
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});

bench-web/src/main.ts

@@ -7,6 +7,7 @@ const modelEl = document.getElementById("model") as HTMLInputElement;
 const taskEl = document.getElementById("task") as HTMLSelectElement;
 const modeEl = document.getElementById("mode") as HTMLSelectElement;
 const repeatsEl = document.getElementById("repeats") as HTMLInputElement;
+const deviceEl = document.getElementById("device") as HTMLSelectElement;
 
 function now() { return performance.now(); }
 function percentile(values: number[], q: number) {
@@ -15,11 +16,11 @@ function percentile(values: number[], q: number) {
   const i0 = Math.floor(i), i1 = Math.ceil(i);
   return i0 === i1 ? a[i0] : a[i0] + (a[i1] - a[i0]) * (i - i0);
 }
-async function clearCaches({ clearSession=false }: { clearSession?: boolean } = {}) {
+async function clearCaches({ clearSession = false }: { clearSession?: boolean } = {}) {
   try {
     const keys = await caches.keys();
     await Promise.all(keys.map((k) => caches.delete(k)));
-  } catch {}
+  } catch { }
   try {
     const anyIDB: any = indexedDB as any;
     if (typeof anyIDB.databases === "function") {
@@ -29,26 +30,26 @@ async function clearCaches({ clearSession=false }: { clearSession?: boolean } =
       indexedDB.deleteDatabase("transformers-cache");
       indexedDB.deleteDatabase("model-cache");
     }
-  } catch {}
+  } catch { }
   try {
     localStorage.clear();
     if (clearSession) sessionStorage.clear();
-  } catch {}
+  } catch { }
 }
-async function benchOnce(modelId: string, task: string) {
+async function benchOnce(modelId: string, task: string, device: string) {
   const t0 = now();
-  const pipe = await pipeline(task, modelId, {});
+  const pipe = await pipeline(task, modelId, { device });
   const t1 = now();
   const t2 = now();
   await pipe("The quick brown fox jumps over the lazy dog.");
   const t3 = now();
   return { load_ms: +(t1 - t0).toFixed(1), first_infer_ms: +(t3 - t2).toFixed(1) };
 }
-async function runMany(modelId: string, task: string, repeats: number) {
+async function runMany(modelId: string, task: string, repeats: number, device: string) {
   const loads: number[] = [];
   const firsts: number[] = [];
   for (let i = 0; i < repeats; i++) {
-    const r = await benchOnce(modelId, task);
+    const r = await benchOnce(modelId, task, device);
     loads.push(r.load_ms);
     firsts.push(r.first_infer_ms);
   }
@@ -57,11 +58,11 @@ async function runMany(modelId: string, task: string, repeats: number) {
     first_infer_ms: { p50: +percentile(firsts, 0.5).toFixed(1), p90: +percentile(firsts, 0.9).toFixed(1), raw: firsts },
   };
 }
-async function runCold(modelId: string, task: string, repeats: number) {
+async function runCold(modelId: string, task: string, repeats: number, device: string) {
   statusEl.textContent = "clearing caches (cold)...";
   await clearCaches();
   statusEl.textContent = "running (cold)...";
-  const metrics = await runMany(modelId, task, repeats);
+  const metrics = await runMany(modelId, task, repeats, device);
   return {
     platform: "browser",
     runtime: navigator.userAgent,
@@ -69,32 +70,40 @@ async function runCold(modelId: string, task: string, repeats: number) {
     repeats,
     model: modelId,
     task,
+    device,
     metrics,
     notes: "Only the 1st iteration is strictly cold in a single page session."
   };
 }
-async function runWarm(modelId: string, task: string, repeats: number) {
+async function runWarmDirect(modelId: string, task: string, repeats: number, device: string) {
+  statusEl.textContent = "prefetching (warmup) ...";
+  const p = await pipeline(task, modelId, { device });
+  await p("warmup");
+  statusEl.textContent = "running (warm)...";
+  const metrics = await runMany(modelId, task, repeats, device);
+  return {
+    platform: "browser",
+    runtime: navigator.userAgent,
+    mode: "warm",
+    repeats,
+    model: modelId,
+    task,
+    device,
+    metrics
+  };
+}
+async function runWarm(modelId: string, task: string, repeats: number, device: string) {
   const flag = sessionStorage.getItem("__warm_ready__");
   if (!flag) {
     statusEl.textContent = "prefetching (warmup) ...";
-    const p = await pipeline(task, modelId, {});
+    const p = await pipeline(task, modelId, { device });
     await p("warmup");
-    sessionStorage.setItem("__warm_ready__", JSON.stringify({ modelId, task, repeats }));
+    sessionStorage.setItem("__warm_ready__", JSON.stringify({ modelId, task, repeats, device }));
     location.reload();
     return null;
   } else {
     sessionStorage.removeItem("__warm_ready__");
-    statusEl.textContent = "running (warm)...";
-    const metrics = await runMany(modelId, task, repeats);
-    return {
-      platform: "browser",
-      runtime: navigator.userAgent,
-      mode: "warm",
-      repeats,
-      model: modelId,
-      task,
-      metrics
-    };
+    return await runWarmDirect(modelId, task, repeats, device);
   }
 }
 async function run() {
@@ -102,12 +111,14 @@ async function run() {
   const task = taskEl.value;
   const mode = modeEl.value as "warm" | "cold";
   const repeats = Math.max(1, parseInt(repeatsEl.value || "3", 10));
+  const device = deviceEl.value;
   out.textContent = "{}";
   if (mode === "cold") {
-    const r = await runCold(modelId, task, repeats);
+    const r = await runCold(modelId, task, repeats, device);
     if (r) { out.textContent = JSON.stringify(r, null, 2); statusEl.textContent = "done (cold)"; }
   } else {
-    const r = await runWarm(modelId, task, repeats);
+    const r = await runWarm(modelId, task, repeats, device);
+    console.log("warm run result:", r);
     if (r) { out.textContent = JSON.stringify(r, null, 2); statusEl.textContent = "done (warm)"; }
   }
 }
@@ -120,3 +131,13 @@ async function run() {
 btn.addEventListener("click", () => {
   run().catch((e) => { out.textContent = String(e); statusEl.textContent = "error"; console.error(e); });
 });
+
+// Expose for CLI use
+(window as any).runBenchmarkCLI = async function (params: { modelId: string, task: string, mode: string, repeats: number, device: string }) {
+  if (params.mode === "cold") {
+    return await runCold(params.modelId, params.task, params.repeats, params.device);
+  } else {
+    // For warm, use the direct function that skips reload logic
+    return await runWarmDirect(params.modelId, params.task, params.repeats, params.device);
+  }
+};