Warm/Cold tests, repeated tests

README.md

@@ -1,30 +1,19 @@
-# transformersjs-bench-min (Minimal Template)
+# transformersjs-bench-min (warm/cold + repeats + p50/p90)
 
-This zip contains two tiny templates to benchmark model *load* and *first inference* times with Transformers.js.
-
-- `bench-node/`: Node.js CLI (WASM backend). Prints a JSON result to stdout.
-- `bench-web/`: Vite + TypeScript browser page. Shows a JSON result on screen.
+Includes:
+- `bench-node/`: Node CLI with `--mode warm|cold`, `--repeats`, `--cache-dir`.
+- `bench-web/`: Browser app with warm (prefetch+reload) / cold (clear caches) and repeats.
 
 ## Quick start
-
-### Node CLI
+### Node
 ```bash
 cd bench-node
 npm i
-npm run bench
-# or model/task override:
-npm run bench -- Xenova/distilbert-base-uncased feature-extraction
+npm run bench -- Xenova/distilbert-base-uncased feature-extraction --mode warm --repeats 5 --cache-dir .bench-cache/warm
 ```
-
-### Browser app (Vite)
+### Web
 ```bash
 cd bench-web
 npm i
 npm run dev
-# open http://localhost:5173 and click "Run benchmark"
 ```
-
-## Notes
-- Models are fetched from the Hugging Face Hub/CDN the first time.
-- Browser backend selection (WebGPU/WASM) is handled internally by the library.
-- This matches your requested dependency versions.

bench-node/.gitignore

@@ -0,0 +1 @@
+.bench-cache

bench-node/README.md

@@ -1,4 +1,4 @@
-# bench-node (Transformers.js minimal benchmark)
+# bench-node (warm/cold, repeats, p50/p90)
 
 ## Setup
 ```bash
@@ -6,22 +6,11 @@ cd bench-node
 npm i
 ```
 
-## Run
+## Run examples
 ```bash
-# default: Xenova/distilbert-base-uncased + feature-extraction
-npm run bench
+# Warm: prefetch once (not measured) -> measure 5 times
+npm run bench -- Xenova/distilbert-base-uncased feature-extraction --mode warm --repeats 5 --cache-dir .bench-cache/warm
 
-# override model/task
-npm run bench -- Xenova/distilbert-base-uncased feature-extraction
-```
-
-Output example:
-```json
-{
-  "platform": "node",
-  "runtime": "node-22.x",
-  "model": "Xenova/distilbert-base-uncased",
-  "task": "feature-extraction",
-  "metrics": { "load_ms": 1234.5, "first_infer_ms": 98.7 }
-}
+# Cold: delete cache before each run, measure 3 times
+npm run bench -- Xenova/distilbert-base-uncased feature-extraction --mode cold --repeats 3 --cache-dir .bench-cache/cold
 ```

bench-node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "bench-node",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "bench-node",
-      "version": "0.0.1",
+      "version": "0.0.2",
       "dependencies": {
         "@huggingface/transformers": "^3.7.4"
       },
@@ -992,9 +992,9 @@
       "license": "BSD-3-Clause"
     },
     "node_modules/@types/node": {
-      "version": "24.6.1",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.6.1.tgz",
-      "integrity": "sha512-ljvjjs3DNXummeIaooB4cLBKg2U6SPI6Hjra/9rRIy7CpM0HpLtG9HptkMKAb4HYWy5S7HUvJEuWgr/y0U8SHw==",
+      "version": "24.6.2",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.6.2.tgz",
+      "integrity": "sha512-d2L25Y4j+W3ZlNAeMKcy7yDsK425ibcAOO2t7aPTz6gNMH0z2GThtwENCDc0d/Pw9wgyRqE5Px1wkV7naz8ang==",
       "license": "MIT",
       "dependencies": {
         "undici-types": "~7.13.0"

bench-node/package.json

@@ -2,7 +2,7 @@
   "name": "bench-node",
   "private": true,
   "type": "module",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "scripts": {
     "bench": "tsx src/index.ts",
     "build": "tsc -p tsconfig.json"

bench-node/src/index.ts

@@ -1,51 +1,96 @@
-import { pipeline } from "@huggingface/transformers";
+import { pipeline, env } from "@huggingface/transformers";
 import { performance } from "node:perf_hooks";
+import fs from "node:fs";
+import path from "node:path";
 
-// Minimal Node benchmark:
-// Measures model load time and first inference latency.
-// Default model/task can be overridden by CLI args.
-//
-// Usage:
-//   npm run bench -- [model-id] [task]
-// Example:
-//   npm run bench -- Xenova/distilbert-base-uncased feature-extraction
+// Node benchmark with warm/cold modes, repeats, p50/p90
 
 const modelId = process.argv[2] || "Xenova/distilbert-base-uncased";
 const task = process.argv[3] || "feature-extraction";
 
-async function main() {
-  console.log(`Model: ${modelId}`);
-  console.log(`Task : ${task}`);
+function getArg(name: string, def?: string) {
+  const i = process.argv.indexOf(`--${name}`);
+  if (i !== -1 && i + 1 < process.argv.length) return process.argv[i + 1];
+  return def;
+}
+
+const mode = (getArg("mode", "warm") as "warm" | "cold");
+const repeats = Math.max(1, parseInt(getArg("repeats", "3") || "3", 10));
+const cacheDir = getArg("cache-dir", path.resolve(".bench-cache/default"))!;
 
+// Point library cache to a dedicated directory for controllable cold/warm behavior
+env.cacheDir = cacheDir;
+
+function ensureEmptyDir(dir: string) {
+  if (fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true });
+  fs.mkdirSync(dir, { recursive: true });
+}
+
+function percentile(values: number[], q: number) {
+  const a = [...values].sort((x, y) => x - y);
+  const i = (a.length - 1) * q;
+  const i0 = Math.floor(i), i1 = Math.ceil(i);
+  return i0 === i1 ? a[i0] : a[i0] + (a[i1] - a[i0]) * (i - i0);
+}
+
+async function benchOnce() {
   const t0 = performance.now();
-  const pipe = await pipeline(task, modelId, {
-    // You can tweak backend settings here if needed.
-    // For Node, WASM backend is used by default.
-  });
+  const pipe = await pipeline(task, modelId, {});
   const t1 = performance.now();
 
-  const input = "The quick brown fox jumps over the lazy dog.";
-
   const t2 = performance.now();
-  await pipe(input);
+  await pipe("The quick brown fox jumps over the lazy dog.");
   const t3 = performance.now();
 
-  const loadMs = (t1 - t0).toFixed(1);
-  const firstInferMs = (t3 - t2).toFixed(1);
+  return { load_ms: +(t1 - t0).toFixed(1), first_infer_ms: +(t3 - t2).toFixed(1) };
+}
+
+async function main() {
+  console.log(`Model  : ${modelId}`);
+  console.log(`Task   : ${task}`);
+  console.log(`Mode   : ${mode}`);
+  console.log(`Repeats: ${repeats}`);
+  console.log(`Cache  : ${cacheDir}`);
+
+  const loads: number[] = [];
+  const firsts: number[] = [];
+
+  if (mode === "warm") {
+    // Fresh cache dir, prefetch once (not measured), then measure N times
+    ensureEmptyDir(cacheDir);
+    const warm = await pipeline(task, modelId, {});
+    await warm("warmup");
 
-  console.log(JSON.stringify({
+    for (let i = 0; i < repeats; i++) {
+      const r = await benchOnce();
+      loads.push(r.load_ms);
+      firsts.push(r.first_infer_ms);
+    }
+  } else {
+    // cold: delete cache dir before each measured run
+    for (let i = 0; i < repeats; i++) {
+      ensureEmptyDir(cacheDir);
+      const r = await benchOnce();
+      loads.push(r.load_ms);
+      firsts.push(r.first_infer_ms);
+    }
+  }
+
+  const result = {
     platform: "node",
     runtime: `node-${process.versions.node}`,
     model: modelId,
     task,
+    mode,
+    repeats,
+    cacheDir,
     metrics: {
-      load_ms: Number(loadMs),
-      first_infer_ms: Number(firstInferMs),
+      load_ms: { p50: +percentile(loads, 0.5).toFixed(1), p90: +percentile(loads, 0.9).toFixed(1), raw: loads },
+      first_infer_ms: { p50: +percentile(firsts, 0.5).toFixed(1), p90: +percentile(firsts, 0.9).toFixed(1), raw: firsts }
     }
-  }, null, 2));
+  };
+
+  console.log(JSON.stringify(result, null, 2));
 }
 
-main().catch((e) => {
-  console.error(e);
-  process.exit(1);
-});
+main().catch((e) => { console.error(e); process.exit(1); });

bench-web/README.md

@@ -1,4 +1,4 @@
-# bench-web (Transformers.js minimal browser benchmark)
+# bench-web (warm/cold, repeats, p50/p90)
 
 ## Setup
 ```bash
@@ -12,6 +12,7 @@ npm run dev
 # open http://localhost:5173
 ```
 
-- Pick a model/task (default `Xenova/distilbert-base-uncased` + `feature-extraction`), click "Run benchmark".
-- The page prints a small JSON with load time and first inference latency.
-- Works with WASM by default. If your browser supports WebGPU, the library may use it automatically.
+## How it works
+- **warm**: prefetch once (non-measured) → auto-reload → measure `repeats` times with disk caches populated.
+- **cold**: clear Cache Storage & IndexedDB, then measure in the same tab
+  - Note: only the 1st iteration is strictly cold within a single page session.

bench-web/index.html

@@ -8,7 +8,7 @@
       body { font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif; margin: 2rem; }
       button { padding: 0.6rem 1rem; font-size: 1rem; }
       pre { background: #f6f8fa; padding: 1rem; border-radius: 8px; overflow: auto; }
-      .row { display: flex; gap: 0.5rem; align-items: center; margin-bottom: 0.5rem; }
+      .row { display: flex; gap: 0.5rem; align-items: center; margin-bottom: 0.5rem; flex-wrap: wrap; }
       label { font-weight: 600; }
       input, select { padding: 0.4rem 0.6rem; }
     </style>
@@ -18,14 +18,21 @@
     <div class="row">
       <label for="model">Model</label>
       <input id="model" value="Xenova/distilbert-base-uncased" size="34" />
-    </div>
-    <div class="row">
       <label for="task">Task</label>
       <select id="task">
         <option value="feature-extraction" selected>feature-extraction</option>
         <option value="text-classification">text-classification</option>
       </select>
     </div>
+    <div class="row">
+      <label for="mode">Mode</label>
+      <select id="mode">
+        <option value="warm" selected>warm (prefetch + reload)</option>
+        <option value="cold">cold (clear caches once)</option>
+      </select>
+      <label for="repeats">Repeats</label>
+      <input id="repeats" type="number" value="3" min="1" style="width: 5rem;" />
+    </div>
     <div class="row">
       <button id="run">Run benchmark</button>
       <span id="status"></span>

bench-web/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "bench-web",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "bench-web",
-      "version": "0.0.1",
+      "version": "0.0.2",
       "dependencies": {
         "@huggingface/transformers": "^3.7.4"
       },
@@ -1307,9 +1307,9 @@
       "license": "MIT"
     },
     "node_modules/@types/node": {
-      "version": "24.6.1",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.6.1.tgz",
-      "integrity": "sha512-ljvjjs3DNXummeIaooB4cLBKg2U6SPI6Hjra/9rRIy7CpM0HpLtG9HptkMKAb4HYWy5S7HUvJEuWgr/y0U8SHw==",
+      "version": "24.6.2",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.6.2.tgz",
+      "integrity": "sha512-d2L25Y4j+W3ZlNAeMKcy7yDsK425ibcAOO2t7aPTz6gNMH0z2GThtwENCDc0d/Pw9wgyRqE5Px1wkV7naz8ang==",
       "license": "MIT",
       "dependencies": {
         "undici-types": "~7.13.0"

bench-web/package.json

@@ -1,7 +1,7 @@
 {
   "name": "bench-web",
   "private": true,
-  "version": "0.0.1",
+  "version": "0.0.2",
   "type": "module",
   "scripts": {
     "dev": "vite",

bench-web/src/main.ts

@@ -5,49 +5,118 @@ const out = document.getElementById("out") as HTMLPreElement;
 const statusEl = document.getElementById("status") as HTMLSpanElement;
 const modelEl = document.getElementById("model") as HTMLInputElement;
 const taskEl = document.getElementById("task") as HTMLSelectElement;
+const modeEl = document.getElementById("mode") as HTMLSelectElement;
+const repeatsEl = document.getElementById("repeats") as HTMLInputElement;
 
-function now() {
-  return performance.now();
+function now() { return performance.now(); }
+function percentile(values: number[], q: number) {
+  const a = [...values].sort((x, y) => x - y);
+  const i = (a.length - 1) * q;
+  const i0 = Math.floor(i), i1 = Math.ceil(i);
+  return i0 === i1 ? a[i0] : a[i0] + (a[i1] - a[i0]) * (i - i0);
 }
-
-async function run() {
-  const modelId = modelEl.value.trim() || "Xenova/distilbert-base-uncased";
-  const task = taskEl.value;
-  statusEl.textContent = "loading...";
-
+async function clearCaches({ clearSession=false }: { clearSession?: boolean } = {}) {
+  try {
+    const keys = await caches.keys();
+    await Promise.all(keys.map((k) => caches.delete(k)));
+  } catch {}
+  try {
+    const anyIDB: any = indexedDB as any;
+    if (typeof anyIDB.databases === "function") {
+      const dbs = await anyIDB.databases();
+      await Promise.all(dbs.map((d: any) => d?.name ? indexedDB.deleteDatabase(d.name) : undefined));
+    } else {
+      indexedDB.deleteDatabase("transformers-cache");
+      indexedDB.deleteDatabase("model-cache");
+    }
+  } catch {}
+  try {
+    localStorage.clear();
+    if (clearSession) sessionStorage.clear();
+  } catch {}
+}
+async function benchOnce(modelId: string, task: string) {
   const t0 = now();
-  const pipe = await pipeline(task, modelId, {
-    // For browser, the library will pick the best available backend (WebGPU/WASM).
-  });
+  const pipe = await pipeline(task, modelId, {});
   const t1 = now();
-
-  const input = "The quick brown fox jumps over the lazy dog.";
-  statusEl.textContent = "running inference...";
-
   const t2 = now();
-  await pipe(input);
+  await pipe("The quick brown fox jumps over the lazy dog.");
   const t3 = now();
-
-  const result = {
+  return { load_ms: +(t1 - t0).toFixed(1), first_infer_ms: +(t3 - t2).toFixed(1) };
+}
+async function runMany(modelId: string, task: string, repeats: number) {
+  const loads: number[] = [];
+  const firsts: number[] = [];
+  for (let i = 0; i < repeats; i++) {
+    const r = await benchOnce(modelId, task);
+    loads.push(r.load_ms);
+    firsts.push(r.first_infer_ms);
+  }
+  return {
+    load_ms: { p50: +percentile(loads, 0.5).toFixed(1), p90: +percentile(loads, 0.9).toFixed(1), raw: loads },
+    first_infer_ms: { p50: +percentile(firsts, 0.5).toFixed(1), p90: +percentile(firsts, 0.9).toFixed(1), raw: firsts },
+  };
+}
+async function runCold(modelId: string, task: string, repeats: number) {
+  statusEl.textContent = "clearing caches (cold)...";
+  await clearCaches();
+  statusEl.textContent = "running (cold)...";
+  const metrics = await runMany(modelId, task, repeats);
+  return {
     platform: "browser",
     runtime: navigator.userAgent,
-    backend_hint: ("gpu" in navigator) ? "webgpu-or-wasm" : "wasm",
+    mode: "cold",
+    repeats,
     model: modelId,
     task,
-    metrics: {
-      load_ms: +(t1 - t0).toFixed(1),
-      first_infer_ms: +(t3 - t2).toFixed(1),
-    }
+    metrics,
+    notes: "Only the 1st iteration is strictly cold in a single page session."
   };
-
-  out.textContent = JSON.stringify(result, null, 2);
-  statusEl.textContent = "done";
 }
-
+async function runWarm(modelId: string, task: string, repeats: number) {
+  const flag = sessionStorage.getItem("__warm_ready__");
+  if (!flag) {
+    statusEl.textContent = "prefetching (warmup) ...";
+    const p = await pipeline(task, modelId, {});
+    await p("warmup");
+    sessionStorage.setItem("__warm_ready__", JSON.stringify({ modelId, task, repeats }));
+    location.reload();
+    return null;
+  } else {
+    sessionStorage.removeItem("__warm_ready__");
+    statusEl.textContent = "running (warm)...";
+    const metrics = await runMany(modelId, task, repeats);
+    return {
+      platform: "browser",
+      runtime: navigator.userAgent,
+      mode: "warm",
+      repeats,
+      model: modelId,
+      task,
+      metrics
+    };
+  }
+}
+async function run() {
+  const modelId = modelEl.value.trim() || "Xenova/distilbert-base-uncased";
+  const task = taskEl.value;
+  const mode = modeEl.value as "warm" | "cold";
+  const repeats = Math.max(1, parseInt(repeatsEl.value || "3", 10));
+  out.textContent = "{}";
+  if (mode === "cold") {
+    const r = await runCold(modelId, task, repeats);
+    if (r) { out.textContent = JSON.stringify(r, null, 2); statusEl.textContent = "done (cold)"; }
+  } else {
+    const r = await runWarm(modelId, task, repeats);
+    if (r) { out.textContent = JSON.stringify(r, null, 2); statusEl.textContent = "done (warm)"; }
+  }
+}
+(async () => {
+  const flag = sessionStorage.getItem("__warm_ready__");
+  if (flag) {
+    try { await run(); } catch (e) { console.error(e); }
+  }
+})();
 btn.addEventListener("click", () => {
-  run().catch((e) => {
-    out.textContent = String(e);
-    statusEl.textContent = "error";
-    console.error(e);
-  });
+  run().catch((e) => { out.textContent = String(e); statusEl.textContent = "error"; console.error(e); });
 });

bench-web/vite.config.ts

@@ -1,7 +1,2 @@
 import { defineConfig } from "vite";
-
-export default defineConfig({
-  server: {
-    port: 5173
-  }
-});
+export default defineConfig({ server: { port: 5173 } });