Add environmental metrics

bench/src/core/sysinfo.ts

@@ -0,0 +1,37 @@
+import os from "os";
+
+export interface SystemInfo {
+  cpu: {
+    model: string;
+    cores: number;
+    threads: number;
+  };
+  memory: {
+    total: string;
+    available: string;
+  };
+  platform: string;
+  arch: string;
+  nodeVersion: string;
+}
+
+export function getSystemInfo(): SystemInfo {
+  const cpus = os.cpus();
+  const totalMemory = os.totalmem();
+  const freeMemory = os.freemem();
+
+  return {
+    cpu: {
+      model: cpus[0]?.model || "Unknown",
+      cores: os.cpus().length,
+      threads: os.cpus().length, // In Node.js, this is the same as logical cores
+    },
+    memory: {
+      total: `${(totalMemory / 1024 / 1024 / 1024).toFixed(2)} GB`,
+      available: `${(freeMemory / 1024 / 1024 / 1024).toFixed(2)} GB`,
+    },
+    platform: os.platform(),
+    arch: os.arch(),
+    nodeVersion: process.version,
+  };
+}

bench/src/core/types.ts

@@ -16,10 +16,15 @@ export interface BenchmarkResult {
   repeats: number;
   batchSize: number;
   dtype?: string;
-  metrics: {
+  metrics?: {
     load_ms: { p50: number; p90: number; raw: number[] };
     first_infer_ms: { p50: number; p90: number; raw: number[] };
     subsequent_infer_ms: { p50: number; p90: number; raw: number[] };
   };
+  error?: {
+    type: string;
+    message: string;
+    stage?: "load" | "inference";
+  };
   [key: string]: any;
 }

bench/src/node/benchmark.ts

@@ -4,6 +4,7 @@ import path from "node:path";
 import { BenchmarkOptions, BenchmarkResult } from "../core/types.js";
 import { BenchmarkRawResult, aggregateMetrics } from "../core/metrics.js";
 import { ensureEmptyDir } from "./cache.js";
+import { getSystemInfo } from "../core/sysinfo.js";
 
 async function benchOnce(
   modelId: string,
@@ -79,6 +80,7 @@ export async function runNodeBenchmark(options: BenchmarkOptions): Promise<Bench
   }
 
   const metrics = aggregateMetrics(results);
+  const sysInfo = getSystemInfo();
 
   const result: BenchmarkResult = {
     platform: "node",
@@ -90,6 +92,7 @@ export async function runNodeBenchmark(options: BenchmarkOptions): Promise<Bench
     batchSize,
     cacheDir,
     metrics,
+    environment: sysInfo,
   };
 
   if (dtype) result.dtype = dtype;

bench/src/web/benchmark.ts

@@ -2,6 +2,7 @@ import { pipeline } from "@huggingface/transformers";
 import { BenchmarkRawResult, aggregateMetrics } from "../core/metrics.js";
 import { BenchmarkResult } from "../core/types.js";
 import { clearCaches } from "./cache.js";
+import { getBrowserEnvInfo } from "./envinfo.js";
 
 function now() {
   return performance.now();
@@ -13,34 +14,55 @@ async function benchOnce(
   device: string,
   dtype: string | undefined,
   batchSize: number
-): Promise<BenchmarkRawResult> {
-  const t0 = now();
-  const options: any = { device };
-  if (dtype) options.dtype = dtype;
-  const pipe = await pipeline(task, modelId, options);
-  const t1 = now();
-
-  // Prepare batch input
-  const inputs = Array(batchSize).fill("The quick brown fox jumps over the lazy dog.");
-
-  const t2 = now();
-  await pipe(inputs);
-  const t3 = now();
-
-  // Run additional inferences to measure subsequent performance
-  const subsequentTimes: number[] = [];
-  for (let i = 0; i < 3; i++) {
-    const t4 = now();
+): Promise<BenchmarkRawResult | { error: { type: string; message: string; stage: "load" | "inference" } }> {
+  try {
+    const t0 = now();
+    const options: any = { device };
+    if (dtype) options.dtype = dtype;
+    const pipe = await pipeline(task, modelId, options);
+    const t1 = now();
+
+    // Prepare batch input
+    const inputs = Array(batchSize).fill("The quick brown fox jumps over the lazy dog.");
+
+    const t2 = now();
     await pipe(inputs);
-    const t5 = now();
-    subsequentTimes.push(+(t5 - t4).toFixed(1));
-  }
+    const t3 = now();
 
-  return {
-    load_ms: +(t1 - t0).toFixed(1),
-    first_infer_ms: +(t3 - t2).toFixed(1),
-    subsequent_infer_ms: subsequentTimes,
-  };
+    // Run additional inferences to measure subsequent performance
+    const subsequentTimes: number[] = [];
+    for (let i = 0; i < 3; i++) {
+      const t4 = now();
+      await pipe(inputs);
+      const t5 = now();
+      subsequentTimes.push(+(t5 - t4).toFixed(1));
+    }
+
+    return {
+      load_ms: +(t1 - t0).toFixed(1),
+      first_infer_ms: +(t3 - t2).toFixed(1),
+      subsequent_infer_ms: subsequentTimes,
+    };
+  } catch (error: any) {
+    // Determine error type and stage
+    const errorMessage = error?.message || String(error);
+    let errorType = "runtime_error";
+    let stage: "load" | "inference" = "load";
+
+    if (errorMessage.includes("Aborted") || errorMessage.includes("out of memory")) {
+      errorType = "memory_error";
+    } else if (errorMessage.includes("Failed to fetch") || errorMessage.includes("network")) {
+      errorType = "network_error";
+    }
+
+    return {
+      error: {
+        type: errorType,
+        message: errorMessage,
+        stage,
+      },
+    };
+  }
 }
 
 export async function runWebBenchmarkCold(
@@ -54,12 +76,18 @@ export async function runWebBenchmarkCold(
   await clearCaches();
 
   const results: BenchmarkRawResult[] = [];
+  let error: { type: string; message: string; stage: "load" | "inference" } | undefined;
+
   for (let i = 0; i < repeats; i++) {
     const r = await benchOnce(modelId, task, device, dtype, batchSize);
+    if ('error' in r) {
+      error = r.error;
+      break;
+    }
     results.push(r);
   }
 
-  const metrics = aggregateMetrics(results);
+  const envInfo = await getBrowserEnvInfo();
 
   const result: BenchmarkResult = {
     platform: "browser",
@@ -70,9 +98,17 @@ export async function runWebBenchmarkCold(
     model: modelId,
     task,
     device,
-    metrics,
+    environment: envInfo,
     notes: "Only the 1st iteration is strictly cold in a single page session.",
   };
+
+  if (error) {
+    result.error = error;
+  } else {
+    const metrics = aggregateMetrics(results);
+    result.metrics = metrics;
+  }
+
   if (dtype) result.dtype = dtype;
   return result;
 }
@@ -85,20 +121,44 @@ export async function runWebBenchmarkWarm(
   dtype?: string,
   batchSize: number = 1
 ): Promise<BenchmarkResult> {
+  let error: { type: string; message: string; stage: "load" | "inference" } | undefined;
+
   // Prefetch/warmup
-  const options: any = { device };
-  if (dtype) options.dtype = dtype;
-  const p = await pipeline(task, modelId, options);
-  const warmupInputs = Array(batchSize).fill("warmup");
-  await p(warmupInputs);
+  try {
+    const options: any = { device };
+    if (dtype) options.dtype = dtype;
+    const p = await pipeline(task, modelId, options);
+    const warmupInputs = Array(batchSize).fill("warmup");
+    await p(warmupInputs);
+  } catch (err: any) {
+    const errorMessage = err?.message || String(err);
+    let errorType = "runtime_error";
+    if (errorMessage.includes("Aborted") || errorMessage.includes("out of memory")) {
+      errorType = "memory_error";
+    } else if (errorMessage.includes("Failed to fetch") || errorMessage.includes("network")) {
+      errorType = "network_error";
+    }
+    error = {
+      type: errorType,
+      message: errorMessage,
+      stage: "load",
+    };
+  }
 
   const results: BenchmarkRawResult[] = [];
-  for (let i = 0; i < repeats; i++) {
-    const r = await benchOnce(modelId, task, device, dtype, batchSize);
-    results.push(r);
+
+  if (!error) {
+    for (let i = 0; i < repeats; i++) {
+      const r = await benchOnce(modelId, task, device, dtype, batchSize);
+      if ('error' in r) {
+        error = r.error;
+        break;
+      }
+      results.push(r);
+    }
   }
 
-  const metrics = aggregateMetrics(results);
+  const envInfo = await getBrowserEnvInfo();
 
   const result: BenchmarkResult = {
     platform: "browser",
@@ -109,8 +169,16 @@ export async function runWebBenchmarkWarm(
     model: modelId,
     task,
     device,
-    metrics,
+    environment: envInfo,
   };
+
+  if (error) {
+    result.error = error;
+  } else {
+    const metrics = aggregateMetrics(results);
+    result.metrics = metrics;
+  }
+
   if (dtype) result.dtype = dtype;
   return result;
 }

bench/src/web/cli.ts

@@ -48,7 +48,12 @@ async function main() {
 
   // Build args based on mode
   const args = device === "wasm"
-    ? ["--disable-gpu", "--disable-software-rasterizer"]
+    ? [
+        "--disable-gpu",
+        "--disable-software-rasterizer",
+        // Increase WASM memory limits for large models
+        "--js-flags=--max-old-space-size=8192",
+      ]
     : [
         // Official WebGPU flags from Chrome team
         // https://developer.chrome.com/blog/supercharge-web-ai-testing#enable-webgpu
@@ -96,6 +101,11 @@ async function main() {
       }
     });
 
+    // Catch page errors
+    page.on("pageerror", (error) => {
+      console.error(`[browser error]`, error.message);
+    });
+
     // Navigate to the app
     await page.goto(url);
 
@@ -152,6 +162,21 @@ async function main() {
 
     console.log("\n" + JSON.stringify(result, null, 2));
 
+    // Log helpful messages if there's an error
+    if (result.error) {
+      console.error("\n❌ Benchmark completed with error:");
+      console.error(`   Type: ${result.error.type}`);
+      console.error(`   Stage: ${result.error.stage}`);
+      console.error(`   Message: ${result.error.message}`);
+
+      if (result.error.type === "memory_error" && device === "wasm") {
+        console.error("\nSuggestions:");
+        console.error("  1. Try using --device=webgpu instead of --device=wasm");
+        console.error("  2. Use a smaller model variant");
+        console.error("  3. Reduce the batch size with --batch-size=1");
+      }
+    }
+
   } finally {
     await browser.close();
     await server.close();

bench/src/web/envinfo.ts

@@ -0,0 +1,68 @@
+export interface BrowserEnvInfo {
+  userAgent: string;
+  platform: string;
+  cpuCores: number;
+  memory?: {
+    deviceMemory?: number; // GB
+  };
+  gpu?: {
+    vendor?: string;
+    renderer?: string;
+    webgpuAdapter?: string;
+  };
+}
+
+export async function getBrowserEnvInfo(): Promise<BrowserEnvInfo> {
+  const info: BrowserEnvInfo = {
+    userAgent: navigator.userAgent,
+    platform: navigator.platform,
+    cpuCores: navigator.hardwareConcurrency || 0,
+  };
+
+  // Memory info (Chrome only)
+  if ('deviceMemory' in navigator) {
+    info.memory = {
+      deviceMemory: (navigator as any).deviceMemory,
+    };
+  }
+
+  // GPU info
+  const gpu: BrowserEnvInfo['gpu'] = {};
+
+  // Try to get WebGL renderer info
+  try {
+    const canvas = document.createElement('canvas');
+    const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
+    if (gl) {
+      const debugInfo = (gl as any).getExtension('WEBGL_debug_renderer_info');
+      if (debugInfo) {
+        gpu.vendor = gl.getParameter(debugInfo.UNMASKED_VENDOR_WEBGL);
+        gpu.renderer = gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL);
+      }
+    }
+  } catch (e) {
+    // WebGL not available or blocked
+  }
+
+  // Try to get WebGPU adapter info
+  if ('gpu' in navigator) {
+    try {
+      const adapter = await (navigator as any).gpu.requestAdapter();
+      if (adapter?.info) {
+        gpu.webgpuAdapter = adapter.info.description ||
+                            adapter.info.vendor ||
+                            'WebGPU Available';
+      } else if (adapter) {
+        gpu.webgpuAdapter = 'WebGPU Available';
+      }
+    } catch (e) {
+      // WebGPU not available
+    }
+  }
+
+  if (Object.keys(gpu).length > 0) {
+    info.gpu = gpu;
+  }
+
+  return info;
+}