Per-task benchmarks

bench/src/core/task-inputs.ts

@@ -0,0 +1,140 @@
+/**
+ * Task-specific input generation for benchmarking
+ * Based on Transformers.js usage examples
+ */
+
+export interface TaskInput {
+  inputs: any;
+  options?: any;
+}
+
+/**
+ * Generate appropriate input for a given task type
+ */
+export function getTaskInput(task: string, batchSize: number = 1): TaskInput {
+  // Normalize task name
+  const normalizedTask = task.toLowerCase().trim();
+
+  // Text tasks
+  if (normalizedTask === "fill-mask") {
+    const inputs = Array(batchSize).fill("The goal of life is [MASK].");
+    return { inputs };
+  }
+
+  if (normalizedTask === "question-answering") {
+    const question = "Who was Jim Henson?";
+    const context = "Jim Henson was a nice puppet.";
+    // For batch, repeat the same question-context pair
+    const inputs = Array(batchSize).fill({ question, context });
+    return { inputs: batchSize === 1 ? { question, context } : inputs };
+  }
+
+  if (normalizedTask === "summarization") {
+    const text =
+      "The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, " +
+      "and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. " +
+      "During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest " +
+      "man-made structure in the world, a title it held for 41 years until the Chrysler Building in New " +
+      "York City was finished in 1930.";
+    const inputs = Array(batchSize).fill(text);
+    return { inputs, options: { max_new_tokens: 100 } };
+  }
+
+  if (normalizedTask === "sentiment-analysis" || normalizedTask === "text-classification") {
+    const inputs = Array(batchSize).fill("I love transformers!");
+    return { inputs };
+  }
+
+  if (normalizedTask === "text-generation") {
+    const inputs = Array(batchSize).fill("Once upon a time, there was");
+    return { inputs, options: { max_new_tokens: 10 } };
+  }
+
+  if (normalizedTask === "text2text-generation") {
+    const inputs = Array(batchSize).fill("how can I become more healthy?");
+    return { inputs, options: { max_new_tokens: 100 } };
+  }
+
+  if (normalizedTask === "token-classification" || normalizedTask === "ner") {
+    const inputs = Array(batchSize).fill("My name is Sarah and I live in London");
+    return { inputs };
+  }
+
+  if (normalizedTask === "translation") {
+    const inputs = Array(batchSize).fill("Life is like a box of chocolate.");
+    return { inputs, options: { src_lang: "eng_Latn", tgt_lang: "fra_Latn" } };
+  }
+
+  if (normalizedTask === "zero-shot-classification") {
+    const text = "I love transformers!";
+    const labels = ["positive", "negative"];
+    // For batching, would need to handle differently - for now use single input
+    return { inputs: batchSize === 1 ? [text, labels] : Array(batchSize).fill([text, labels]) };
+  }
+
+  if (normalizedTask === "feature-extraction") {
+    const inputs = Array(batchSize).fill("This is a simple test.");
+    return { inputs };
+  }
+
+  // Vision tasks - use public image URLs
+  if (normalizedTask === "background-removal") {
+    const url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/portrait-of-woman_small.jpg";
+    const inputs = Array(batchSize).fill(url);
+    return { inputs };
+  }
+
+  if (normalizedTask === "depth-estimation") {
+    const url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg";
+    const inputs = Array(batchSize).fill(url);
+    return { inputs };
+  }
+
+  if (normalizedTask === "image-classification") {
+    const url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/tiger.jpg";
+    const inputs = Array(batchSize).fill(url);
+    return { inputs };
+  }
+
+  if (normalizedTask === "image-segmentation") {
+    const url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg";
+    const inputs = Array(batchSize).fill(url);
+    return { inputs };
+  }
+
+  if (normalizedTask === "image-to-image") {
+    const url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/tiger.jpg";
+    const inputs = Array(batchSize).fill(url);
+    return { inputs };
+  }
+
+  if (normalizedTask === "object-detection") {
+    const url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg";
+    const inputs = Array(batchSize).fill(url);
+    return { inputs, options: { threshold: 0.9 } };
+  }
+
+  if (normalizedTask === "image-feature-extraction") {
+    const url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/cats.png";
+    const inputs = Array(batchSize).fill(url);
+    return { inputs };
+  }
+
+  // Audio tasks
+  if (normalizedTask === "audio-classification") {
+    const url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav";
+    const inputs = Array(batchSize).fill(url);
+    return { inputs };
+  }
+
+  if (normalizedTask === "automatic-speech-recognition") {
+    const url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav";
+    const inputs = Array(batchSize).fill(url);
+    return { inputs };
+  }
+
+  // Default fallback for unknown tasks - use text
+  console.warn(`Unknown task type: ${task}, using default text input`);
+  const inputs = Array(batchSize).fill("The quick brown fox jumps over the lazy dog.");
+  return { inputs };
+}

bench/src/node/benchmark.ts

@@ -5,6 +5,7 @@ import { BenchmarkOptions, BenchmarkResult } from "../core/types.js";
 import { BenchmarkRawResult, aggregateMetrics } from "../core/metrics.js";
 import { ensureEmptyDir } from "./cache.js";
 import { getSystemInfo } from "../core/sysinfo.js";
+import { getTaskInput } from "../core/task-inputs.js";
 
 async function benchOnce(
   modelId: string,
@@ -18,18 +19,18 @@ async function benchOnce(
   const pipe = await pipeline(task, modelId, options);
   const t1 = performance.now();
 
-  // Prepare batch input
-  const inputs = Array(batchSize).fill("The quick brown fox jumps over the lazy dog.");
+  // Get task-appropriate input
+  const { inputs, options: taskOptions } = getTaskInput(task, batchSize);
 
   const t2 = performance.now();
-  await pipe(inputs);
+  await pipe(inputs, taskOptions);
   const t3 = performance.now();
 
   // Run additional inferences to measure subsequent performance
   const subsequentTimes: number[] = [];
   for (let i = 0; i < 3; i++) {
     const t4 = performance.now();
-    await pipe(inputs);
+    await pipe(inputs, taskOptions);
     const t5 = performance.now();
     subsequentTimes.push(+(t5 - t4).toFixed(1));
   }
@@ -63,8 +64,8 @@ export async function runNodeBenchmark(options: BenchmarkOptions): Promise<Bench
     const warmOptions: any = {};
     if (dtype) warmOptions.dtype = dtype;
     const warm = await pipeline(task, modelId, warmOptions);
-    const warmupInputs = Array(batchSize).fill("warmup");
-    await warm(warmupInputs);
+    const { inputs: warmupInputs, options: taskOptions } = getTaskInput(task, batchSize);
+    await warm(warmupInputs, taskOptions);
 
     for (let i = 0; i < repeats; i++) {
       const r = await benchOnce(modelId, task, dtype, batchSize);

bench/src/web/benchmark.ts

@@ -3,6 +3,7 @@ import { BenchmarkRawResult, aggregateMetrics } from "../core/metrics.js";
 import { BenchmarkResult } from "../core/types.js";
 import { clearCaches } from "./cache.js";
 import { getBrowserEnvInfo } from "./envinfo.js";
+import { getTaskInput } from "../core/task-inputs.js";
 
 function now() {
   return performance.now();
@@ -22,18 +23,18 @@ async function benchOnce(
     const pipe = await pipeline(task, modelId, options);
     const t1 = now();
 
-    // Prepare batch input
-    const inputs = Array(batchSize).fill("The quick brown fox jumps over the lazy dog.");
+    // Get task-appropriate input
+    const { inputs, options: taskOptions } = getTaskInput(task, batchSize);
 
     const t2 = now();
-    await pipe(inputs);
+    await pipe(inputs, taskOptions);
     const t3 = now();
 
     // Run additional inferences to measure subsequent performance
     const subsequentTimes: number[] = [];
     for (let i = 0; i < 3; i++) {
       const t4 = now();
-      await pipe(inputs);
+      await pipe(inputs, taskOptions);
       const t5 = now();
       subsequentTimes.push(+(t5 - t4).toFixed(1));
     }
@@ -128,8 +129,8 @@ export async function runWebBenchmarkWarm(
     const options: any = { device };
     if (dtype) options.dtype = dtype;
     const p = await pipeline(task, modelId, options);
-    const warmupInputs = Array(batchSize).fill("warmup");
-    await p(warmupInputs);
+    const { inputs: warmupInputs, options: taskOptions } = getTaskInput(task, batchSize);
+    await p(warmupInputs, taskOptions);
   } catch (err: any) {
     const errorMessage = err?.message || String(err);
     let errorType = "runtime_error";