whitphx HF Staff commited on
Commit
030f336
·
1 Parent(s): 7cb11f5

Add env info into the ID

Browse files
bench/src/core/benchmark-id.ts CHANGED
@@ -8,6 +8,27 @@
8
  * 4. Are sortable and searchable
9
  */
10
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  export interface BenchmarkSettings {
12
  platform: "node" | "web";
13
  modelId: string;
@@ -18,6 +39,7 @@ export interface BenchmarkSettings {
18
  batchSize?: number;
19
  browser?: string;
20
  headed?: boolean;
 
21
  }
22
 
23
  /**
@@ -48,6 +70,17 @@ export function generateBenchmarkId(settings: BenchmarkSettings): string {
48
  return `${task}/${modelId}/${filenameParts.join("_")}`;
49
  }
50
 
 
 
 
 
 
 
 
 
 
 
 
51
  /**
52
  * Generate the filename parts (everything except task and model ID)
53
  */
@@ -88,6 +121,52 @@ function generateFilenameParts(settings: BenchmarkSettings): string[] {
88
  parts.push("headed");
89
  }
90
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
  return parts;
92
  }
93
 
 
8
  * 4. Are sortable and searchable
9
  */
10
 
11
+ export interface EnvironmentInfo {
12
+ // Node.js format
13
+ cpu?: {
14
+ model?: string;
15
+ cores?: number;
16
+ };
17
+ memory?: {
18
+ total?: string;
19
+ deviceMemory?: number; // Web browser format (GB)
20
+ };
21
+ gpu?: {
22
+ vendor?: string;
23
+ renderer?: string;
24
+ };
25
+ platform?: string;
26
+ arch?: string;
27
+
28
+ // Web browser format (direct fields)
29
+ cpuCores?: number;
30
+ }
31
+
32
  export interface BenchmarkSettings {
33
  platform: "node" | "web";
34
  modelId: string;
 
39
  batchSize?: number;
40
  browser?: string;
41
  headed?: boolean;
42
+ environment?: EnvironmentInfo;
43
  }
44
 
45
  /**
 
70
  return `${task}/${modelId}/${filenameParts.join("_")}`;
71
  }
72
 
73
+ /**
74
+ * Sanitize environment strings for use in filenames
75
+ */
76
+ function sanitizeEnvString(str: string): string {
77
+ return str
78
+ .toLowerCase()
79
+ .replace(/\s+/g, "-")
80
+ .replace(/[^a-z0-9\-]/g, "")
81
+ .substring(0, 20); // Limit length
82
+ }
83
+
84
  /**
85
  * Generate the filename parts (everything except task and model ID)
86
  */
 
121
  parts.push("headed");
122
  }
123
 
124
+ // 8. Environment info (CPU, memory, architecture, GPU)
125
+ if (settings.environment) {
126
+ const env = settings.environment;
127
+
128
+ // CPU model (sanitized, first significant part) - Node.js only
129
+ if (env.cpu?.model) {
130
+ const cpuName = sanitizeEnvString(env.cpu.model.split(/[\s(]/)[0]);
131
+ if (cpuName) {
132
+ parts.push(cpuName);
133
+ }
134
+ }
135
+
136
+ // CPU cores (support both Node.js and web browser formats)
137
+ const cores = env.cpu?.cores || env.cpuCores;
138
+ if (cores) {
139
+ parts.push(`${cores}c`);
140
+ }
141
+
142
+ // Memory (support both Node.js and web browser formats)
143
+ if (env.memory?.total) {
144
+ // Node.js format: Parse memory string like "32.00 GB" -> "32gb"
145
+ const memMatch = env.memory.total.match(/^(\d+)/);
146
+ if (memMatch) {
147
+ parts.push(`${memMatch[1]}gb`);
148
+ }
149
+ } else if (env.memory?.deviceMemory) {
150
+ // Web browser format: deviceMemory is already in GB
151
+ parts.push(`${env.memory.deviceMemory}gb`);
152
+ }
153
+
154
+ // Architecture (Node.js only, browser uses platform like "MacIntel")
155
+ if (env.arch) {
156
+ parts.push(env.arch);
157
+ }
158
+
159
+ // GPU vendor/renderer for web (if using webgpu)
160
+ if (settings.platform === "web" && settings.device === "webgpu" && env.gpu) {
161
+ if (env.gpu.vendor) {
162
+ const gpuVendor = sanitizeEnvString(env.gpu.vendor.split(/[\s(]/)[0]);
163
+ if (gpuVendor && gpuVendor !== "google") { // Skip "Google Inc."
164
+ parts.push(`gpu-${gpuVendor}`);
165
+ }
166
+ }
167
+ }
168
+ }
169
+
170
  return parts;
171
  }
172
 
bench/src/server/hf-dataset.ts CHANGED
@@ -46,6 +46,14 @@ export class HFDatasetUploader {
46
  batchSize: benchmark.batchSize,
47
  browser: benchmark.browser,
48
  headed: benchmark.headed,
 
 
 
 
 
 
 
 
49
  };
50
 
51
  const { fullPath } = generateBenchmarkPath(settings);
@@ -53,6 +61,55 @@ export class HFDatasetUploader {
53
  return fullPath.replace(/\.jsonl$/, ".json");
54
  }
55
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
  /**
57
  * Upload a benchmark result to HF Dataset
58
  * Overwrites the file if it already exists
@@ -64,8 +121,9 @@ export class HFDatasetUploader {
64
 
65
  const filePath = this.getHFFilePath(benchmark);
66
 
67
- // Convert benchmark to JSON string
68
- const content = JSON.stringify(benchmark, null, 2);
 
69
  const blob = new Blob([content], { type: "application/json" });
70
 
71
  try {
 
46
  batchSize: benchmark.batchSize,
47
  browser: benchmark.browser,
48
  headed: benchmark.headed,
49
+ environment: benchmark.result?.environment ? {
50
+ cpu: benchmark.result.environment.cpu,
51
+ memory: benchmark.result.environment.memory,
52
+ gpu: benchmark.result.environment.gpu,
53
+ platform: benchmark.result.environment.platform,
54
+ arch: benchmark.result.environment.arch,
55
+ cpuCores: benchmark.result.environment.cpuCores, // Web browser format
56
+ } : undefined,
57
  };
58
 
59
  const { fullPath } = generateBenchmarkPath(settings);
 
61
  return fullPath.replace(/\.jsonl$/, ".json");
62
  }
63
 
64
+ /**
65
+ * Transform benchmark data for HF Dataset upload
66
+ * Lifts frequently-accessed fields to top level for easier browsing
67
+ */
68
+ private transformForUpload(benchmark: QueuedBenchmark): any {
69
+ const result = benchmark.result || {};
70
+
71
+ return {
72
+ // Top-level metadata
73
+ id: benchmark.id,
74
+ status: benchmark.status,
75
+ timestamp: benchmark.timestamp,
76
+ startedAt: benchmark.startedAt,
77
+ completedAt: benchmark.completedAt,
78
+
79
+ // Configuration (lifted to top level)
80
+ platform: benchmark.platform,
81
+ modelId: benchmark.modelId,
82
+ task: benchmark.task,
83
+ mode: benchmark.mode,
84
+ device: benchmark.device,
85
+ dtype: benchmark.dtype,
86
+ batchSize: benchmark.batchSize,
87
+ repeats: benchmark.repeats,
88
+
89
+ // Browser-specific (only if web platform)
90
+ ...(benchmark.platform === "web" && {
91
+ browser: benchmark.browser,
92
+ headed: benchmark.headed,
93
+ }),
94
+
95
+ // Runtime info (lifted from result)
96
+ runtime: result.runtime,
97
+
98
+ // Metrics (lifted to top level for easy access)
99
+ metrics: result.metrics,
100
+
101
+ // Environment (lifted to top level for easy access)
102
+ environment: result.environment,
103
+
104
+ // Error info (if present)
105
+ ...(result.error && { error: result.error }),
106
+
107
+ // Additional metadata
108
+ ...(result.cacheDir && { cacheDir: result.cacheDir }),
109
+ ...(result.notes && { notes: result.notes }),
110
+ };
111
+ }
112
+
113
  /**
114
  * Upload a benchmark result to HF Dataset
115
  * Overwrites the file if it already exists
 
121
 
122
  const filePath = this.getHFFilePath(benchmark);
123
 
124
+ // Transform and convert benchmark to JSON string
125
+ const transformed = this.transformForUpload(benchmark);
126
+ const content = JSON.stringify(transformed, null, 2);
127
  const blob = new Blob([content], { type: "application/json" });
128
 
129
  try {
bench/src/server/storage.ts CHANGED
@@ -26,6 +26,14 @@ export class BenchmarkStorage {
26
  batchSize: benchmark.batchSize,
27
  browser: benchmark.browser,
28
  headed: benchmark.headed,
 
 
 
 
 
 
 
 
29
  };
30
 
31
  const { dir, filename } = generateBenchmarkPath(settings);
 
26
  batchSize: benchmark.batchSize,
27
  browser: benchmark.browser,
28
  headed: benchmark.headed,
29
+ environment: benchmark.result?.environment ? {
30
+ cpu: benchmark.result.environment.cpu,
31
+ memory: benchmark.result.environment.memory,
32
+ gpu: benchmark.result.environment.gpu,
33
+ platform: benchmark.result.environment.platform,
34
+ arch: benchmark.result.environment.arch,
35
+ cpuCores: benchmark.result.environment.cpuCores, // Web browser format
36
+ } : undefined,
37
  };
38
 
39
  const { dir, filename } = generateBenchmarkPath(settings);