Spaces:
Runtime error
Runtime error
Add env info into the ID
Browse files- bench/src/core/benchmark-id.ts +79 -0
- bench/src/server/hf-dataset.ts +60 -2
- bench/src/server/storage.ts +8 -0
bench/src/core/benchmark-id.ts
CHANGED
|
@@ -8,6 +8,27 @@
|
|
| 8 |
* 4. Are sortable and searchable
|
| 9 |
*/
|
| 10 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
export interface BenchmarkSettings {
|
| 12 |
platform: "node" | "web";
|
| 13 |
modelId: string;
|
|
@@ -18,6 +39,7 @@ export interface BenchmarkSettings {
|
|
| 18 |
batchSize?: number;
|
| 19 |
browser?: string;
|
| 20 |
headed?: boolean;
|
|
|
|
| 21 |
}
|
| 22 |
|
| 23 |
/**
|
|
@@ -48,6 +70,17 @@ export function generateBenchmarkId(settings: BenchmarkSettings): string {
|
|
| 48 |
return `${task}/${modelId}/${filenameParts.join("_")}`;
|
| 49 |
}
|
| 50 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
/**
|
| 52 |
* Generate the filename parts (everything except task and model ID)
|
| 53 |
*/
|
|
@@ -88,6 +121,52 @@ function generateFilenameParts(settings: BenchmarkSettings): string[] {
|
|
| 88 |
parts.push("headed");
|
| 89 |
}
|
| 90 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
return parts;
|
| 92 |
}
|
| 93 |
|
|
|
|
| 8 |
* 4. Are sortable and searchable
|
| 9 |
*/
|
| 10 |
|
| 11 |
+
export interface EnvironmentInfo {
|
| 12 |
+
// Node.js format
|
| 13 |
+
cpu?: {
|
| 14 |
+
model?: string;
|
| 15 |
+
cores?: number;
|
| 16 |
+
};
|
| 17 |
+
memory?: {
|
| 18 |
+
total?: string;
|
| 19 |
+
deviceMemory?: number; // Web browser format (GB)
|
| 20 |
+
};
|
| 21 |
+
gpu?: {
|
| 22 |
+
vendor?: string;
|
| 23 |
+
renderer?: string;
|
| 24 |
+
};
|
| 25 |
+
platform?: string;
|
| 26 |
+
arch?: string;
|
| 27 |
+
|
| 28 |
+
// Web browser format (direct fields)
|
| 29 |
+
cpuCores?: number;
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
export interface BenchmarkSettings {
|
| 33 |
platform: "node" | "web";
|
| 34 |
modelId: string;
|
|
|
|
| 39 |
batchSize?: number;
|
| 40 |
browser?: string;
|
| 41 |
headed?: boolean;
|
| 42 |
+
environment?: EnvironmentInfo;
|
| 43 |
}
|
| 44 |
|
| 45 |
/**
|
|
|
|
| 70 |
return `${task}/${modelId}/${filenameParts.join("_")}`;
|
| 71 |
}
|
| 72 |
|
| 73 |
+
/**
|
| 74 |
+
* Sanitize environment strings for use in filenames
|
| 75 |
+
*/
|
| 76 |
+
function sanitizeEnvString(str: string): string {
|
| 77 |
+
return str
|
| 78 |
+
.toLowerCase()
|
| 79 |
+
.replace(/\s+/g, "-")
|
| 80 |
+
.replace(/[^a-z0-9\-]/g, "")
|
| 81 |
+
.substring(0, 20); // Limit length
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
/**
|
| 85 |
* Generate the filename parts (everything except task and model ID)
|
| 86 |
*/
|
|
|
|
| 121 |
parts.push("headed");
|
| 122 |
}
|
| 123 |
|
| 124 |
+
// 8. Environment info (CPU, memory, architecture, GPU)
|
| 125 |
+
if (settings.environment) {
|
| 126 |
+
const env = settings.environment;
|
| 127 |
+
|
| 128 |
+
// CPU model (sanitized, first significant part) - Node.js only
|
| 129 |
+
if (env.cpu?.model) {
|
| 130 |
+
const cpuName = sanitizeEnvString(env.cpu.model.split(/[\s(]/)[0]);
|
| 131 |
+
if (cpuName) {
|
| 132 |
+
parts.push(cpuName);
|
| 133 |
+
}
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
// CPU cores (support both Node.js and web browser formats)
|
| 137 |
+
const cores = env.cpu?.cores || env.cpuCores;
|
| 138 |
+
if (cores) {
|
| 139 |
+
parts.push(`${cores}c`);
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
// Memory (support both Node.js and web browser formats)
|
| 143 |
+
if (env.memory?.total) {
|
| 144 |
+
// Node.js format: Parse memory string like "32.00 GB" -> "32gb"
|
| 145 |
+
const memMatch = env.memory.total.match(/^(\d+)/);
|
| 146 |
+
if (memMatch) {
|
| 147 |
+
parts.push(`${memMatch[1]}gb`);
|
| 148 |
+
}
|
| 149 |
+
} else if (env.memory?.deviceMemory) {
|
| 150 |
+
// Web browser format: deviceMemory is already in GB
|
| 151 |
+
parts.push(`${env.memory.deviceMemory}gb`);
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
// Architecture (Node.js only, browser uses platform like "MacIntel")
|
| 155 |
+
if (env.arch) {
|
| 156 |
+
parts.push(env.arch);
|
| 157 |
+
}
|
| 158 |
+
|
| 159 |
+
// GPU vendor/renderer for web (if using webgpu)
|
| 160 |
+
if (settings.platform === "web" && settings.device === "webgpu" && env.gpu) {
|
| 161 |
+
if (env.gpu.vendor) {
|
| 162 |
+
const gpuVendor = sanitizeEnvString(env.gpu.vendor.split(/[\s(]/)[0]);
|
| 163 |
+
if (gpuVendor && gpuVendor !== "google") { // Skip "Google Inc."
|
| 164 |
+
parts.push(`gpu-${gpuVendor}`);
|
| 165 |
+
}
|
| 166 |
+
}
|
| 167 |
+
}
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
return parts;
|
| 171 |
}
|
| 172 |
|
bench/src/server/hf-dataset.ts
CHANGED
|
@@ -46,6 +46,14 @@ export class HFDatasetUploader {
|
|
| 46 |
batchSize: benchmark.batchSize,
|
| 47 |
browser: benchmark.browser,
|
| 48 |
headed: benchmark.headed,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
};
|
| 50 |
|
| 51 |
const { fullPath } = generateBenchmarkPath(settings);
|
|
@@ -53,6 +61,55 @@ export class HFDatasetUploader {
|
|
| 53 |
return fullPath.replace(/\.jsonl$/, ".json");
|
| 54 |
}
|
| 55 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
/**
|
| 57 |
* Upload a benchmark result to HF Dataset
|
| 58 |
* Overwrites the file if it already exists
|
|
@@ -64,8 +121,9 @@ export class HFDatasetUploader {
|
|
| 64 |
|
| 65 |
const filePath = this.getHFFilePath(benchmark);
|
| 66 |
|
| 67 |
-
//
|
| 68 |
-
const
|
|
|
|
| 69 |
const blob = new Blob([content], { type: "application/json" });
|
| 70 |
|
| 71 |
try {
|
|
|
|
| 46 |
batchSize: benchmark.batchSize,
|
| 47 |
browser: benchmark.browser,
|
| 48 |
headed: benchmark.headed,
|
| 49 |
+
environment: benchmark.result?.environment ? {
|
| 50 |
+
cpu: benchmark.result.environment.cpu,
|
| 51 |
+
memory: benchmark.result.environment.memory,
|
| 52 |
+
gpu: benchmark.result.environment.gpu,
|
| 53 |
+
platform: benchmark.result.environment.platform,
|
| 54 |
+
arch: benchmark.result.environment.arch,
|
| 55 |
+
cpuCores: benchmark.result.environment.cpuCores, // Web browser format
|
| 56 |
+
} : undefined,
|
| 57 |
};
|
| 58 |
|
| 59 |
const { fullPath } = generateBenchmarkPath(settings);
|
|
|
|
| 61 |
return fullPath.replace(/\.jsonl$/, ".json");
|
| 62 |
}
|
| 63 |
|
| 64 |
+
/**
|
| 65 |
+
* Transform benchmark data for HF Dataset upload
|
| 66 |
+
* Lifts frequently-accessed fields to top level for easier browsing
|
| 67 |
+
*/
|
| 68 |
+
private transformForUpload(benchmark: QueuedBenchmark): any {
|
| 69 |
+
const result = benchmark.result || {};
|
| 70 |
+
|
| 71 |
+
return {
|
| 72 |
+
// Top-level metadata
|
| 73 |
+
id: benchmark.id,
|
| 74 |
+
status: benchmark.status,
|
| 75 |
+
timestamp: benchmark.timestamp,
|
| 76 |
+
startedAt: benchmark.startedAt,
|
| 77 |
+
completedAt: benchmark.completedAt,
|
| 78 |
+
|
| 79 |
+
// Configuration (lifted to top level)
|
| 80 |
+
platform: benchmark.platform,
|
| 81 |
+
modelId: benchmark.modelId,
|
| 82 |
+
task: benchmark.task,
|
| 83 |
+
mode: benchmark.mode,
|
| 84 |
+
device: benchmark.device,
|
| 85 |
+
dtype: benchmark.dtype,
|
| 86 |
+
batchSize: benchmark.batchSize,
|
| 87 |
+
repeats: benchmark.repeats,
|
| 88 |
+
|
| 89 |
+
// Browser-specific (only if web platform)
|
| 90 |
+
...(benchmark.platform === "web" && {
|
| 91 |
+
browser: benchmark.browser,
|
| 92 |
+
headed: benchmark.headed,
|
| 93 |
+
}),
|
| 94 |
+
|
| 95 |
+
// Runtime info (lifted from result)
|
| 96 |
+
runtime: result.runtime,
|
| 97 |
+
|
| 98 |
+
// Metrics (lifted to top level for easy access)
|
| 99 |
+
metrics: result.metrics,
|
| 100 |
+
|
| 101 |
+
// Environment (lifted to top level for easy access)
|
| 102 |
+
environment: result.environment,
|
| 103 |
+
|
| 104 |
+
// Error info (if present)
|
| 105 |
+
...(result.error && { error: result.error }),
|
| 106 |
+
|
| 107 |
+
// Additional metadata
|
| 108 |
+
...(result.cacheDir && { cacheDir: result.cacheDir }),
|
| 109 |
+
...(result.notes && { notes: result.notes }),
|
| 110 |
+
};
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
/**
|
| 114 |
* Upload a benchmark result to HF Dataset
|
| 115 |
* Overwrites the file if it already exists
|
|
|
|
| 121 |
|
| 122 |
const filePath = this.getHFFilePath(benchmark);
|
| 123 |
|
| 124 |
+
// Transform and convert benchmark to JSON string
|
| 125 |
+
const transformed = this.transformForUpload(benchmark);
|
| 126 |
+
const content = JSON.stringify(transformed, null, 2);
|
| 127 |
const blob = new Blob([content], { type: "application/json" });
|
| 128 |
|
| 129 |
try {
|
bench/src/server/storage.ts
CHANGED
|
@@ -26,6 +26,14 @@ export class BenchmarkStorage {
|
|
| 26 |
batchSize: benchmark.batchSize,
|
| 27 |
browser: benchmark.browser,
|
| 28 |
headed: benchmark.headed,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
};
|
| 30 |
|
| 31 |
const { dir, filename } = generateBenchmarkPath(settings);
|
|
|
|
| 26 |
batchSize: benchmark.batchSize,
|
| 27 |
browser: benchmark.browser,
|
| 28 |
headed: benchmark.headed,
|
| 29 |
+
environment: benchmark.result?.environment ? {
|
| 30 |
+
cpu: benchmark.result.environment.cpu,
|
| 31 |
+
memory: benchmark.result.environment.memory,
|
| 32 |
+
gpu: benchmark.result.environment.gpu,
|
| 33 |
+
platform: benchmark.result.environment.platform,
|
| 34 |
+
arch: benchmark.result.environment.arch,
|
| 35 |
+
cpuCores: benchmark.result.environment.cpuCores, // Web browser format
|
| 36 |
+
} : undefined,
|
| 37 |
};
|
| 38 |
|
| 39 |
const { dir, filename } = generateBenchmarkPath(settings);
|