""" Data loader module for loading benchmark results from HuggingFace Dataset. """ import json import logging from pathlib import Path from typing import List, Dict, Any, Optional from datetime import datetime import pandas as pd from huggingface_hub import snapshot_download, list_models logger = logging.getLogger(__name__) def load_benchmark_data( dataset_repo: str, token: Optional[str] = None, ) -> pd.DataFrame: """Load benchmark data from HuggingFace Dataset repository. Args: dataset_repo: HuggingFace dataset repository ID (e.g., "username/dataset-name") token: HuggingFace API token (optional, for private datasets) Returns: DataFrame containing all benchmark results """ if not dataset_repo: return pd.DataFrame() try: # Download the entire repository snapshot logger.info(f"Downloading dataset snapshot from {dataset_repo}...") local_dir = snapshot_download( repo_id=dataset_repo, repo_type="dataset", token=token, ) logger.info(f"Dataset downloaded to {local_dir}") # Find all JSON files in the downloaded directory local_path = Path(local_dir) json_files = list(local_path.rglob("*.json")) if not json_files: logger.warning("No JSON files found in dataset") return pd.DataFrame() logger.info(f"Found {len(json_files)} JSON files") # Load all benchmark results all_results = [] for file_path in json_files: try: with open(file_path, "r") as f: result = json.load(f) if result: flattened = flatten_result(result) all_results.append(flattened) except Exception as e: logger.error(f"Error loading {file_path}: {e}") continue if not all_results: return pd.DataFrame() logger.info(f"Loaded {len(all_results)} benchmark results") # Convert to DataFrame df = pd.DataFrame(all_results) # Enrich with HuggingFace model metadata df = enrich_with_hf_metadata(df) # Add first-timer-friendly score df = add_first_timer_score(df) # Sort by model name and timestamp if "modelId" in df.columns and "timestamp" in df.columns: df = df.sort_values(["modelId", "timestamp"], ascending=[True, False]) return df except Exception as e: logger.error(f"Error loading benchmark data: {e}") return pd.DataFrame() def flatten_result(result: Dict[str, Any]) -> Dict[str, Any]: """Flatten nested benchmark result for display. The HF Dataset format is already flattened by the bench service, so we just need to extract the relevant fields. Args: result: Raw benchmark result dictionary Returns: Flattened dictionary with extracted fields """ # Convert timestamp from milliseconds to datetime timestamp_ms = result.get("timestamp", 0) timestamp_dt = None if timestamp_ms: try: timestamp_dt = datetime.fromtimestamp(timestamp_ms / 1000) except (ValueError, OSError): timestamp_dt = None # Determine actual status - if there's an error, it should be "failed" status = result.get("status", "") if "error" in result: status = "failed" flat = { "id": result.get("id", ""), "platform": result.get("platform", ""), "modelId": result.get("modelId", ""), "task": result.get("task", ""), "mode": result.get("mode", ""), "repeats": result.get("repeats", 0), "batchSize": result.get("batchSize", 0), "device": result.get("device", ""), "browser": result.get("browser", ""), "dtype": result.get("dtype", ""), "headed": result.get("headed", False), "status": status, "timestamp": timestamp_dt, "runtime": result.get("runtime", ""), # Initialize metric fields with None (will be filled if metrics exist) "load_ms_p50": None, "load_ms_p90": None, "first_infer_ms_p50": None, "first_infer_ms_p90": None, "subsequent_infer_ms_p50": None, "subsequent_infer_ms_p90": None, } # Extract metrics if available (already at top level) if "metrics" in result: metrics = result["metrics"] # Load time if "load_ms" in metrics and "p50" in metrics["load_ms"]: flat["load_ms_p50"] = metrics["load_ms"]["p50"] flat["load_ms_p90"] = metrics["load_ms"]["p90"] # First inference time if "first_infer_ms" in metrics and "p50" in metrics["first_infer_ms"]: flat["first_infer_ms_p50"] = metrics["first_infer_ms"]["p50"] flat["first_infer_ms_p90"] = metrics["first_infer_ms"]["p90"] # Subsequent inference time if "subsequent_infer_ms" in metrics and "p50" in metrics["subsequent_infer_ms"]: flat["subsequent_infer_ms_p50"] = metrics["subsequent_infer_ms"]["p50"] flat["subsequent_infer_ms_p90"] = metrics["subsequent_infer_ms"]["p90"] # Extract environment info (already at top level) if "environment" in result: env = result["environment"] flat["cpuCores"] = env.get("cpuCores", 0) if "memory" in env: flat["memory_gb"] = env["memory"].get("deviceMemory", 0) # Calculate duration if "completedAt" in result and "startedAt" in result: flat["duration_s"] = (result["completedAt"] - result["startedAt"]) / 1000 return flat def enrich_with_hf_metadata(df: pd.DataFrame) -> pd.DataFrame: """Enrich benchmark data with HuggingFace model metadata (downloads, likes). Args: df: DataFrame containing benchmark results token: HuggingFace API token (optional) Returns: DataFrame with added downloads and likes columns """ if df.empty or "modelId" not in df.columns: return df # Get unique model IDs model_ids = df["modelId"].unique().tolist() # Fetch metadata for all models model_metadata = {} logger.info(f"Fetching metadata for {len(model_ids)} models from HuggingFace...") try: for model in list_models(filter=["transformers.js"]): if model.id in model_ids: model_metadata[model.id] = { "downloads": model.downloads or 0, "likes": model.likes or 0, } # Break early if we have all models if len(model_metadata) == len(model_ids): break except Exception as e: logger.error(f"Error fetching HuggingFace metadata: {e}") # Add metadata to dataframe df["downloads"] = df["modelId"].map(lambda x: model_metadata.get(x, {}).get("downloads", 0)) df["likes"] = df["modelId"].map(lambda x: model_metadata.get(x, {}).get("likes", 0)) return df def add_first_timer_score(df: pd.DataFrame) -> pd.DataFrame: """Add first-timer-friendly score to all rows in the dataframe. The score is calculated per task, normalized from 0-100 where: - Higher score = better for first-timers - Based on: downloads (25%), likes (15%), load time (30%), inference time (30%) Args: df: DataFrame containing benchmark results Returns: DataFrame with added 'first_timer_score' column """ if df.empty: return df # Filter only successful benchmarks filtered = df[df["status"] == "completed"].copy() if "status" in df.columns else df.copy() if filtered.empty: # Add empty score column for failed benchmarks df["first_timer_score"] = None return df # Check if task column exists if "task" not in filtered.columns: df["first_timer_score"] = None return df # Calculate score per task for task in filtered["task"].unique(): task_mask = filtered["task"] == task task_df = filtered[task_mask].copy() if task_df.empty: continue # Normalize metrics within this task (0-1 scale) # Downloads score (0-1, higher is better) if "downloads" in task_df.columns: max_downloads = task_df["downloads"].max() downloads_score = task_df["downloads"] / max_downloads if max_downloads > 0 else 0 else: downloads_score = 0 # Likes score (0-1, higher is better) if "likes" in task_df.columns: max_likes = task_df["likes"].max() likes_score = task_df["likes"] / max_likes if max_likes > 0 else 0 else: likes_score = 0 # Load time score (0-1, lower time is better) if "load_ms_p50" in task_df.columns: max_load = task_df["load_ms_p50"].max() load_score = 1 - (task_df["load_ms_p50"] / max_load) if max_load > 0 else 0 else: load_score = 0 # Inference time score (0-1, lower time is better) if "first_infer_ms_p50" in task_df.columns: max_infer = task_df["first_infer_ms_p50"].max() infer_score = 1 - (task_df["first_infer_ms_p50"] / max_infer) if max_infer > 0 else 0 else: infer_score = 0 # Calculate weighted score and scale to 0-100 weighted_score = ( (downloads_score * 0.25) + (likes_score * 0.15) + (load_score * 0.30) + (infer_score * 0.30) ) * 100 # Assign scores back to the filtered dataframe filtered.loc[task_mask, "first_timer_score"] = weighted_score # Merge scores back to original dataframe if "first_timer_score" in filtered.columns: df = df.merge( filtered[["id", "first_timer_score"]], on="id", how="left" ) else: df["first_timer_score"] = None return df def filter_excluded_models(df: pd.DataFrame) -> pd.DataFrame: """Filter out models that should be excluded from recommendations. This function removes test models and other non-production models that should not be recommended to users. Args: df: DataFrame containing model data with a 'modelId' column Returns: DataFrame with excluded models removed """ if df.empty or "modelId" not in df.columns: return df # Exclude tiny-random test models (e.g., Xenova/tiny-random-RoFormerForMaskedLM) # These are small test models not meant for production use filtered = df[~df["modelId"].str.contains("tiny-random", case=False, na=False)] return filtered def get_first_timer_friendly_models(df: pd.DataFrame, limit_per_task: int = 3) -> pd.DataFrame: """Identify first-timer-friendly models based on popularity and performance, grouped by task. A model is considered first-timer-friendly if it: - Has high downloads (popular) - Has fast load times (easy to start) - Has fast inference times (quick results) - Successfully completed benchmarks Args: df: DataFrame containing benchmark results limit_per_task: Maximum number of models to return per task Returns: DataFrame with top first-timer-friendly models per task """ if df.empty: return pd.DataFrame() # Filter only successful benchmarks filtered = df[df["status"] == "completed"].copy() if "status" in df.columns else df.copy() # Exclude test models and other non-production models filtered = filter_excluded_models(filtered) if filtered.empty: return pd.DataFrame() # Check if task column exists if "task" not in filtered.columns: logger.warning("Task column not found in dataframe") return pd.DataFrame() # Calculate first-timer-friendliness score per task all_results = [] for task in filtered["task"].unique(): task_df = filtered[filtered["task"] == task].copy() if task_df.empty: continue # Normalize metrics within this task (lower is better for times, higher is better for popularity) # Downloads score (0-1, higher is better) if "downloads" in task_df.columns: max_downloads = task_df["downloads"].max() task_df["downloads_score"] = task_df["downloads"] / max_downloads if max_downloads > 0 else 0 else: task_df["downloads_score"] = 0 # Likes score (0-1, higher is better) if "likes" in task_df.columns: max_likes = task_df["likes"].max() task_df["likes_score"] = task_df["likes"] / max_likes if max_likes > 0 else 0 else: task_df["likes_score"] = 0 # Load time score (0-1, lower time is better) if "load_ms_p50" in task_df.columns: max_load = task_df["load_ms_p50"].max() task_df["load_score"] = 1 - (task_df["load_ms_p50"] / max_load) if max_load > 0 else 0 else: task_df["load_score"] = 0 # Inference time score (0-1, lower time is better) if "first_infer_ms_p50" in task_df.columns: max_infer = task_df["first_infer_ms_p50"].max() task_df["infer_score"] = 1 - (task_df["first_infer_ms_p50"] / max_infer) if max_infer > 0 else 0 else: task_df["infer_score"] = 0 # Calculate weighted first-timer-friendliness score # Weights: popularity (40%), load time (30%), inference time (30%) task_df["first_timer_score"] = ( (task_df["downloads_score"] * 0.25) + (task_df["likes_score"] * 0.15) + (task_df["load_score"] * 0.30) + (task_df["infer_score"] * 0.30) ) # Group by model and take best score for each model within this task # Filter out NaN scores before getting idxmax idx_max_series = task_df.groupby("modelId")["first_timer_score"].idxmax() # Drop NaN indices valid_indices = idx_max_series.dropna() if valid_indices.empty: continue best_per_model = task_df.loc[valid_indices] # Sort by first-timer score and take top N for this task top_for_task = best_per_model.sort_values("first_timer_score", ascending=False).head(limit_per_task) # Drop intermediate scoring columns score_cols = ["downloads_score", "likes_score", "load_score", "infer_score", "first_timer_score"] top_for_task = top_for_task.drop(columns=[col for col in score_cols if col in top_for_task.columns]) all_results.append(top_for_task) if not all_results: return pd.DataFrame() # Combine all results result = pd.concat(all_results, ignore_index=True) # Sort by task name for better organization if "task" in result.columns: result = result.sort_values("task") return result def get_webgpu_beginner_friendly_models( df: pd.DataFrame, limit_per_task: int = 5 ) -> pd.DataFrame: """Get top beginner-friendly models that are WebGPU compatible, grouped by task. A model is included if it: - Has high first_timer_score (popular, fast to load, fast inference) - Has successful WebGPU benchmark results (device=webgpu, status=completed) Args: df: DataFrame containing benchmark results limit_per_task: Maximum number of models to return per task (default: 5) Returns: DataFrame with top WebGPU-compatible beginner-friendly models per task """ if df.empty: return pd.DataFrame() # Filter for WebGPU benchmarks that completed successfully webgpu_filter = ( (df["device"] == "webgpu") & (df["status"] == "completed") ) # Check if required columns exist if "device" not in df.columns or "status" not in df.columns: logger.warning("Required columns (device, status) not found in dataframe") return pd.DataFrame() filtered = df[webgpu_filter].copy() # Exclude test models and other non-production models filtered = filter_excluded_models(filtered) if filtered.empty: logger.warning("No successful WebGPU benchmarks found") return pd.DataFrame() # Check if required columns exist if "task" not in filtered.columns or "first_timer_score" not in filtered.columns: logger.warning("Required columns (task, first_timer_score) not found in filtered dataframe") return pd.DataFrame() # Group by task and get top models all_results = [] for task in filtered["task"].unique(): task_df = filtered[filtered["task"] == task].copy() if task_df.empty: continue # Remove rows with NaN first_timer_score task_df = task_df.dropna(subset=["first_timer_score"]) if task_df.empty: continue # For each model, get the benchmark with the highest first_timer_score idx_max_series = task_df.groupby("modelId")["first_timer_score"].idxmax() valid_indices = idx_max_series.dropna() if valid_indices.empty: continue best_per_model = task_df.loc[valid_indices] # Sort by first_timer_score (descending) and take top N top_for_task = best_per_model.sort_values( "first_timer_score", ascending=False ).head(limit_per_task) all_results.append(top_for_task) if not all_results: logger.warning("No models found after filtering and grouping") return pd.DataFrame() # Combine all results result = pd.concat(all_results, ignore_index=True) # Sort by task, then by first_timer_score (descending) if "task" in result.columns and "first_timer_score" in result.columns: result = result.sort_values( ["task", "first_timer_score"], ascending=[True, False] ) return result def _get_usage_example(task_type: str, repo_id: str) -> tuple[str, str | None]: """Get usage example code snippet for a given task type. Args: task_type: The task type (e.g., 'text-generation', 'image-classification') repo_id: The model repository ID (e.g., 'Xenova/gpt2') Returns: Tuple of (code_snippet, description) """ if task_type == "fill-mask": return f"""const unmasker = await pipeline('fill-mask', '{repo_id}'); const output = await unmasker('The goal of life is [MASK].'); """, 'Perform masked language modelling (a.k.a. "fill-mask")' elif task_type == "question-answering": return f"""const answerer = await pipeline('question-answering', '{repo_id}'); const question = 'Who was Jim Henson?'; const context = 'Jim Henson was a nice puppet.'; const output = await answerer(question, context); """, 'Run question answering' elif task_type == "summarization": return f"""const generator = await pipeline('summarization', '{repo_id}'); const text = 'The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, ' + 'and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. ' + 'During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest ' + 'man-made structure in the world, a title it held for 41 years until the Chrysler Building in New ' + 'York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to ' + 'the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the ' + 'Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second ' + 'tallest free-standing structure in France after the Millau Viaduct.'; const output = await generator(text, {{ max_new_tokens: 100, }}); """, 'Summarization' elif task_type == "sentiment-analysis" or task_type == "text-classification": return f"""const classifier = await pipeline('{task_type}', '{repo_id}'); const output = await classifier('I love transformers!'); """, None elif task_type == "text-generation": return f"""const generator = await pipeline('text-generation', '{repo_id}'); const output = await generator('Once upon a time, there was', {{ max_new_tokens: 10 }}); """, 'Text generation' elif task_type == "text2text-generation": return f"""const generator = await pipeline('text2text-generation', '{repo_id}'); const output = await generator('how can I become more healthy?', {{ max_new_tokens: 100, }}); """, 'Text-to-text generation' elif task_type == "token-classification" or task_type == "ner": return f"""const classifier = await pipeline('token-classification', '{repo_id}'); const output = await classifier('My name is Sarah and I live in London'); """, 'Perform named entity recognition' elif task_type == "translation": return f"""const translator = await pipeline('translation', '{repo_id}'); const output = await translator('Life is like a box of chocolate.', {{ src_lang: '...', tgt_lang: '...', }}); """, 'Multilingual translation' elif task_type == "zero-shot-classification": return f"""const classifier = await pipeline('zero-shot-classification', '{repo_id}'); const output = await classifier( 'I love transformers!', ['positive', 'negative'] ); """, 'Zero shot classification' elif task_type == "feature-extraction": return f"""const extractor = await pipeline('feature-extraction', '{repo_id}'); const output = await extractor('This is a simple test.'); """, 'Run feature extraction' # Vision elif task_type == "background-removal": return f"""const segmenter = await pipeline('background-removal', '{repo_id}'); const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/portrait-of-woman_small.jpg'; const output = await segmenter(url); """, 'Perform background removal' elif task_type == "depth-estimation": return f"""const depth_estimator = await pipeline('depth-estimation', '{repo_id}'); const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg'; const out = await depth_estimator(url); """, 'Depth estimation' elif task_type == "image-classification": return f"""const classifier = await pipeline('image-classification', '{repo_id}'); const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/tiger.jpg'; const output = await classifier(url); """, 'Classify an image' elif task_type == "image-segmentation": return f"""const segmenter = await pipeline('image-segmentation', '{repo_id}'); const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg'; const output = await segmenter(url); """, 'Perform image segmentation' elif task_type == "image-to-image": return f"""const processor = await pipeline('image-to-image', '{repo_id}'); const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/tiger.jpg'; const output = await processor(url); """, None elif task_type == "object-detection": return f"""const detector = await pipeline('object-detection', '{repo_id}'); const img = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg'; const output = await detector(img, {{ threshold: 0.9 }}); """, 'Run object-detection' elif task_type == "image-feature-extraction": return f"""const image_feature_extractor = await pipeline('image-feature-extraction', '{repo_id}'); const url = 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/cats.png'; const features = await image_feature_extractor(url); """, 'Perform image feature extraction' # Audio elif task_type == "audio-classification": return f"""const classifier = await pipeline('audio-classification', '{repo_id}'); const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav'; const output = await classifier(url); """, 'Perform audio classification' elif task_type == "automatic-speech-recognition": return f"""const transcriber = await pipeline('automatic-speech-recognition', '{repo_id}'); const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav'; const output = await transcriber(url); """, 'Transcribe audio from a URL' elif task_type == "text-to-audio" or task_type == "text-to-speech": return f"""const synthesizer = await pipeline('text-to-speech', '{repo_id}'); const output = await synthesizer('Hello, my dog is cute'); """, 'Generate audio from text' # Multimodal elif task_type == "document-question-answering": return f"""const qa_pipeline = await pipeline('document-question-answering', '{repo_id}'); const image = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/invoice.png'; const question = 'What is the invoice number?'; const output = await qa_pipeline(image, question); """, 'Answer questions about a document' elif task_type == "image-to-text": return f"""const captioner = await pipeline('image-to-text', '{repo_id}'); const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg'; const output = await captioner(url); """, 'Generate a caption for an image' elif task_type == "zero-shot-audio-classification": return f"""const classifier = await pipeline('zero-shot-audio-classification', '{repo_id}'); const audio = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/dog_barking.wav'; const candidate_labels = ['dog', 'vaccum cleaner']; const scores = await classifier(audio, candidate_labels); """, 'Perform zero-shot audio classification' elif task_type == "zero-shot-image-classification": return f"""const classifier = await pipeline('zero-shot-image-classification', '{repo_id}'); const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/tiger.jpg'; const output = await classifier(url, ['tiger', 'horse', 'dog']); """, 'Zero shot image classification' elif task_type == "zero-shot-object-detection": return f"""const detector = await pipeline('zero-shot-object-detection', '{repo_id}'); const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/astronaut.png'; const candidate_labels = ['human face', 'rocket', 'helmet', 'american flag']; const output = await detector(url, candidate_labels); """, 'Zero-shot object detection' else: logger.warning(f"No usage example found for task type: {task_type}") return f"""const pipe = await pipeline('{task_type}', '{repo_id}'); const result = await pipe('input text or data'); console.log(result); """, None def format_recommended_models_as_markdown(df: pd.DataFrame) -> str: """Format recommended WebGPU models as markdown for llms.txt embedding. Args: df: DataFrame containing recommended models (output from get_webgpu_beginner_friendly_models) Returns: Formatted markdown string """ if df.empty: return "No recommended models available." markdown_lines = [ "# Recommended Transformers.js Models for First-Time Trials", "", "This guide provides curated model recommendations for each task type, selected for their:", "- **Popularity**: Widely used with strong community support", "- **Performance**: Fast loading and inference times", "- **WebGPU Compatibility**: GPU-accelerated in modern browsers", "", "**Important:** These recommendations are designed for initial experimentation and learning. " "Many other models are available for each task. " "**You should evaluate and choose the best model for your specific use case, performance requirements, and constraints.**", "", "## About the Model Recommendations", "", "The models below are selected for their popularity and ease of use, making them ideal for initial experimentation. " "**This list does not cover all available models** - you should evaluate and select the best model for your specific use case and requirements.", "", ] # Group by task if "task" not in df.columns: return "No task information available." for task in sorted(df["task"].unique()): task_df = df[df["task"] == task].copy() if task_df.empty: continue # Add task header markdown_lines.append(f"## {task.title()}") markdown_lines.append("") # Sort by first_timer_score descending if "first_timer_score" in task_df.columns: task_df = task_df.sort_values("first_timer_score", ascending=False) # Get the first/best model for the usage example first_row = task_df.iloc[0] first_model_id = first_row.get("modelId", "") # Add usage example using the top model if first_model_id: code_snippet, description = _get_usage_example(task, first_model_id) if description: markdown_lines.append(f"**Usage Example:** {description}") else: markdown_lines.append("**Usage Example:**") markdown_lines.append("") markdown_lines.append("```javascript") markdown_lines.append(code_snippet.strip()) markdown_lines.append("```") markdown_lines.append("") # Add section header for model recommendations markdown_lines.append("### Recommended Models for First-Time Trials") markdown_lines.append("") # Add each model for idx, row in task_df.iterrows(): model_id = row.get("modelId", "Unknown") score = row.get("first_timer_score", None) downloads = row.get("downloads", 0) likes = row.get("likes", 0) load_time = row.get("load_ms_p50", None) infer_time = row.get("first_infer_ms_p50", None) # Model entry markdown_lines.append(f"#### {model_id}") markdown_lines.append("") # WebGPU compatibility markdown_lines.append("**WebGPU Compatible:** ✅ Yes") markdown_lines.append("") # Metrics metrics = [] if load_time is not None: metrics.append(f"Load: {load_time:.1f}ms") if infer_time is not None: metrics.append(f"Inference: {infer_time:.1f}ms") if downloads: if downloads >= 1_000_000: downloads_str = f"{downloads / 1_000_000:.1f}M" elif downloads >= 1_000: downloads_str = f"{downloads / 1_000:.1f}k" else: downloads_str = str(downloads) metrics.append(f"Downloads: {downloads_str}") if likes: metrics.append(f"Likes: {likes}") if metrics: markdown_lines.append(f"**Metrics:** {' | '.join(metrics)}") markdown_lines.append("") markdown_lines.append("---") markdown_lines.append("") # Add footer markdown_lines.extend([ "## About These Recommendations", "", "### Selection Criteria", "", "Models in this guide are selected based on:", "- **Popularity**: High download counts and community engagement on HuggingFace Hub", "- **Performance**: Fast loading and inference times based on benchmark results", "- **Compatibility**: Verified WebGPU support for GPU-accelerated browser execution", "", "### Understanding Benchmark Metrics", "", "**Important:** All performance metrics (load time, inference time, etc.) are measured in a controlled benchmark environment. " "These metrics are useful for **comparing models against each other**, but they may not reflect the actual performance you'll experience in your specific environment. " "Factors that affect real-world performance include:", "- Hardware specifications (CPU, GPU, memory)", "- Browser type and version", "- Operating system", "- Network conditions (for model loading)", "- Concurrent processes and system load", "", "**We recommend** benchmarking models in your own environment with your actual use case to get accurate performance measurements.", "", "### For Production Use", "", "These recommendations are optimized for first-time trials and learning. " "For production applications, consider:", "- Evaluating multiple models for your specific use case", "- Testing with your actual data and performance requirements", "- Reviewing the full benchmark results for comprehensive comparisons", "- Exploring specialized models that may better fit your needs", "", "Visit the full leaderboard to explore all available models and their benchmark results.", ]) return "\n".join(markdown_lines) def get_unique_values(df: pd.DataFrame, column: str) -> List[str]: """Get unique values from a column for dropdown choices. Args: df: DataFrame to extract values from column: Column name Returns: List of unique values with "All" as first item """ if df.empty or column not in df.columns: return ["All"] values = df[column].dropna().unique().tolist() return ["All"] + sorted([str(v) for v in values])