solves monitoring integration

add_demo_data.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""
+Add demo training data to an existing experiment
+This will populate the experiment with realistic training metrics for visualization
+"""
+
+import json
+import logging
+import numpy as np
+from datetime import datetime
+from trackio_api_client import TrackioAPIClient
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def add_demo_training_data(experiment_id: str, num_steps: int = 50):
+    """Add realistic demo training data to an experiment"""
+    
+    client = TrackioAPIClient("https://tonic-test-trackio-test.hf.space")
+    
+    print(f"🎯 Adding demo training data to experiment: {experiment_id}")
+    print(f"📊 Will add {num_steps} metric entries...")
+    
+    # Simulate realistic training metrics
+    for step in range(0, num_steps * 25, 25):  # Every 25 steps
+        # Simulate loss decreasing over time with some noise
+        base_loss = 2.0 * np.exp(-step / 500)
+        noise = 0.1 * np.random.random()
+        loss = max(0.1, base_loss + noise)
+        
+        # Simulate accuracy increasing over time
+        base_accuracy = 0.3 + 0.6 * (1 - np.exp(-step / 300))
+        accuracy = min(0.95, base_accuracy + 0.05 * np.random.random())
+        
+        # Simulate learning rate decay
+        lr = 3.5e-6 * (0.9 ** (step // 200))
+        
+        # Simulate GPU memory usage
+        gpu_memory = 20 + 5 * np.random.random()
+        
+        # Simulate training time per step
+        training_time = 0.5 + 0.2 * np.random.random()
+        
+        metrics = {
+            "loss": round(loss, 4),
+            "accuracy": round(accuracy, 4),
+            "learning_rate": round(lr, 8),
+            "gpu_memory_gb": round(gpu_memory, 2),
+            "training_time_per_step": round(training_time, 3),
+            "epoch": step // 100 + 1,
+            "samples_per_second": round(50 + 20 * np.random.random(), 1)
+        }
+        
+        # Log metrics to the experiment
+        result = client.log_metrics(experiment_id, metrics, step)
+        
+        if "success" in result:
+            print(f"✅ Step {step}: Loss={loss:.4f}, Accuracy={accuracy:.4f}")
+        else:
+            print(f"❌ Step {step}: Failed to log metrics - {result}")
+    
+    print(f"\n🎉 Demo data added successfully!")
+    print(f"📊 Total steps logged: {num_steps}")
+    print(f"🔗 View in Trackio Space: https://tonic-test-trackio-test.hf.space")
+    print(f"📈 Go to 'Visualizations' tab and select experiment: {experiment_id}")
+
+def main():
+    """Main function"""
+    print("🚀 Trackio Demo Data Generator")
+    print("=" * 50)
+    
+    # Your experiment ID from the logs
+    experiment_id = "exp_20250720_101955"  # petit-elle-l-aime-3-balanced
+    
+    print(f"📋 Target experiment: {experiment_id}")
+    print(f"📝 Experiment name: petit-elle-l-aime-3-balanced")
+    
+    # Add demo data
+    add_demo_training_data(experiment_id, num_steps=50)
+    
+    print("\n" + "=" * 50)
+    print("🎯 Next Steps:")
+    print("1. Go to https://tonic-test-trackio-test.hf.space")
+    print("2. Click on '📊 Visualizations' tab")
+    print("3. Enter your experiment ID: exp_20250720_101955")
+    print("4. Select a metric (loss, accuracy, etc.)")
+    print("5. Click 'Create Plot' to see the training curves!")
+    print("=" * 50)
+
+if __name__ == "__main__":
+    main() 

app.py

@@ -25,7 +25,38 @@ class TrackioSpace:
     def __init__(self):
         self.experiments = {}
         self.current_experiment = None
+        self.data_file = "trackio_experiments.json"
+        self._load_experiments()
         
+    def _load_experiments(self):
+        """Load experiments from file"""
+        try:
+            if os.path.exists(self.data_file):
+                with open(self.data_file, 'r') as f:
+                    data = json.load(f)
+                    self.experiments = data.get('experiments', {})
+                    self.current_experiment = data.get('current_experiment')
+                logger.info(f"Loaded {len(self.experiments)} experiments from {self.data_file}")
+            else:
+                logger.info("No existing experiment data found, starting fresh")
+        except Exception as e:
+            logger.error(f"Failed to load experiments: {e}")
+            self.experiments = {}
+    
+    def _save_experiments(self):
+        """Save experiments to file"""
+        try:
+            data = {
+                'experiments': self.experiments,
+                'current_experiment': self.current_experiment,
+                'last_updated': datetime.now().isoformat()
+            }
+            with open(self.data_file, 'w') as f:
+                json.dump(data, f, indent=2, default=str)
+            logger.debug(f"Saved {len(self.experiments)} experiments to {self.data_file}")
+        except Exception as e:
+            logger.error(f"Failed to save experiments: {e}")
+    
     def create_experiment(self, name: str, description: str = "") -> Dict[str, Any]:
         """Create a new experiment"""
         experiment_id = f"exp_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
@@ -44,6 +75,7 @@ class TrackioSpace:
         
         self.experiments[experiment_id] = experiment
         self.current_experiment = experiment_id
+        self._save_experiments()
         
         logger.info(f"Created experiment: {experiment_id} - {name}")
         return experiment
@@ -60,6 +92,7 @@ class TrackioSpace:
         }
         
         self.experiments[experiment_id]['metrics'].append(metric_entry)
+        self._save_experiments()
         logger.info(f"Logged metrics for experiment {experiment_id}: {metrics}")
     
     def log_parameters(self, experiment_id: str, parameters: Dict[str, Any]):
@@ -68,6 +101,7 @@ class TrackioSpace:
             raise ValueError(f"Experiment {experiment_id} not found")
         
         self.experiments[experiment_id]['parameters'].update(parameters)
+        self._save_experiments()
         logger.info(f"Logged parameters for experiment {experiment_id}: {parameters}")
     
     def log_artifact(self, experiment_id: str, artifact_name: str, artifact_data: str):
@@ -82,6 +116,7 @@ class TrackioSpace:
         }
         
         self.experiments[experiment_id]['artifacts'].append(artifact_entry)
+        self._save_experiments()
         logger.info(f"Logged artifact for experiment {experiment_id}: {artifact_name}")
     
     def get_experiment(self, experiment_id: str) -> Optional[Dict[str, Any]]:
@@ -100,6 +135,7 @@ class TrackioSpace:
         """Update experiment status"""
         if experiment_id in self.experiments:
             self.experiments[experiment_id]['status'] = status
+            self._save_experiments()
             logger.info(f"Updated experiment {experiment_id} status to {status}")
     
     def get_metrics_dataframe(self, experiment_id: str) -> pd.DataFrame:
@@ -127,6 +163,87 @@ class TrackioSpace:
 # Initialize Trackio space
 trackio_space = TrackioSpace()
 
+# Initialize API client for remote data
+api_client = None
+try:
+    from trackio_api_client import TrackioAPIClient
+    api_client = TrackioAPIClient("https://tonic-test-trackio-test.hf.space")
+    logger.info("✅ API client initialized for remote data access")
+except ImportError:
+    logger.warning("⚠️ API client not available, using local data only")
+
+def get_remote_experiment_data(experiment_id: str) -> Dict[str, Any]:
+    """Get experiment data from remote API"""
+    if api_client is None:
+        return None
+    
+    try:
+        # Get experiment details from API
+        details_result = api_client.get_experiment_details(experiment_id)
+        if "success" in details_result:
+            return {"remote": True, "data": details_result["data"]}
+        else:
+            logger.warning(f"Failed to get remote data for {experiment_id}: {details_result}")
+            return None
+    except Exception as e:
+        logger.error(f"Error getting remote data: {e}")
+        return None
+
+def parse_remote_metrics_data(experiment_details: str) -> pd.DataFrame:
+    """Parse metrics data from remote experiment details"""
+    try:
+        # Look for metrics in the experiment details
+        lines = experiment_details.split('\n')
+        metrics_data = []
+        
+        for line in lines:
+            if 'Step:' in line and 'Metrics:' in line:
+                # Extract step and metrics from the line
+                try:
+                    # Parse step number
+                    step_part = line.split('Step:')[1].split('Metrics:')[0].strip()
+                    step = int(step_part)
+                    
+                    # Parse metrics JSON
+                    metrics_part = line.split('Metrics:')[1].strip()
+                    metrics = json.loads(metrics_part)
+                    
+                    # Add timestamp
+                    row = {'step': step, 'timestamp': datetime.now().isoformat()}
+                    row.update(metrics)
+                    metrics_data.append(row)
+                    
+                except (ValueError, json.JSONDecodeError) as e:
+                    logger.warning(f"Failed to parse metrics line: {line} - {e}")
+                    continue
+        
+        if metrics_data:
+            return pd.DataFrame(metrics_data)
+        else:
+            return pd.DataFrame()
+            
+    except Exception as e:
+        logger.error(f"Error parsing remote metrics: {e}")
+        return pd.DataFrame()
+
+def get_metrics_dataframe(experiment_id: str) -> pd.DataFrame:
+    """Get metrics as a pandas DataFrame for plotting - tries remote first, then local"""
+    # Try to get remote data first
+    remote_data = get_remote_experiment_data(experiment_id)
+    if remote_data:
+        logger.info(f"Using remote data for {experiment_id}")
+        # Parse the remote experiment details to extract metrics
+        df = parse_remote_metrics_data(remote_data["data"])
+        if not df.empty:
+            logger.info(f"Found {len(df)} metrics entries from remote data")
+            return df
+        else:
+            logger.warning(f"No metrics found in remote data for {experiment_id}")
+    
+    # Fall back to local data
+    logger.info(f"Using local data for {experiment_id}")
+    return trackio_space.get_metrics_dataframe(experiment_id)
+
 def create_experiment_interface(name: str, description: str) -> str:
     """Create a new experiment"""
     try:
@@ -236,7 +353,7 @@ def update_experiment_status_interface(experiment_id: str, status: str) -> str:
 def create_metrics_plot(experiment_id: str, metric_name: str = "loss") -> go.Figure:
     """Create a plot for a specific metric"""
     try:
-        df = trackio_space.get_metrics_dataframe(experiment_id)
+        df = get_metrics_dataframe(experiment_id)
         if df.empty:
             # Return empty plot
             fig = go.Figure()
@@ -283,7 +400,7 @@ def create_experiment_comparison(experiment_ids: str) -> go.Figure:
         fig = go.Figure()
         
         for exp_id in exp_ids:
-            df = trackio_space.get_metrics_dataframe(exp_id)
+            df = get_metrics_dataframe(exp_id)
             if not df.empty and 'loss' in df.columns:
                 fig.add_trace(go.Scatter(
                     x=df['step'],
@@ -335,6 +452,35 @@ def simulate_training_data(experiment_id: str):
     except Exception as e:
         return f"❌ Error simulating data: {str(e)}"
 
+def create_demo_experiment():
+    """Create a demo experiment with training data"""
+    try:
+        # Create demo experiment
+        experiment = trackio_space.create_experiment(
+            "demo_smollm3_training",
+            "Demo experiment with simulated training data"
+        )
+        
+        experiment_id = experiment['id']
+        
+        # Add some demo parameters
+        parameters = {
+            "model_name": "HuggingFaceTB/SmolLM3-3B",
+            "batch_size": 8,
+            "learning_rate": 3.5e-6,
+            "max_iters": 18000,
+            "mixed_precision": "bf16",
+            "dataset": "legmlai/openhermes-fr"
+        }
+        trackio_space.log_parameters(experiment_id, parameters)
+        
+        # Add demo training data
+        simulate_training_data(experiment_id)
+        
+        return f"✅ Demo experiment created: {experiment_id}\nYou can now test the visualization with this experiment!"
+    except Exception as e:
+        return f"❌ Error creating demo experiment: {str(e)}"
+
 # Create Gradio interface
 with gr.Blocks(title="Trackio - Experiment Tracking", theme=gr.themes.Soft()) as demo:
     gr.Markdown("# 🚀 Trackio Experiment Tracking & Monitoring")
@@ -518,11 +664,12 @@ with gr.Blocks(title="Trackio - Experiment Tracking", theme=gr.themes.Soft()) as
                         placeholder="exp_20231201_143022"
                     )
                     demo_btn = gr.Button("Generate Demo Data", variant="primary")
+                    create_demo_btn = gr.Button("Create Demo Experiment", variant="secondary")
                 
                 with gr.Column():
                     demo_output = gr.Textbox(
                         label="Result",
-                        lines=3,
+                        lines=5,
                         interactive=False
                     )
             
@@ -531,6 +678,12 @@ with gr.Blocks(title="Trackio - Experiment Tracking", theme=gr.themes.Soft()) as
                 inputs=[demo_exp_id],
                 outputs=demo_output
             )
+            
+            create_demo_btn.click(
+                create_demo_experiment,
+                inputs=[],
+                outputs=demo_output
+            )
         
         # Update Status Tab
         with gr.Tab("Update Status"):

test_monitoring_integration.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""
+Test monitoring integration for real experiment
+"""
+
+import os
+import sys
+from pathlib import Path
+
+# Add the current directory to the path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+def test_monitoring_setup():
+    """Test that monitoring is correctly configured"""
+    
+    print("🔍 Testing Monitoring Integration")
+    print("=" * 50)
+    
+    # Test 1: Check if monitoring module can be imported
+    try:
+        from monitoring import SmolLM3Monitor, create_monitor_from_config
+        print("✅ Monitoring module imported successfully")
+    except ImportError as e:
+        print(f"❌ Failed to import monitoring module: {e}")
+        return False
+    
+    # Test 2: Check if API client can be imported
+    try:
+        from trackio_api_client import TrackioAPIClient
+        print("✅ Trackio API client imported successfully")
+    except ImportError as e:
+        print(f"❌ Failed to import Trackio API client: {e}")
+        return False
+    
+    # Test 3: Test configuration loading
+    try:
+        from config.train_smollm3_openhermes_fr_a100_balanced import get_config
+        config = get_config("config/train_smollm3_openhermes_fr_a100_balanced.py")
+        print("✅ Configuration loaded successfully")
+        print(f"   Model: {config.model_name}")
+        print(f"   Batch size: {config.batch_size}")
+        print(f"   Max iterations: {config.max_iters}")
+        print(f"   Enable tracking: {config.enable_tracking}")
+        print(f"   Trackio URL: {config.trackio_url}")
+    except Exception as e:
+        print(f"❌ Failed to load configuration: {e}")
+        return False
+    
+    # Test 4: Test monitor creation
+    try:
+        # Set the Trackio URL for testing
+        config.trackio_url = "https://tonic-test-trackio-test.hf.space"
+        config.experiment_name = "test_monitoring_integration"
+        
+        monitor = create_monitor_from_config(config)
+        print("✅ Monitor created successfully")
+        print(f"   Experiment name: {monitor.experiment_name}")
+        print(f"   Enable tracking: {monitor.enable_tracking}")
+        print(f"   Log metrics: {monitor.log_metrics}")
+        print(f"   Log artifacts: {monitor.log_artifacts}")
+        
+        if monitor.enable_tracking and monitor.trackio_client:
+            print("✅ Trackio client initialized")
+            if monitor.experiment_id:
+                print(f"   Experiment ID: {monitor.experiment_id}")
+            else:
+                print("   ⚠️ No experiment ID (will be created during training)")
+        else:
+            print("   ⚠️ Trackio client not initialized")
+            
+    except Exception as e:
+        print(f"❌ Failed to create monitor: {e}")
+        return False
+    
+    # Test 5: Test callback creation
+    try:
+        callback = monitor.create_monitoring_callback()
+        if callback:
+            print("✅ Monitoring callback created successfully")
+        else:
+            print("   ⚠️ No monitoring callback (tracking disabled)")
+    except Exception as e:
+        print(f"❌ Failed to create callback: {e}")
+        return False
+    
+    print("\n" + "=" * 50)
+    print("🎯 Monitoring Integration Test Complete")
+    print("=" * 50)
+    
+    return True
+
+def test_real_experiment_command():
+    """Test the real experiment command"""
+    
+    print("\n🚀 Testing Real Experiment Command")
+    print("=" * 50)
+    
+    # Build the command
+    cmd = [
+        "python", "run_a100_large_experiment.py",
+        "--config", "config/train_smollm3_openhermes_fr_a100_balanced.py",
+        "--experiment-name", "petit-elle-l-aime-3-balanced-real",
+        "--output-dir", "./outputs/balanced-real",
+        "--trackio-url", "https://tonic-test-trackio-test.hf.space"
+    ]
+    
+    print("Command to run:")
+    print(" ".join(cmd))
+    
+    print("\nThis command will:")
+    print("✅ Load the balanced A100 configuration")
+    print("✅ Create a real experiment in Trackio")
+    print("✅ Log real training metrics every 25 steps")
+    print("✅ Save checkpoints every 2000 steps")
+    print("✅ Monitor progress in real-time")
+    
+    print("\nExpected training parameters:")
+    print("   Model: HuggingFaceTB/SmolLM3-3B")
+    print("   Batch size: 8")
+    print("   Gradient accumulation: 16")
+    print("   Effective batch size: 128")
+    print("   Learning rate: 3.5e-6")
+    print("   Max iterations: 18000")
+    print("   Mixed precision: bf16")
+    print("   Max sequence length: 12288")
+    
+    print("\n" + "=" * 50)
+    print("🎯 Ready to run real experiment!")
+    print("=" * 50)
+
+if __name__ == "__main__":
+    # Test monitoring integration
+    if test_monitoring_setup():
+        # Show real experiment command
+        test_real_experiment_command()
+    else:
+        print("\n❌ Monitoring integration test failed. Please fix issues before running real experiment.") 

test_persistence.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""
+Test data persistence in Trackio Space
+"""
+
+import requests
+import json
+import time
+import re
+
+def test_persistence():
+    """Test if experiment data persists across restarts"""
+    
+    print("🔍 Testing Data Persistence")
+    print("=" * 50)
+    
+    # Test creating an experiment via API
+    url = 'https://tonic-test-trackio-test.hf.space/gradio_api/call/create_experiment_interface'
+    payload = {'data': ['test_persistence', 'Testing data persistence']}
+    
+    response = requests.post(url, json=payload)
+    if response.status_code == 200:
+        data = response.json()
+        if 'event_id' in data:
+            event_id = data['event_id']
+            print(f'✅ Experiment created with event ID: {event_id}')
+            
+            # Get the result
+            get_url = f'{url}/{event_id}'
+            time.sleep(2)
+            
+            get_response = requests.get(get_url)
+            if get_response.status_code == 200:
+                result = get_response.text
+                print(f'✅ Experiment creation result: {result[:200]}...')
+                
+                # Extract experiment ID
+                match = re.search(r'exp_\d{8}_\d{6}', result)
+                if match:
+                    experiment_id = match.group()
+                    print(f'📋 Experiment ID: {experiment_id}')
+                    
+                    # Test logging metrics
+                    metrics_url = 'https://tonic-test-trackio-test.hf.space/gradio_api/call/log_metrics_interface'
+                    metrics_payload = {
+                        'data': [experiment_id, '{"loss": 1.5, "accuracy": 0.8}', '100']
+                    }
+                    
+                    metrics_response = requests.post(metrics_url, json=metrics_payload)
+                    if metrics_response.status_code == 200:
+                        print('✅ Metrics logged successfully')
+                        
+                        # Test getting experiment details
+                        details_url = 'https://tonic-test-trackio-test.hf.space/gradio_api/call/get_experiment_details'
+                        details_payload = {'data': [experiment_id]}
+                        
+                        details_response = requests.post(details_url, json=details_payload)
+                        if details_response.status_code == 200:
+                            details_data = details_response.json()
+                            if 'event_id' in details_data:
+                                details_event_id = details_data['event_id']
+                                
+                                # Get details result
+                                details_get_url = f'{details_url}/{details_event_id}'
+                                time.sleep(2)
+                                
+                                details_get_response = requests.get(details_get_url)
+                                if details_get_response.status_code == 200:
+                                    details_result = details_get_response.text
+                                    print(f'✅ Experiment details retrieved: {details_result[:200]}...')
+                                    
+                                    if 'metrics' in details_result.lower():
+                                        print('✅ Found metrics in experiment details')
+                                    else:
+                                        print('❌ No metrics found in experiment details')
+                                else:
+                                    print(f'❌ Failed to get details result: {details_get_response.status_code}')
+                            else:
+                                print('❌ No event_id in details response')
+                        else:
+                            print(f'❌ Failed to get experiment details: {details_response.status_code}')
+                    else:
+                        print(f'❌ Failed to log metrics: {metrics_response.status_code}')
+                else:
+                    print('❌ Could not extract experiment ID')
+            else:
+                print(f'❌ Failed to get result: {get_response.status_code}')
+        else:
+            print('❌ No event_id in response')
+    else:
+        print(f'❌ Failed to create experiment: {response.status_code}')
+    
+    print("\n" + "=" * 50)
+    print("🎯 Next Steps:")
+    print("1. Check the Trackio Space: https://tonic-test-trackio-test.hf.space")
+    print("2. Go to '📊 Visualizations' tab")
+    print("3. Enter the experiment ID above")
+    print("4. Test if the visualization shows data")
+    print("=" * 50)
+
+if __name__ == "__main__":
+    test_persistence() 

test_real_data.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""
+Test real training data logging and retrieval
+"""
+
+import json
+import logging
+from trackio_api_client import TrackioAPIClient
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def test_real_training_data():
+    """Test if real training data is being logged and can be retrieved"""
+    
+    client = TrackioAPIClient("https://tonic-test-trackio-test.hf.space")
+    
+    # Your experiment ID
+    experiment_id = "exp_20250720_101955"
+    
+    print("🔍 Testing Real Training Data")
+    print("=" * 50)
+    
+    # 1. Test getting experiment details
+    print(f"\n1. Getting experiment details for {experiment_id}...")
+    details_result = client.get_experiment_details(experiment_id)
+    
+    if "success" in details_result:
+        print("✅ Experiment details retrieved successfully")
+        try:
+            details_preview = details_result['data'][:200]
+            print(f"Details: {details_preview}...")
+        except UnicodeEncodeError:
+            print(f"Details: {details_result['data'][:100].encode('utf-8', errors='ignore').decode('utf-8')}...")
+        
+        # Look for metrics in the details
+        if "metrics" in details_result['data'].lower():
+            print("✅ Found metrics in experiment details")
+        else:
+            print("❌ No metrics found in experiment details")
+    else:
+        print(f"❌ Failed to get experiment details: {details_result}")
+    
+    # 2. Test getting training metrics specifically
+    print(f"\n2. Getting training metrics for {experiment_id}...")
+    metrics_result = client.get_training_metrics(experiment_id)
+    
+    if "success" in metrics_result:
+        print("✅ Training metrics retrieved successfully")
+        print(f"Metrics: {metrics_result['data'][:200]}...")
+    else:
+        print(f"❌ Failed to get training metrics: {metrics_result}")
+    
+    # 3. Test getting metrics history
+    print(f"\n3. Getting metrics history for {experiment_id}...")
+    history_result = client.get_experiment_metrics_history(experiment_id)
+    
+    if "success" in history_result:
+        print("✅ Metrics history retrieved successfully")
+        print(f"History: {history_result['data'][:200]}...")
+    else:
+        print(f"❌ Failed to get metrics history: {history_result}")
+    
+    # 4. List all experiments to see what's available
+    print(f"\n4. Listing all experiments...")
+    list_result = client.list_experiments()
+    
+    if "success" in list_result:
+        print("✅ Experiments listed successfully")
+        try:
+            response_preview = list_result['data'][:300]
+            print(f"Response: {response_preview}...")
+        except UnicodeEncodeError:
+            print(f"Response: {list_result['data'][:150].encode('utf-8', errors='ignore').decode('utf-8')}...")
+    else:
+        print(f"❌ Failed to list experiments: {list_result}")
+    
+    print("\n" + "=" * 50)
+    print("🎯 Analysis Complete")
+    print("=" * 50)
+
+def log_real_training_step(experiment_id: str, step: int):
+    """Log a single real training step for testing"""
+    
+    client = TrackioAPIClient("https://tonic-test-trackio-test.hf.space")
+    
+    # Real training metrics
+    metrics = {
+        "loss": 1.2345,
+        "accuracy": 0.8567,
+        "learning_rate": 3.5e-6,
+        "gpu_memory_gb": 22.5,
+        "training_time_per_step": 0.8,
+        "epoch": 1,
+        "samples_per_second": 45.2
+    }
+    
+    print(f"📊 Logging real training step {step}...")
+    result = client.log_metrics(experiment_id, metrics, step)
+    
+    if "success" in result:
+        print(f"✅ Step {step} logged successfully")
+        print(f"Metrics: {metrics}")
+    else:
+        print(f"❌ Failed to log step {step}: {result}")
+
+if __name__ == "__main__":
+    # Test existing data
+    test_real_training_data()
+    
+    # Optionally log a test step
+    print("\n" + "=" * 50)
+    print("🧪 Testing Real Data Logging")
+    print("=" * 50)
+    
+    experiment_id = "exp_20250720_101955"
+    log_real_training_step(experiment_id, 1000)
+    
+    print("\n" + "=" * 50)
+    print("🎯 Next Steps:")
+    print("1. Run your actual training: python run_a100_large_experiment.py")
+    print("2. The training will log real metrics every 25 steps")
+    print("3. Check the visualization tab in your Trackio Space")
+    print("4. Real training data should appear as training progresses")
+    print("=" * 50) 

trackio_api_client.py

@@ -258,6 +258,32 @@ class TrackioAPIClient:
         else:
             logger.error(f"Failed to simulate training data: {result}")
             return result
+    
+    def get_training_metrics(self, experiment_id: str) -> Dict[str, Any]:
+        """Get training metrics for an experiment"""
+        logger.info(f"Getting training metrics for experiment {experiment_id}")
+        
+        result = self._make_api_call("get_training_metrics", [experiment_id])
+        
+        if "success" in result:
+            logger.info(f"Training metrics retrieved: {result['data'][:100]}...")
+            return result
+        else:
+            logger.error(f"Failed to get training metrics: {result}")
+            return result
+    
+    def get_experiment_metrics_history(self, experiment_id: str) -> Dict[str, Any]:
+        """Get complete metrics history for an experiment"""
+        logger.info(f"Getting metrics history for experiment {experiment_id}")
+        
+        result = self._make_api_call("get_metrics_history", [experiment_id])
+        
+        if "success" in result:
+            logger.info(f"Metrics history retrieved: {result['data'][:100]}...")
+            return result
+        else:
+            logger.error(f"Failed to get metrics history: {result}")
+            return result
 
 def test_simple_connection():
     """Test basic connectivity to the Space"""