make clear distinction between ensemble agents and smart agents

utils/monitoring_agents.py

@@ -0,0 +1,135 @@
+import logging
+import time
+import torch
+import psutil # Ensure psutil is imported here as well
+
+logger = logging.getLogger(__name__)
+
+class EnsembleMonitorAgent:
+    def __init__(self):
+        self.performance_metrics = {
+            "model_accuracy": {},
+            "response_times": {},
+            "confidence_distribution": {},
+            "consensus_rate": 0.0
+        }
+        self.alerts = []
+    
+    def monitor_prediction(self, model_id, prediction, confidence, response_time):
+        """Monitor individual model performance"""
+        if model_id not in self.performance_metrics["model_accuracy"]:
+            self.performance_metrics["model_accuracy"][model_id] = []
+            self.performance_metrics["response_times"][model_id] = []
+            self.performance_metrics["confidence_distribution"][model_id] = []
+        
+        self.performance_metrics["response_times"][model_id].append(response_time)
+        self.performance_metrics["confidence_distribution"][model_id].append(confidence)
+        
+        # Check for performance issues
+        self._check_performance_issues(model_id)
+    
+    def _check_performance_issues(self, model_id):
+        """Check for any performance anomalies"""
+        response_times = self.performance_metrics["response_times"][model_id]
+        if len(response_times) > 10:
+            avg_time = sum(response_times[-10:]) / 10
+            if avg_time > 2.0:  # More than 2 seconds
+                self.alerts.append(f"High latency detected for {model_id}: {avg_time:.2f}s")
+
+class WeightOptimizationAgent:
+    def __init__(self, weight_manager):
+        self.weight_manager = weight_manager
+        self.prediction_history = []  # Stores (ensemble_prediction_label, assumed_actual_label)
+        self.optimization_threshold = 0.05  # 5% change in accuracy triggers optimization
+        self.min_history_for_optimization = 20 # Minimum samples before optimizing
+    
+    def analyze_performance(self, ensemble_prediction_label, actual_label=None):
+        """Analyze ensemble performance and record for optimization"""
+        # If actual_label is not provided, assume ensemble is correct if not UNCERTAIN
+        assumed_actual_label = actual_label
+        if assumed_actual_label is None and ensemble_prediction_label != "UNCERTAIN":
+            assumed_actual_label = ensemble_prediction_label
+        
+        self.prediction_history.append((ensemble_prediction_label, assumed_actual_label))
+        
+        if len(self.prediction_history) >= self.min_history_for_optimization and self._should_optimize():
+            self._optimize_weights()
+    
+    def _calculate_accuracy(self, history_subset):
+        """Calculates accuracy based on history where actual_label is known."""
+        correct_predictions = 0
+        total_known = 0
+        for ensemble_pred, actual_label in history_subset:
+            if actual_label is not None:
+                total_known += 1
+                if ensemble_pred == actual_label:
+                    correct_predictions += 1
+        return correct_predictions / total_known if total_known > 0 else 0.0
+
+    def _should_optimize(self):
+        """Determine if weights should be optimized based on recent performance change."""
+        if len(self.prediction_history) < self.min_history_for_optimization * 2: # Need enough history for comparison
+            return False
+        
+        # Compare accuracy of recent batch with previous batch
+        recent_batch = self.prediction_history[-self.min_history_for_optimization:]
+        previous_batch = self.prediction_history[-self.min_history_for_optimization*2:-self.min_history_for_optimization]
+        
+        recent_accuracy = self._calculate_accuracy(recent_batch)
+        previous_accuracy = self._calculate_accuracy(previous_batch)
+        
+        # Trigger optimization if there's a significant drop in accuracy
+        if previous_accuracy > 0 and (previous_accuracy - recent_accuracy) / previous_accuracy > self.optimization_threshold:
+            logger.warning(f"Performance degradation detected (from {previous_accuracy:.2f} to {recent_accuracy:.2f}). Triggering weight optimization.")
+            return True
+        return False
+    
+    def _optimize_weights(self):
+        """Optimize model weights based on performance."""
+        logger.info("Optimizing model weights based on recent performance.")
+        # Placeholder for sophisticated optimization logic.
+        # This is where you would adjust self.weight_manager.base_weights
+        # based on which models contributed more to correct predictions or errors.
+        # For now, it's just a log message.
+
+
+class SystemHealthAgent:
+    def __init__(self):
+        self.health_metrics = {
+            "memory_usage": [],
+            "gpu_utilization": [],
+            "model_load_times": {},
+            "error_rates": {}
+        }
+    
+    def monitor_system_health(self):
+        """Monitor overall system health"""
+        self._check_memory_usage()
+        self._check_gpu_utilization()
+        # You might add _check_model_health() here later
+    
+    def _check_memory_usage(self):
+        """Monitor memory usage"""
+        try:
+            import psutil
+            memory = psutil.virtual_memory()
+            self.health_metrics["memory_usage"].append(memory.percent)
+            
+            if memory.percent > 90:
+                logger.warning(f"High memory usage detected: {memory.percent}%")
+        except ImportError:
+            logger.warning("psutil not installed. Cannot monitor memory usage.")
+    
+    def _check_gpu_utilization(self):
+        """Monitor GPU utilization if available"""
+        if torch.cuda.is_available():
+            try:
+                gpu_util = torch.cuda.memory_allocated() / torch.cuda.max_memory_allocated()
+                self.health_metrics["gpu_utilization"].append(gpu_util)
+                
+                if gpu_util > 0.9:
+                    logger.warning(f"High GPU utilization detected: {gpu_util*100:.2f}%")
+            except Exception as e:
+                logger.warning(f"Error monitoring GPU utilization: {e}")
+        else:
+            logger.info("CUDA not available. Skipping GPU utilization monitoring.") 

utils/weight_management.py

@@ -0,0 +1,107 @@
+import logging
+import torch
+
+logger = logging.getLogger(__name__)
+
+class ContextualWeightOverrideAgent:
+    def __init__(self):
+        self.context_overrides = {
+            # Example: when image is outdoor, model_X is penalized, model_Y is boosted
+            "outdoor": {
+                "model_1": 0.8, # Example: Reduce weight of model_1 by 20% for outdoor scenes
+                "model_5": 1.2, # Example: Boost weight of model_5 by 20% for outdoor scenes
+            },
+            "low_light": {
+                "model_2": 0.7, 
+                "model_7": 1.3,
+            },
+            "sunny": {
+                "model_3": 0.9,
+                "model_4": 1.1,
+            }
+            # Add more contexts and their specific model weight adjustments here
+        }
+
+    def get_overrides(self, context_tags: list[str]) -> dict:
+        """Returns combined weight overrides for given context tags."""
+        combined_overrides = {}
+        for tag in context_tags:
+            if tag in self.context_overrides:
+                for model_id, multiplier in self.context_overrides[tag].items():
+                    # If a model appears in multiple contexts, we can decide how to combine (e.g., multiply, average, take max)
+                    # For now, let's just take the last one if there are conflicts, or multiply for simple cumulative effect.
+                    combined_overrides[model_id] = combined_overrides.get(model_id, 1.0) * multiplier
+        return combined_overrides
+
+
+class ModelWeightManager:
+    def __init__(self):
+        self.base_weights = {
+            "model_1": 0.15,  # SwinV2 Based
+            "model_2": 0.15,  # ViT Based
+            "model_3": 0.15,  # SDXL Dataset
+            "model_4": 0.15,  # SDXL + FLUX
+            "model_5": 0.15,  # ViT Based
+            "model_5b": 0.10, # ViT Based, Newer Dataset
+            "model_6": 0.10,  # Swin, Midj + SDXL
+            "model_7": 0.05   # ViT
+        }
+        self.situation_weights = {
+            "high_confidence": 1.2,    # Boost weights for high confidence predictions
+            "low_confidence": 0.8,     # Reduce weights for low confidence
+            "conflict": 0.5,          # Reduce weights when models disagree
+            "consensus": 1.5          # Boost weights when models agree
+        }
+        self.context_override_agent = ContextualWeightOverrideAgent()
+    
+    def adjust_weights(self, predictions, confidence_scores, context_tags: list[str] = None):
+        """Dynamically adjust weights based on prediction patterns and optional context."""
+        adjusted_weights = self.base_weights.copy()
+
+        # 1. Apply contextual overrides first
+        if context_tags:
+            overrides = self.context_override_agent.get_overrides(context_tags)
+            for model_id, multiplier in overrides.items():
+                adjusted_weights[model_id] = adjusted_weights.get(model_id, 0.0) * multiplier
+        
+        # 2. Apply situation-based adjustments (consensus, conflict, confidence)
+        # Check for consensus
+        if self._has_consensus(predictions):
+            for model in adjusted_weights:
+                adjusted_weights[model] *= self.situation_weights["consensus"]
+        
+        # Check for conflicts
+        if self._has_conflicts(predictions):
+            for model in adjusted_weights:
+                adjusted_weights[model] *= self.situation_weights["conflict"]
+        
+        # Adjust based on confidence
+        for model, confidence in confidence_scores.items():
+            if confidence > 0.8:
+                adjusted_weights[model] *= self.situation_weights["high_confidence"]
+            elif confidence < 0.5:
+                adjusted_weights[model] *= self.situation_weights["low_confidence"]
+        
+        return self._normalize_weights(adjusted_weights)
+    
+    def _has_consensus(self, predictions):
+        """Check if models agree on prediction"""
+        # Ensure all predictions are not None before checking for consensus
+        non_none_predictions = [p for p in predictions.values() if p is not None and p != "Error"]
+        return len(non_none_predictions) > 0 and len(set(non_none_predictions)) == 1
+    
+    def _has_conflicts(self, predictions):
+        """Check if models have conflicting predictions"""
+        # Ensure all predictions are not None before checking for conflicts
+        non_none_predictions = [p for p in predictions.values() if p is not None and p != "Error"]
+        return len(non_none_predictions) > 1 and len(set(non_none_predictions)) > 1
+    
+    def _normalize_weights(self, weights):
+        """Normalize weights to sum to 1"""
+        total = sum(weights.values())
+        if total == 0:
+            # Handle case where all weights became zero due to aggressive multipliers
+            # This could assign equal weights or revert to base weights
+            logger.warning("All weights became zero after adjustments. Reverting to base weights.")
+            return {k: 1.0/len(self.base_weights) for k in self.base_weights}
+        return {k: v/total for k, v in weights.items()}