Implement circuit breaker pattern and enhanced fallback logic

src/llm/base_provider.py

@@ -1,13 +1,23 @@
+import time
+import logging
 from abc import ABC, abstractmethod
 from typing import List, Dict, Optional, Union
 
+logger = logging.getLogger(__name__)
+
 class LLMProvider(ABC):
-    """Abstract base class for all LLM providers"""
+    """Abstract base class for all LLM providers with circuit breaker"""
     
     def __init__(self, model_name: str, timeout: int = 30, max_retries: int = 3):
         self.model_name = model_name
         self.timeout = timeout
         self.max_retries = max_retries
+        
+        # Circuit breaker properties
+        self.failure_count = 0
+        self.last_failure_time = None
+        self.circuit_open = False
+        self.reset_timeout = 60  # Reset circuit after 60 seconds
 
     @abstractmethod
     def generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[str]:
@@ -18,3 +28,58 @@ class LLMProvider(ABC):
     def stream_generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[Union[str, List[str]]]:
         """Generate a response with streaming support"""
         pass
+
+    def _check_circuit_breaker(self) -> bool:
+        """Check if circuit breaker is open (preventing calls)"""
+        if not self.circuit_open:
+            return True
+            
+        # Check if enough time has passed to reset
+        if self.last_failure_time and (time.time() - self.last_failure_time) > self.reset_timeout:
+            logger.info("Circuit breaker reset - allowing call")
+            self.circuit_open = False
+            self.failure_count = 0
+            return True
+            
+        logger.warning("Circuit breaker is OPEN - preventing call")
+        return False
+
+    def _handle_failure(self, error: Exception):
+        """Handle failure and update circuit breaker"""
+        self.failure_count += 1
+        self.last_failure_time = time.time()
+        
+        # Open circuit after 3 failures
+        if self.failure_count >= 3:
+            self.circuit_open = True
+            logger.warning(f"Circuit breaker OPEN for {self.__class__.__name__} after {self.failure_count} failures")
+        
+        raise error
+
+    def _retry_with_backoff(self, func, *args, **kwargs):
+        """Retry logic with exponential backoff"""
+        last_exception = None
+        
+        for attempt in range(self.max_retries):
+            try:
+                if not self._check_circuit_breaker():
+                    raise Exception("Circuit breaker is open")
+                    
+                result = func(*args, **kwargs)
+                # Reset failure count on success
+                self.failure_count = 0
+                self.circuit_open = False
+                return result
+                
+            except Exception as e:
+                last_exception = e
+                self._handle_failure(e)
+                
+                if attempt < self.max_retries - 1:
+                    sleep_time = min((2 ** attempt) * 1.0, 10.0)  # Cap at 10 seconds
+                    logger.warning(f"Attempt {attempt + 1} failed: {str(e)}. Retrying in {sleep_time}s...")
+                    time.sleep(sleep_time)
+                else:
+                    logger.error(f"All {self.max_retries} attempts failed. Last error: {str(e)}")
+        
+        raise last_exception

src/llm/factory.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Optional
+from typing import Optional, List
 from src.llm.base_provider import LLMProvider
 from src.llm.hf_provider import HuggingFaceProvider
 from src.llm.ollama_provider import OllamaProvider
@@ -12,7 +12,7 @@ class ProviderNotAvailableError(Exception):
     pass
 
 class LLMFactory:
-    """Factory for creating LLM providers with fallback support"""
+    """Factory for creating LLM providers with intelligent fallback"""
     
     _instance = None
 
@@ -34,44 +34,77 @@ class LLMFactory:
         Raises:
             ProviderNotAvailableError: When no providers are available
         """
-        # Check preferred provider first
-        if preferred_provider == "huggingface" and config.hf_token:
-            try:
-                return HuggingFaceProvider(
-                    model_name="DavidAU/OpenAi-GPT-oss-20b-abliterated-uncensored-NEO-Imatrix-gguf"
-                )
-            except Exception as e:
-                logger.warning(f"Failed to initialize HF provider: {e}")
+        # Build provider chain based on configuration and preference
+        provider_chain = self._build_provider_chain(preferred_provider)
         
-        elif preferred_provider == "ollama" and config.ollama_host:
+        # Try providers in order
+        for provider_name, provider_class, model_name in provider_chain:
             try:
-                return OllamaProvider(
-                    model_name=config.local_model_name
-                )
+                logger.info(f"Attempting to initialize {provider_name} provider...")
+                provider = provider_class(model_name=model_name)
+                # Test that provider is working
+                if self._test_provider(provider):
+                    logger.info(f"Successfully initialized {provider_name} provider")
+                    return provider
+                else:
+                    logger.warning(f"{provider_name} provider failed validation test")
             except Exception as e:
-                logger.warning(f"Failed to initialize Ollama provider: {e}")
+                logger.warning(f"Failed to initialize {provider_name} provider: {e}")
+                continue
+        
+        raise ProviderNotAvailableError("No LLM providers are available or configured")
 
-        # Fallback logic based on configuration
+    def _build_provider_chain(self, preferred_provider: Optional[str]) -> List[tuple]:
+        """Build provider chain based on preference and configuration"""
+        chain = []
+        
+        # Add preferred provider first if specified
+        if preferred_provider:
+            provider_info = self._get_provider_info(preferred_provider)
+            if provider_info:
+                chain.append(provider_info)
+        
+        # Add fallback providers based on configuration
         if config.use_fallback:
-            # Try HF first if configured
+            # Add HF if configured
             if config.hf_token:
-                try:
-                    return HuggingFaceProvider(
-                        model_name="DavidAU/OpenAi-GPT-oss-20b-abliterated-uncensored-NEO-Imatrix-gguf"
-                    )
-                except Exception as e:
-                    logger.warning(f"Failed to initialize HF provider: {e}")
+                chain.append((
+                    "huggingface",
+                    HuggingFaceProvider,
+                    "DavidAU/OpenAi-GPT-oss-20b-abliterated-uncensored-NEO-Imatrix-gguf"
+                ))
             
-            # Then try Ollama if configured
+            # Add Ollama if configured
             if config.ollama_host:
-                try:
-                    return OllamaProvider(
-                        model_name=config.local_model_name
-                    )
-                except Exception as e:
-                    logger.warning(f"Failed to initialize Ollama provider: {e}")
+                chain.append((
+                    "ollama",
+                    OllamaProvider,
+                    config.local_model_name
+                ))
+        
+        return chain
 
-        raise ProviderNotAvailableError("No LLM providers are available or configured")
+    def _get_provider_info(self, provider_name: str) -> Optional[tuple]:
+        """Get provider class and model info"""
+        provider_map = {
+            "huggingface": (
+                "huggingface",
+                HuggingFaceProvider,
+                "DavidAU/OpenAi-GPT-oss-20b-abliterated-uncensored-NEO-Imatrix-gguf"
+            ),
+            "ollama": (
+                "ollama",
+                OllamaProvider,
+                config.local_model_name
+            )
+        }
+        return provider_map.get(provider_name)
+
+    def _test_provider(self, provider: LLMProvider) -> bool:
+        """Test if provider is working (stub implementation)"""
+        # In a real implementation, you might want to do a lightweight test
+        # For now, we'll assume initialization success means it's working
+        return True
 
 # Global factory instance
 llm_factory = LLMFactory()

src/llm/hf_provider.py

@@ -34,6 +34,14 @@ class HuggingFaceProvider(LLMProvider):
 
     def generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[str]:
         """Generate a response synchronously"""
+        return self._retry_with_backoff(self._generate_impl, prompt, conversation_history)
+
+    def stream_generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[Union[str, List[str]]]:
+        """Generate a response with streaming support"""
+        return self._retry_with_backoff(self._stream_generate_impl, prompt, conversation_history)
+
+    def _generate_impl(self, prompt: str, conversation_history: List[Dict]) -> str:
+        """Implementation of synchronous generation"""
         try:
             response = self.client.chat.completions.create(
                 model=self.model_name,
@@ -44,9 +52,8 @@ class HuggingFaceProvider(LLMProvider):
             )
             return response.choices[0].message.content
         except Exception as e:
-            logger.error(f"HF generation failed: {e}")
             # Handle scale-to-zero behavior
-            if "503" in str(e) or "service unavailable" in str(e).lower():
+            if self._is_scale_to_zero_error(e):
                 logger.info("HF endpoint is scaling up, waiting...")
                 time.sleep(60)  # Wait for endpoint to initialize
                 # Retry once
@@ -60,8 +67,8 @@ class HuggingFaceProvider(LLMProvider):
                 return response.choices[0].message.content
             raise
 
-    def stream_generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[Union[str, List[str]]]:
-        """Generate a response with streaming support"""
+    def _stream_generate_impl(self, prompt: str, conversation_history: List[Dict]) -> List[str]:
+        """Implementation of streaming generation"""
         try:
             response = self.client.chat.completions.create(
                 model=self.model_name,
@@ -78,9 +85,8 @@ class HuggingFaceProvider(LLMProvider):
                     chunks.append(content)
             return chunks
         except Exception as e:
-            logger.error(f"HF stream generation failed: {e}")
             # Handle scale-to-zero behavior
-            if "503" in str(e) or "service unavailable" in str(e).lower():
+            if self._is_scale_to_zero_error(e):
                 logger.info("HF endpoint is scaling up, waiting...")
                 time.sleep(60)  # Wait for endpoint to initialize
                 # Retry once
@@ -99,3 +105,14 @@ class HuggingFaceProvider(LLMProvider):
                         chunks.append(content)
                 return chunks
             raise
+
+    def _is_scale_to_zero_error(self, error: Exception) -> bool:
+        """Check if the error is related to scale-to-zero initialization"""
+        error_str = str(error).lower()
+        scale_to_zero_indicators = [
+            "503",
+            "service unavailable",
+            "initializing",
+            "cold start"
+        ]
+        return any(indicator in error_str for indicator in scale_to_zero_indicators)

src/llm/ollama_provider.py

@@ -30,6 +30,14 @@ class OllamaProvider(LLMProvider):
 
     def generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[str]:
         """Generate a response synchronously"""
+        return self._retry_with_backoff(self._generate_impl, prompt, conversation_history)
+
+    def stream_generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[Union[str, List[str]]]:
+        """Generate a response with streaming support"""
+        return self._retry_with_backoff(self._stream_generate_impl, prompt, conversation_history)
+
+    def _generate_impl(self, prompt: str, conversation_history: List[Dict]) -> str:
+        """Implementation of synchronous generation"""
         try:
             url = f"{self.host}/api/chat"
             payload = {
@@ -51,8 +59,8 @@ class OllamaProvider(LLMProvider):
             logger.error(f"Ollama generation failed: {e}")
             raise
 
-    def stream_generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[Union[str, List[str]]]:
-        """Generate a response with streaming support"""
+    def _stream_generate_impl(self, prompt: str, conversation_history: List[Dict]) -> List[str]:
+        """Implementation of streaming generation"""
         try:
             url = f"{self.host}/api/chat"
             payload = {