Implement hybrid AI architecture with HF Endpoint heavy lifting and Ollama local caching

src/llm/factory.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Optional
 from src.llm.base_provider import LLMProvider
+from src.llm.hybrid_provider import HybridProvider
 from src.llm.hf_provider import HuggingFaceProvider
 from src.llm.ollama_provider import OllamaProvider
 from utils.config import config
@@ -13,7 +14,7 @@ class ProviderNotAvailableError(Exception):
     pass
 
 class LLMFactory:
-    """Factory for creating LLM providers with intelligent fallback"""
+    """Factory for creating LLM providers with hybrid approach"""
     
     _instance = None
 
@@ -25,29 +26,37 @@ class LLMFactory:
     def get_provider(self, preferred_provider: Optional[str] = None) -> LLMProvider:
         """
         Get an LLM provider based on preference and availability.
-        Priority: HF Endpoint > Ollama > Error
+        Default: Hybrid approach (HF primary + Ollama backup/cache)
         """
-        # Check if HF token is available and endpoint is ready
-        if config.hf_token:
-            status = hf_monitor.get_endpoint_status()
-            if status["available"]:
+        try:
+            # Always try hybrid provider first (uses both HF and Ollama intelligently)
+            logger.info("Initializing Hybrid Provider (HF + Ollama)")
+            return HybridProvider(
+                model_name="hybrid_model"
+            )
+        except Exception as e:
+            logger.warning(f"Failed to initialize Hybrid provider: {e}")
+            
+            # Fallback to individual providers
+            if config.hf_token:
+                status = hf_monitor.get_endpoint_status()
+                if status["available"]:
+                    try:
+                        logger.info("Falling back to HF Endpoint")
+                        return HuggingFaceProvider(
+                            model_name="DavidAU/OpenAi-GPT-oss-20b-abliterated-uncensored-NEO-Imatrix-gguf"
+                        )
+                    except Exception as hf_error:
+                        logger.warning(f"Failed to initialize HF provider: {hf_error}")
+            
+            if config.ollama_host:
                 try:
-                    logger.info("Using HF Endpoint as primary provider")
-                    return HuggingFaceProvider(
-                        model_name="DavidAU/OpenAi-GPT-oss-20b-abliterated-uncensored-NEO-Imatrix-gguf"
+                    logger.info("Falling back to Ollama")
+                    return OllamaProvider(
+                        model_name=config.local_model_name
                     )
-                except Exception as e:
-                    logger.warning(f"Failed to initialize HF provider: {e}")
-        
-        # Try Ollama as fallback
-        if config.ollama_host:
-            try:
-                logger.info("Using Ollama as provider")
-                return OllamaProvider(
-                    model_name=config.local_model_name
-                )
-            except Exception as e:
-                logger.warning(f"Failed to initialize Ollama provider: {e}")
+                except Exception as ollama_error:
+                    logger.warning(f"Failed to initialize Ollama provider: {ollama_error}")
         
         raise ProviderNotAvailableError("No LLM providers are available or configured")
 

src/llm/hybrid_provider.py

@@ -0,0 +1,159 @@
+import time
+import logging
+from typing import List, Dict, Optional, Union
+from src.llm.base_provider import LLMProvider
+from src.llm.hf_provider import HuggingFaceProvider
+from src.llm.ollama_provider import OllamaProvider
+from core.session import session_manager
+from utils.config import config
+
+logger = logging.getLogger(__name__)
+
+class HybridProvider(LLMProvider):
+    """Hybrid provider that uses HF for heavy lifting and Ollama for local caching/summarization"""
+    
+    def __init__(self, model_name: str, timeout: int = 120, max_retries: int = 2):
+        super().__init__(model_name, timeout, max_retries)
+        self.hf_provider = None
+        self.ollama_provider = None
+        
+        # Initialize providers
+        try:
+            if config.hf_token:
+                self.hf_provider = HuggingFaceProvider(
+                    model_name="DavidAU/OpenAi-GPT-oss-20b-abliterated-uncensored-NEO-Imatrix-gguf",
+                    timeout=120
+                )
+        except Exception as e:
+            logger.warning(f"Failed to initialize HF provider: {e}")
+            
+        try:
+            if config.ollama_host:
+                self.ollama_provider = OllamaProvider(
+                    model_name=config.local_model_name,
+                    timeout=60
+                )
+        except Exception as e:
+            logger.warning(f"Failed to initialize Ollama provider: {e}")
+
+    def generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[str]:
+        """Generate response using hybrid approach"""
+        try:
+            # Step 1: Get heavy lifting from HF Endpoint
+            hf_response = self._get_hf_response(prompt, conversation_history)
+            
+            if not hf_response:
+                raise Exception("HF Endpoint failed to provide response")
+            
+            # Step 2: Store HF response in local cache via Ollama
+            self._cache_response_locally(prompt, hf_response, conversation_history)
+            
+            # Step 3: Optionally create local summary (if needed)
+            # For now, return HF response directly but with local backup
+            return hf_response
+            
+        except Exception as e:
+            logger.error(f"Hybrid generation failed: {e}")
+            
+            # Fallback to Ollama if available
+            if self.ollama_provider:
+                try:
+                    logger.info("Falling back to Ollama for local response")
+                    return self.ollama_provider.generate(prompt, conversation_history)
+                except Exception as fallback_error:
+                    logger.error(f"Ollama fallback also failed: {fallback_error}")
+            
+            raise Exception(f"Both HF Endpoint and Ollama failed: {str(e)}")
+
+    def stream_generate(self, prompt: str, conversation_history: List[Dict]) -> Optional[Union[str, List[str]]]:
+        """Stream response using hybrid approach"""
+        try:
+            # Get streaming response from HF
+            if self.hf_provider:
+                return self.hf_provider.stream_generate(prompt, conversation_history)
+            elif self.ollama_provider:
+                return self.ollama_provider.stream_generate(prompt, conversation_history)
+            else:
+                raise Exception("No providers available")
+        except Exception as e:
+            logger.error(f"Hybrid stream generation failed: {e}")
+            raise
+
+    def _get_hf_response(self, prompt: str, conversation_history: List[Dict]) -> Optional[str]:
+        """Get response from HF Endpoint with fallback handling"""
+        if not self.hf_provider:
+            return None
+            
+        try:
+            logger.info("🚀 Getting detailed response from HF Endpoint...")
+            response = self.hf_provider.generate(prompt, conversation_history)
+            logger.info("✅ HF Endpoint response received")
+            return response
+        except Exception as e:
+            logger.error(f"HF Endpoint failed: {e}")
+            # Don't raise here, let hybrid provider handle fallback
+            return None
+
+    def _cache_response_locally(self, prompt: str, response: str, conversation_history: List[Dict]):
+        """Cache HF response locally using Ollama for future reference"""
+        if not self.ollama_provider:
+            return
+            
+        try:
+            # Create a simplified cache entry
+            cache_prompt = f"Cache this response for future reference:\n\nQuestion: {prompt}\n\nResponse: {response[:500]}..."
+            
+            # Store in local Ollama for quick retrieval
+            # This helps if HF connection fails later
+            logger.info("💾 Caching response locally with Ollama...")
+            self.ollama_provider.generate(cache_prompt, [])
+            
+            # Also store in Redis session for persistence
+            self._store_in_session_cache(prompt, response)
+            
+        except Exception as e:
+            logger.warning(f"Failed to cache response locally: {e}")
+
+    def _store_in_session_cache(self, prompt: str, response: str):
+        """Store response in Redis session cache"""
+        try:
+            user_session = session_manager.get_session("default_user")
+            cache = user_session.get("response_cache", {})
+            
+            # Simple cache key
+            cache_key = hash(prompt) % 1000000
+            cache[str(cache_key)] = {
+                "prompt": prompt,
+                "response": response,
+                "timestamp": time.time()
+            }
+            
+            # Keep only last 50 cached responses
+            if len(cache) > 50:
+                # Remove oldest entries
+                sorted_keys = sorted(cache.keys(), key=lambda k: cache[k]["timestamp"])
+                for key in sorted_keys[:-50]:
+                    del cache[key]
+            
+            user_session["response_cache"] = cache
+            session_manager.update_session("default_user", user_session)
+            
+        except Exception as e:
+            logger.warning(f"Failed to store in session cache: {e}")
+
+    def get_cached_response(self, prompt: str) -> Optional[str]:
+        """Get cached response if available"""
+        try:
+            user_session = session_manager.get_session("default_user")
+            cache = user_session.get("response_cache", {})
+            
+            cache_key = str(hash(prompt) % 1000000)
+            if cache_key in cache:
+                cached_entry = cache[cache_key]
+                # Check if cache is still valid (1 hour)
+                if time.time() - cached_entry["timestamp"] < 3600:
+                    return cached_entry["response"]
+        except Exception as e:
+            logger.warning(f"Failed to retrieve cached response: {e}")
+        
+        return None

src/ui/chat_handler.py

@@ -4,12 +4,11 @@ import logging
 from typing import Optional
 from src.llm.factory import llm_factory, ProviderNotAvailableError
 from core.session import session_manager
-from utils.config import config
 
 logger = logging.getLogger(__name__)
 
 class ChatHandler:
-    """Handles chat interactions with better UI feedback"""
+    """Handles chat interactions with hybrid AI approach"""
     
     def __init__(self):
         self.is_processing = False
@@ -54,7 +53,7 @@ class ChatHandler:
             st.session_state.last_processed_message = ""
     
     def process_ai_response(self, user_input: str, selected_model: str):
-        """Process AI response after user message is displayed"""
+        """Process AI response with hybrid approach"""
         if not user_input or not user_input.strip():
             return
             
@@ -65,14 +64,25 @@ class ChatHandler:
                 response_placeholder = st.empty()
                 
                 try:
-                    # Determine provider based on selection and availability
-                    provider_name = self._get_best_provider(selected_model)
-                    status_placeholder.info(f"🚀 Contacting {self._get_provider_display_name(provider_name)}...")
+                    # Get hybrid provider
+                    status_placeholder.info("🚀 Contacting AI providers...")
+                    provider = llm_factory.get_provider()
                     
-                    # Get response with timeout handling
+                    # Show which approach is being used
+                    if hasattr(provider, 'hf_provider') and provider.hf_provider:
+                        status_placeholder.info("🧠 Getting detailed response from HF Endpoint...")
+                    else:
+                        status_placeholder.info("🦙 Getting response from local Ollama...")
+                    
+                    # Get response
                     response = None
                     try:
-                        response = self._get_ai_response(user_input, provider_name)
+                        # Get session and conversation history
+                        user_session = session_manager.get_session("default_user")
+                        conversation_history = user_session.get("conversation", []).copy()
+                        conversation_history.append({"role": "user", "content": user_input})
+                        
+                        response = provider.generate(user_input, conversation_history)
                     except Exception as e:
                         logger.error(f"AI response error: {e}")
                         raise
@@ -81,13 +91,19 @@ class ChatHandler:
                         status_placeholder.success("✅ Response received!")
                         response_placeholder.markdown(response)
                         
-                        # Add to session history
+                        # Add to session history with provider info
                         timestamp = time.strftime("%H:%M:%S")
+                        provider_info = "hybrid"
+                        if hasattr(provider, 'hf_provider') and provider.hf_provider:
+                            provider_info = "hf_endpoint"
+                        elif hasattr(provider, 'ollama_provider') and provider.ollama_provider:
+                            provider_info = "ollama"
+                            
                         st.session_state.messages.append({
                             "role": "assistant",
                             "content": response,
                             "timestamp": timestamp,
-                            "provider": provider_name
+                            "provider": provider_info
                         })
                     else:
                         status_placeholder.warning("⚠️ Empty response received")
@@ -97,7 +113,7 @@ class ChatHandler:
                             "role": "assistant",
                             "content": "*No response generated. Please try again.*",
                             "timestamp": timestamp,
-                            "provider": provider_name
+                            "provider": "unknown"
                         })
                         
                 except ProviderNotAvailableError as e:
@@ -112,24 +128,15 @@ class ChatHandler:
                     logger.error(f"Provider not available: {e}")
                     
                 except Exception as e:
-                    # Better user-friendly error messages
                     status_placeholder.error("❌ Request failed")
                     
-                    # More specific error messages
+                    # User-friendly error messages
                     if "timeout" in str(e).lower() or "500" in str(e):
-                        error_message = ("⏰ Request failed. This might be because:\n"
-                                       "• The AI model is taking too long to respond\n"
-                                       "• The provider is overloaded\n\n"
-                                       "**Try one of these solutions:**\n"
-                                       "1. Use the HF Endpoint (🟢 HF Endpoint: Available and ready)\n"
-                                       "2. Wait a moment and try again\n"
-                                       "3. Simplify your question")
-                    elif "connection" in str(e).lower():
-                        error_message = ("🔌 Connection failed. This might be because:\n"
-                                       "• Your Ollama server is offline\n"
-                                       "• Incorrect Ollama URL\n"
-                                       "• Network firewall blocking connection\n"
-                                       "• Try using the HF Endpoint instead")
+                        error_message = ("⏰ Request timed out. The AI model is taking too long to respond.\n\n"
+                                       "**Current setup:**\n"
+                                       "• 🤖 HF Endpoint: Doing heavy lifting\n"
+                                       "• 🦙 Ollama: Providing local backup\n\n"
+                                       "Please try again or simplify your question.")
                     else:
                         error_message = f"Sorry, I encountered an error: {str(e)}"
                     
@@ -151,65 +158,14 @@ class ChatHandler:
             st.session_state.last_processed_message = ""
             time.sleep(0.1)
     
-    def _get_best_provider(self, selected_model: str) -> str:
-        """Determine the best available provider"""
-        from src.services.hf_monitor import hf_monitor
-        
-        # If user selected specific provider, try that
-        if selected_model == "ollama" and config.ollama_host:
-            return "ollama"
-        elif selected_model == "huggingface" and config.hf_token:
-            return "huggingface"
-        
-        # Auto-select based on availability
-        if config.hf_token:
-            status = hf_monitor.get_endpoint_status()
-            if status["available"]:
-                return "huggingface"
-        
-        if config.ollama_host:
-            return "ollama"
-            
-        return "ollama"  # Default fallback
-    
     def _get_provider_display_name(self, provider_name: str) -> str:
         """Get display name for provider"""
         display_names = {
-            "ollama": "🦙 Ollama",
-            "huggingface": "🤗 HF Endpoint"
+            "ollama": "🦙 Ollama (Local)",
+            "hf_endpoint": "🤗 HF Endpoint (Heavy Lifting)",
+            "hybrid": "🔄 Hybrid (HF + Ollama)"
         }
         return display_names.get(provider_name, provider_name)
-    
-    def _get_ai_response(self, user_input: str, provider_name: str) -> Optional[str]:
-        """Get AI response from specified provider"""
-        try:
-            # Get session and conversation history
-            user_session = session_manager.get_session("default_user")
-            conversation_history = user_session.get("conversation", []).copy()
-            
-            # Add current user message
-            conversation_history.append({"role": "user", "content": user_input})
-            
-            # Get provider (with intelligent fallback)
-            provider = llm_factory.get_provider(provider_name)
-            
-            # Generate response with timeout
-            logger.info(f"Generating response with {provider_name} provider")
-            response = provider.generate(user_input, conversation_history)
-            logger.info(f"Received response from {provider_name}: {response[:100] if response else 'None'}")
-            
-            # Update session with conversation
-            if response:
-                conversation = user_session.get("conversation", []).copy()
-                conversation.append({"role": "user", "content": user_input})
-                conversation.append({"role": "assistant", "content": response})
-                session_manager.update_session("default_user", {"conversation": conversation})
-            
-            return response
-            
-        except Exception as e:
-            logger.error(f"AI response generation failed: {e}", exc_info=True)
-            raise
 
 # Global instance
 chat_handler = ChatHandler()