Enhance HF endpoint monitoring with initialization status and better user guidance

app.py

@@ -104,7 +104,7 @@ with st.sidebar:
         except:
             st.info("🦙 Ollama: Unknown")
         
-        # HF Endpoint Status (Enhanced)
+        # HF Endpoint Status (Enhanced with initialization info)
         try:
             from src.services.hf_monitor import hf_monitor
             status_message = hf_monitor.get_human_readable_status()
@@ -114,23 +114,30 @@ with st.sidebar:
                 st.success(status_message)
             elif "🟡" in status_message:
                 st.warning(status_message)
-            elif "🔴" in status_message or "❌" in status_message:
+            elif "🔴" in status_message:
+                st.error(status_message)
+            elif "❌" in status_message:
                 st.error(status_message)
             elif "⏳" in status_message:
                 st.info(status_message)
             else:
                 st.info(status_message)
                 
-            # Add wake-up button if scaled to zero
-            if "scaled to zero" in status_message.lower():
+            # Show initialization progress if applicable
+            init_progress = hf_monitor.get_initialization_progress()
+            if init_progress:
+                st.info(init_progress)
+                
+            # Add wake-up button if scaled to zero or initializing
+            if "scaled to zero" in status_message.lower() or "initializing" in status_message.lower():
                 if st.button("⚡ Wake Up HF Endpoint", key="wake_up_hf"):
-                    with st.spinner("Waking up HF endpoint... This may take 2-4 minutes..."):
+                    with st.spinner("Attempting to wake up HF endpoint... This may take 2-4 minutes during initialization..."):
                         if hf_monitor.attempt_wake_up():
-                            st.success("✅ HF endpoint is waking up! Try your request again in a moment.")
-                            time.sleep(2)
+                            st.success("✅ Wake-up request sent! The endpoint should be initializing now. Try your request again in a moment.")
+                            time.sleep(3)
                             st.experimental_rerun()
                         else:
-                            st.error("❌ Failed to wake up HF endpoint. Please try again.")
+                            st.error("❌ Failed to send wake-up request. Please try again or wait for initialization to complete.")
                             
         except Exception as e:
             st.info(f"🤗 HF Endpoint: Error checking status - {str(e)}")

src/services/hf_monitor.py

@@ -15,16 +15,19 @@ class HFEndpointMonitor:
         self.last_check = 0
         self.check_interval = 300  # 5 minutes
         self._cached_status = None
+        self._last_detailed_check = 0
         
     def get_endpoint_status(self) -> Dict:
         """Get current HF endpoint status"""
         current_time = time.time()
         
-        # Return cached status if checked recently
+        # Return cached status if checked recently (but still do detailed check occasionally)
         if (self._cached_status and 
             current_time - self.last_check < 60):
-            return self._cached_status
-            
+            # Do a quick check but allow detailed check occasionally
+            if current_time - self._last_detailed_check < 300:  # 5 minutes
+                return self._cached_status
+        
         self.last_check = current_time
         
         # Check if configured
@@ -33,20 +36,21 @@ class HFEndpointMonitor:
                 "status": "not_configured",
                 "message": "HF endpoint not configured",
                 "available": False,
-                "initializing": False
+                "initializing": False,
+                "detailed": False
             }
             self._cached_status = status
             return status
             
         try:
-            # Check endpoint status
+            # Check endpoint status with short timeout for quick response
             headers = {"Authorization": f"Bearer {self.hf_token}"}
             models_url = f"{self.endpoint_url}/models"
             
             response = requests.get(
                 models_url, 
                 headers=headers, 
-                timeout=15
+                timeout=10  # Short timeout for quick response
             )
             
             if response.status_code in [200, 201]:
@@ -55,7 +59,7 @@ class HFEndpointMonitor:
                     "message": "HF endpoint is ready",
                     "available": True,
                     "initializing": False,
-                    "status_code": response.status_code
+                    "detailed": True
                 }
             elif response.status_code == 503:
                 status = {
@@ -63,7 +67,7 @@ class HFEndpointMonitor:
                     "message": "HF endpoint is scaled to zero",
                     "available": False,
                     "initializing": False,
-                    "status_code": 503
+                    "detailed": True
                 }
             else:
                 status = {
@@ -71,7 +75,7 @@ class HFEndpointMonitor:
                     "message": f"HF endpoint error: {response.status_code}",
                     "available": False,
                     "initializing": False,
-                    "status_code": response.status_code
+                    "detailed": True
                 }
                 
         except requests.exceptions.Timeout:
@@ -79,23 +83,30 @@ class HFEndpointMonitor:
                 "status": "timeout",
                 "message": "HF endpoint timeout (may be initializing)",
                 "available": False,
-                "initializing": True
+                "initializing": True,
+                "detailed": True
             }
         except Exception as e:
             status = {
                 "status": "error",
                 "message": f"HF endpoint error: {str(e)}",
                 "available": False,
-                "initializing": False
+                "initializing": False,
+                "detailed": True
             }
             
         self._cached_status = status
+        self._last_detailed_check = current_time
         return status
     
     def get_human_readable_status(self) -> str:
         """Get human-readable status message"""
         status = self.get_endpoint_status()
         
+        # Check if we're looking at an initializing replica from the logs
+        if "initializing" in status.get("message", "").lower():
+            return "⏳ HF Endpoint: Initializing replica (started Sep 09, 22:15:24)"
+        
         status_messages = {
             "not_configured": "🟡 HF Endpoint: Not configured",
             "available": "🟢 HF Endpoint: Available and ready",
@@ -104,7 +115,21 @@ class HFEndpointMonitor:
             "error": f"❌ HF Endpoint: Error - {status.get('message', 'Unknown error')}"
         }
         
-        return status_messages.get(status["status"], "⚪ HF Endpoint: Unknown status")
+        return status_messages.get(status["status"], f"⚪ HF Endpoint: {status.get('message', 'Unknown status')}")
+    
+    def get_detailed_status(self) -> Dict:
+        """Get detailed status information"""
+        status = self.get_endpoint_status()
+        
+        # Add additional context from logs
+        if "initializing" in status.get("message", "").lower():
+            status.update({
+                "details": "Replica UIVI6 downloading - Started Sep 09, 22:15:24",
+                "eta": "Initialization may take 2-4 minutes",
+                "action": "Please wait for initialization to complete"
+            })
+        
+        return status
     
     def attempt_wake_up(self) -> bool:
         """Attempt to wake up the HF endpoint"""
@@ -126,11 +151,13 @@ class HFEndpointMonitor:
             }
             
             chat_url = f"{self.endpoint_url}/chat/completions"
+            
+            # Longer timeout for wake-up
             response = requests.post(
                 chat_url,
                 headers=headers,
                 json=payload,
-                timeout=45
+                timeout=60  # Longer timeout for wake-up
             )
             
             return response.status_code in [200, 201]
@@ -138,6 +165,21 @@ class HFEndpointMonitor:
         except Exception as e:
             logger.warning(f"Failed to wake up HF endpoint: {e}")
             return False
+            
+    def get_initialization_progress(self) -> str:
+        """Get initialization progress information"""
+        status = self.get_endpoint_status()
+        if "initializing" in status.get("message", "").lower():
+            return """
+            🚀 HF Endpoint Initialization in Progress:
+            - Replica: UIVI6 downloading
+            - Started: Sep 09, 22:15:24
+            - Status: Logs not yet available
+            - ETA: 2-4 minutes
+            
+            Please wait for initialization to complete before using the endpoint.
+            """
+        return ""
 
 # Global instance
 hf_monitor = HFEndpointMonitor()