add 4 new models

README.md

@@ -8,28 +8,32 @@ sdk_version: 1.44.1
 app_file: app.py
 pinned: false
 license: apache-2.0
-short_description: Run GGUF models (Qwen2.5, Gemma-3, Phi-4) with llama.cpp
+short_description: Run GGUF models (Qwen2.5, Gemma-3, Phi-4, Meta-Llama-3.1, DeepSeek-R1-Distill-Llama, Mistral-7B, Qwen2.5-Coder) with llama.cpp
 ---
 
-This Streamlit app lets you run **chat-based inference** on different GGUF models with `llama.cpp` and `llama-cpp-python`. 
+This Streamlit app enables **chat-based inference** on various GGUF models using `llama.cpp` and `llama-cpp-python`.
 
 ### 🔄 Supported Models:
 - `Qwen/Qwen2.5-7B-Instruct-GGUF` → `qwen2.5-7b-instruct-q2_k.gguf`
-- `unsloth/gemma-3-4b-it-GGUF` → `gemma-3-4b-it-Q5_K_M.gguf`
-- `unsloth/Phi-4-mini-instruct-GGUF` → `Phi-4-mini-instruct-Q5_K_M.gguf`
+- `unsloth/gemma-3-4b-it-GGUF` → `gemma-3-4b-it-Q4_K_M.gguf`
+- `unsloth/Phi-4-mini-instruct-GGUF` → `Phi-4-mini-instruct-Q4_K_M.gguf`
+- `MaziyarPanahi/Meta-Llama-3.1-8B-Instruct-GGUF` → `Meta-Llama-3.1-8B-Instruct.Q2_K.gguf`
+- `unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF` → `DeepSeek-R1-Distill-Llama-8B-Q2_K.gguf`
+- `MaziyarPanahi/Mistral-7B-Instruct-v0.3-GGUF` → `Mistral-7B-Instruct-v0.3.IQ3_XS.gguf`
+- `Qwen/Qwen2.5-Coder-7B-Instruct-GGUF` → `qwen2.5-coder-7b-instruct-q2_k.gguf`
 
 ### ⚙️ Features:
-- Model selection in sidebar
-- Custom system prompt and generation parameters
+- Model selection in the sidebar
+- Customizable system prompt and generation parameters
 - Chat-style UI with streaming responses
 
 ### 🧠 Memory-Safe Design (for HuggingFace Spaces):
-- Only **one model is loaded at a time** (no persistent memory bloat)
-- Uses **manual unloading and `gc.collect()`** to free memory when switching
-- Reduces `n_ctx` context length to stay under 16 GB RAM limit
-- Automatically downloads models only when needed
-- Trims history to the **last 8 user-assistant turns** to avoid context overflow
+- Loads only **one model at a time** to prevent memory bloat
+- Utilizes **manual unloading and `gc.collect()`** to free memory when switching models
+- Adjusts `n_ctx` context length to operate within a 16 GB RAM limit
+- Automatically downloads models as needed
+- Limits history to the **last 8 user-assistant turns** to prevent context overflow
 
-Perfect for deploying multi-GGUF chat models on **free-tier HuggingFace Spaces**!
+Ideal for deploying multiple GGUF chat models on **free-tier HuggingFace Spaces**!
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Refer to the configuration guide at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -3,8 +3,6 @@ from llama_cpp import Llama
 from huggingface_hub import hf_hub_download
 import os
 import gc
-import shutil
-import subprocess
 
 # Available models
 MODELS = {
@@ -23,8 +21,29 @@ MODELS = {
         "filename": "Phi-4-mini-instruct-Q4_K_M.gguf",
         "description": "Phi-4 Mini Instruct (Q4_K_M)"
     },
+    "Meta-Llama-3.1-8B-Instruct (Q2_K)": {
+        "repo_id": "MaziyarPanahi/Meta-Llama-3.1-8B-Instruct-GGUF",
+        "filename": "Meta-Llama-3.1-8B-Instruct.Q2_K.gguf",
+        "description": "Meta Llama 3.1 8B Instruct (Q2_K)"
+    },
+    "DeepSeek-R1-Distill-Llama-8B (Q2_K)": {
+        "repo_id": "unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF",
+        "filename": "DeepSeek-R1-Distill-Llama-8B-Q2_K.gguf",
+        "description": "DeepSeek R1 Distill Llama 8B (Q2_K)"
+    },
+    "Mistral-7B-Instruct-v0.3 (IQ3_XS)": {
+        "repo_id": "MaziyarPanahi/Mistral-7B-Instruct-v0.3-GGUF",
+        "filename": "Mistral-7B-Instruct-v0.3.IQ3_XS.gguf",
+        "description": "Mistral 7B Instruct v0.3 (IQ3_XS)"
+    },
+    "Qwen2.5-Coder-7B-Instruct (Q2_K)": {
+        "repo_id": "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF",
+        "filename": "qwen2.5-coder-7b-instruct-q2_k.gguf",
+        "description": "Qwen2.5 Coder 7B Instruct (Q2_K)"
+    },
 }
 
+# Sidebar for model selection and settings
 with st.sidebar:
     st.header("⚙️ Settings")
     selected_model_name = st.selectbox("Select Model", list(MODELS.keys()))
@@ -35,37 +54,14 @@ with st.sidebar:
     top_p = st.slider("Top-P", 0.1, 1.0, 0.95)
     repeat_penalty = st.slider("Repetition Penalty", 1.0, 2.0, 1.1)
 
-    if st.button("🧹 Clear All Cached Models"):
-        try:
-            for f in os.listdir("models"):
-                if f.endswith(".gguf"):
-                    os.remove(os.path.join("models", f))
-            st.success("Model cache cleared.")
-        except Exception as e:
-            st.error(f"Failed to clear models: {e}")
-
-    if st.button("📦 Show Disk Usage"):
-        try:
-            usage = shutil.disk_usage(".")
-            used = usage.used / (1024**3)
-            free = usage.free / (1024**3)
-            st.info(f"Disk Used: {used:.2f} GB | Free: {free:.2f} GB")
-        except Exception as e:
-            st.error(f"Disk usage error: {e}")
-
 # Model info
 selected_model = MODELS[selected_model_name]
 model_path = os.path.join("models", selected_model["filename"])
 
-# Init state
-if "model_name" not in st.session_state:
-    st.session_state.model_name = None
-if "llm" not in st.session_state:
-    st.session_state.llm = None
-
 # Ensure model directory exists
 os.makedirs("models", exist_ok=True)
 
+# Function to clean up old models
 def cleanup_old_models():
     for f in os.listdir("models"):
         if f.endswith(".gguf") and f != selected_model["filename"]:
@@ -74,6 +70,7 @@ def cleanup_old_models():
             except Exception as e:
                 st.warning(f"Couldn't delete old model {f}: {e}")
 
+# Function to download the selected model
 def download_model():
     with st.spinner(f"Downloading {selected_model['filename']}..."):
         hf_hub_download(
@@ -83,86 +80,26 @@ def download_model():
             local_dir_use_symlinks=False,
         )
 
-def try_load_model(path):
-    try:
-        return Llama(model_path=path, n_ctx=1024, n_threads=2, n_threads_batch=2, n_batch=4, n_gpu_layers=0, use_mlock=False, use_mmap=True, verbose=False)
-    except Exception as e:
-        return str(e)
-
+# Function to validate or download the model
 def validate_or_download_model():
     if not os.path.exists(model_path):
         cleanup_old_models()
         download_model()
-
-    # First load attempt
-    result = try_load_model(model_path)
-    if isinstance(result, str):
-        st.warning(f"Initial load failed: {result}\nAttempting re-download...")
+    try:
+        # Attempt to load the model with minimal resources to validate
+        _ = Llama(model_path=model_path, n_ctx=16, n_threads=1)
+    except Exception as e:
+        st.warning(f"Model file was invalid or corrupt: {e}\nRedownloading...")
         try:
             os.remove(model_path)
         except:
             pass
         cleanup_old_models()
         download_model()
-        result = try_load_model(model_path)
-        if isinstance(result, str):
-            st.error(f"Model still failed after re-download: {result}")
-            st.stop()
-        return result
-    return result
-
-# Load model if changed
-if st.session_state.model_name != selected_model_name:
-    if st.session_state.llm is not None:
-        del st.session_state.llm
-        gc.collect()
-    st.session_state.llm = validate_or_download_model()
-    st.session_state.model_name = selected_model_name
-
-llm = st.session_state.llm
-
-# Chat history state
-if "chat_history" not in st.session_state:
-    st.session_state.chat_history = []
-
-st.title(f"🧠 {selected_model['description']} (Streamlit + GGUF)")
-st.caption(f"Powered by `llama.cpp` | Model: {selected_model['filename']}")
-
-user_input = st.chat_input("Ask something...")
 
-if user_input:
-    # Prevent appending user message if assistant hasn't replied yet
-    if len(st.session_state.chat_history) % 2 == 1:
-        st.warning("Please wait for the assistant to respond before sending another message.")
-    else:
-        st.session_state.chat_history.append({"role": "user", "content": user_input})
+# Validate or download the selected model
+validate_or_download_model()
 
-        with st.chat_message("user"):
-            st.markdown(user_input)
-
-        # Trim conversation history to max 8 turns (user+assistant)
-        MAX_TURNS = 8
-        trimmed_history = st.session_state.chat_history[-MAX_TURNS * 2:]
-        messages = [{"role": "system", "content": system_prompt}] + trimmed_history
-
-        with st.chat_message("assistant"):
-            full_response = ""
-            response_area = st.empty()
-
-            stream = llm.create_chat_completion(
-                messages=messages,
-                max_tokens=max_tokens,
-                temperature=temperature,
-                top_k=top_k,
-                top_p=top_p,
-                repeat_penalty=repeat_penalty,
-                stream=True,
-            )
-
-            for chunk in stream:
-                if "choices" in chunk:
-                    delta = chunk["choices"][0]["delta"].get("content", "")
-                    full_response += delta
-                    response_area.markdown(full_response)
-
-            st.session_state.chat_history.append({"role": "assistant", "content": full_response})
+# Load model if changed
+if "model_name" not in st.session_state or st.session_state.model_name != selected_model_name:
+    if "llm" in st.session_state and st.session_state.llm