Create app.py

app.py

@@ -0,0 +1,134 @@
+import streamlit as st
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+
+# Set Streamlit page configuration
+st.set_page_config(
+    page_title="Qwen2.5-Coder Chat",
+    page_icon="💬",
+    layout="wide",
+)
+
+# Title of the app
+st.title("💬 Qwen2.5-Coder Chat Interface")
+
+# Initialize session state for messages
+if 'messages' not in st.session_state:
+    st.session_state['messages'] = []
+
+# Function to load the model
+@st.cache_resource
+def load_model():
+    model_name = "Qwen/Qwen2.5-Coder-32B-Instruct"  # Replace with your model path or name
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(
+        model_name,
+        torch_dtype=torch.float16,  # Use appropriate dtype
+        device_map='auto'           # Automatically choose device (GPU/CPU)
+    )
+    return tokenizer, model
+
+# Load tokenizer and model
+with st.spinner("Loading model... This may take a while..."):
+    tokenizer, model = load_model()
+
+# Function to generate model response
+def generate_response(prompt, max_tokens=2048):
+    inputs = tokenizer.encode(prompt, return_tensors='pt').to(model.device)
+    
+    # Generate response
+    with torch.no_grad():
+        outputs = model.generate(
+            inputs,
+            max_length=max_tokens,
+            temperature=0.7,       # Adjust for creativity
+            top_p=0.9,             # Nucleus sampling
+            do_sample=True,        # Enable sampling
+            num_return_sequences=1
+        )
+    
+    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    # Remove the prompt from the response
+    response = response[len(prompt):].strip()
+    return response
+
+# Layout: Two columns, main chat and sidebar
+chat_col, sidebar_col = st.columns([4, 1])
+
+with chat_col:
+    # Display chat messages
+    for message in st.session_state['messages']:
+        if message['role'] == 'user':
+            st.markdown(f"**You:** {message['content']}")
+        else:
+            st.markdown(f"**Qwen2.5-Coder:** {message['content']}")
+
+    # Input area for user
+    with st.form(key='chat_form', clear_on_submit=True):
+        user_input = st.text_area("You:", height=100)
+        submit_button = st.form_submit_button(label='Send')
+
+    if submit_button and user_input:
+        # Append user message
+        st.session_state['messages'].append({'role': 'user', 'content': user_input})
+        
+        # Generate and append model response
+        with st.spinner("Qwen2.5-Coder is typing..."):
+            response = generate_response(user_input, max_tokens=2048)
+            st.session_state['messages'].append({'role': 'assistant', 'content': response})
+
+        # Rerun to display new messages
+        st.experimental_rerun()
+
+with sidebar_col:
+    st.sidebar.header("Settings")
+    max_tokens = st.sidebar.slider(
+        "Maximum Tokens",
+        min_value=512,
+        max_value=4096,
+        value=2048,
+        step=256,
+        help="Set the maximum number of tokens for the model's response."
+    )
+    
+    temperature = st.sidebar.slider(
+        "Temperature",
+        min_value=0.1,
+        max_value=1.0,
+        value=0.7,
+        step=0.1,
+        help="Controls the randomness of the model's output."
+    )
+    
+    top_p = st.sidebar.slider(
+        "Top-p (Nucleus Sampling)",
+        min_value=0.1,
+        max_value=1.0,
+        value=0.9,
+        step=0.1,
+        help="Controls the diversity of the model's output."
+    )
+
+    if st.sidebar.button("Clear Chat"):
+        st.session_state['messages'] = []
+        st.experimental_rerun()
+
+# Update the generate_response function to use sidebar settings
+def generate_response(prompt):
+    inputs = tokenizer.encode(prompt, return_tensors='pt').to(model.device)
+    
+    # Generate response
+    with torch.no_grad():
+        outputs = model.generate(
+            inputs,
+            max_length=max_tokens,
+            temperature=temperature,
+            top_p=top_p,
+            do_sample=True,
+            num_return_sequences=1
+        )
+    
+    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    # Remove the prompt from the response
+    response = response[len(prompt):].strip()
+    return response