vision and text

app.py

@@ -2,7 +2,11 @@ import streamlit as st
 from huggingface_hub import InferenceClient
 import os
 from typing import Iterator
-
+from PIL import Image
+import pytesseract
+from PyPDF2 import PdfReader
+import base64
+from together import Together
 
 API_KEY = os.getenv("TOGETHER_API_KEY")
 if not API_KEY:
@@ -12,21 +16,37 @@ if not API_KEY:
 # Initialize the client with Together AI provider
 @st.cache_resource
 def get_client():
-    return InferenceClient(
-        provider="together",
-         api_key=API_KEY
-    )
+    #return InferenceClient(
+    #    provider="together",
+    #     api_key=API_KEY
+    #)
+    return Together(api_key=API_KEY)  # Use Together.ai's official client
 
 def process_file(file) -> str:
     """Process uploaded file and return its content"""
     if file is None:
         return ""
-    
+
     try:
-        content = file.getvalue().decode('utf-8')
-        return content
+        # Handle PDF files
+        if file.type == "application/pdf":
+            text = ""
+            pdf_reader = PdfReader(file)
+            for page in pdf_reader.pages:
+                page_text = page.extract_text()
+                if page_text:
+                    text += page_text + "\n"
+            return text
+
+        # Handle image files
+        elif file.type.startswith("image/"):
+            return base64.b64encode(file.getvalue()).decode("utf-8")
+
+        # Handle text files
+        else:
+            return file.getvalue().decode('utf-8')
     except Exception as e:
-        return f"Error reading file: {str(e)}"
+        return f"Error processing file: {str(e)}"
 
 def generate_response(
     message: str,
@@ -37,51 +57,80 @@ def generate_response(
     top_p: float,
     files=None
 ) -> Iterator[str]:
-    """Generate streaming response from the model"""
     client = get_client()
-    
-    # Process file if uploaded
-        # Process multiple files if uploaded
-    all_content = ""
-    if files:
-        file_contents = [process_file(file) for file in files]
-        all_content = "\n\n".join([
-            f"File {i+1} content:\n{content}" 
-            for i, content in enumerate(file_contents)
-        ])
-    
-    if all_content:
-        message = f"{all_content}\n\nUser message:\n{message}"
 
+    has_images = False
+    content_blocks = []
+    image_content = None  # To store image data
+    image_mime_type = None  # To store MIME type
+
+    if files:
+        for file in files:
+            content = process_file(file)
+            if file.type.startswith("image/"):
+                has_images = True
+                image_content = content  # Already base64 encoded
+                image_mime_type = file.type  # Store MIME type
+            else:
+                content_blocks.append({
+                    "type": "text",
+                    "text": f"File content:\n{content}"
+                })
+
+    # Build messages
     messages = [{"role": "system", "content": system_message}]
 
-    # Add conversation history
+    # Add history
     for user_msg, assistant_msg in history:
-        if user_msg:
-            messages.append({"role": "user", "content": user_msg})
-        if assistant_msg:
-            messages.append({"role": "assistant", "content": assistant_msg})
-
-    # Add current message
-    messages.append({"role": "user", "content": message})
+        messages.append({"role": "user", "content": user_msg})
+        messages.append({"role": "assistant", "content": assistant_msg})
 
     try:
-        stream = client.chat.completions.create(
-            model="deepseek-ai/DeepSeek-R1",
-            messages=messages,
-            max_tokens=max_tokens,
-            stream=True,
-            temperature=temperature,
-            top_p=top_p,
-        )
-        
+        if has_images:
+            # Vision model request
+            vision_messages = [{
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": message},
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:{image_mime_type};base64,{image_content}",
+                        },
+                    },
+                ]
+            }]
+
+            stream = client.chat.completions.create(
+                model="meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo",
+                messages=vision_messages,
+                stream=True,
+            )
+
+        else:
+            # Text-only model request
+            current_message = {
+                "role": "user",
+                "content": [{"type": "text", "text": message}] + content_blocks
+            }
+            messages.append(current_message)
+
+            stream = client.chat.completions.create(
+                model="deepseek-ai/DeepSeek-R1",
+                messages=messages,
+                max_tokens=max_tokens,
+                temperature=temperature,
+                top_p=top_p,
+                stream=True
+            )
+
+        # Stream response
         for chunk in stream:
-            if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.content:
+            if chunk.choices and chunk.choices[0].delta.content:
                 yield chunk.choices[0].delta.content
-                
+
     except Exception as e:
         yield f"Error: {str(e)}"
-
 def main():
     st.set_page_config(page_title="DeepSeek Chat", page_icon="💭", layout="wide")
     
@@ -123,9 +172,10 @@ def main():
         )
         uploaded_file = st.file_uploader(
             "Upload File (optional)",
-               type=['txt', 'py', 'md', 'swift', 'java', 'js', 'ts', 'rb', 'go', 
-                'php', 'c', 'cpp', 'h', 'hpp', 'cs', 'html', 'css', 'kt', 'svelte'],
-                accept_multiple_files=True  # Add this parameter
+            type=['txt', 'py', 'md', 'swift', 'java', 'js', 'ts', 'rb', 'go', 
+                 'php', 'c', 'cpp', 'h', 'hpp', 'cs', 'html', 'css', 'kt', 'svelte',
+                 'pdf', 'png', 'jpg', 'jpeg'],  # Added file types
+            accept_multiple_files=True
         )
 
     # Display chat messages

requirements.txt

@@ -1,2 +1,7 @@
-huggingface_hub==0.25.2
-streamlit
+huggingface_hub==0.29.0
+streamlit
+pytesseract
+PyPDF2
+Pillow 
+pytesseract
+together