Adding multiple images

app.py

@@ -1,7 +1,5 @@
 import gradio as gr
 import os
-
-os.system('pip install dashscope -U')
 import tempfile
 from pathlib import Path
 import secrets
@@ -9,31 +7,29 @@ import dashscope
 from dashscope import MultiModalConversation, Generation
 from PIL import Image
 
-
-# 设置API密钥
+# API key setup
 YOUR_API_TOKEN = os.getenv('YOUR_API_TOKEN')
 dashscope.api_key = YOUR_API_TOKEN
+
+# Global variables
 math_messages = []
+image_descriptions = []
+
 def process_image(image, shouldConvert=False):
-    # 获取上传文件的目录
-    global math_messages
-    math_messages = [] # reset when upload image
     uploaded_file_dir = os.environ.get("GRADIO_TEMP_DIR") or str(
         Path(tempfile.gettempdir()) / "gradio"
     )
     os.makedirs(uploaded_file_dir, exist_ok=True)
     
-    # 创建临时文件路径
     name = f"tmp{secrets.token_hex(20)}.jpg"
     filename = os.path.join(uploaded_file_dir, name)
-    # 保存上传的图片
+    
     if shouldConvert:
         new_img = Image.new('RGB', size=(image.width, image.height), color=(255, 255, 255))
         new_img.paste(image, (0, 0), mask=image)
         image = new_img
     image.save(filename)
     
-    # 调用qwen-vl-max-0809模型处理图片
     messages = [{
         'role': 'system',
         'content': [{'text': 'You are a helpful assistant.'}]
@@ -47,54 +43,53 @@ def process_image(image, shouldConvert=False):
     
     response = MultiModalConversation.call(model='qwen-vl-max-0809', messages=messages)
     
-    # 清理临时文件
     os.remove(filename)
     
     return response.output.choices[0]["message"]["content"]
 
-def get_math_response(image_description, user_question):
+def get_math_response(image_descriptions, user_question):
     global math_messages
     if not math_messages:
         math_messages.append({'role': 'system', 'content': 'You are a helpful math assistant.'})
-    math_messages = math_messages[:1]
-    if image_description is not None:
-        content = f'Image description: {image_description}\n\n'
-    else:
-        content = ''
-    query = f"{content}User question: {user_question}"
-    math_messages.append({'role': 'user', 'content': query})
+    
+    content = "Image descriptions:\n" + "\n".join(image_descriptions) if image_descriptions else ""
+    content += f"\n\nUser question: {user_question}"
+    
+    math_messages.append({'role': 'user', 'content': content})
     response = Generation.call(	
         model="qwen2.5-math-72b-instruct",
         messages=math_messages,	
         result_format='message',
         stream=True
     )
-    answer = None
+    answer = ""
     for resp in response:
         if resp.output is None:
             continue
         answer = resp.output.choices[0].message.content
         yield answer.replace("\\", "\\\\")
-    print(f'query: {query}\nanswer: {answer}')
-    if answer is None:
-        math_messages.pop()
-    else:
-        math_messages.append({'role': 'assistant', 'content': answer})
-
+    
+    math_messages.append({'role': 'assistant', 'content': answer})
 
-def math_chat_bot(image, sketchpad, question, state):
-    current_tab_index = state["tab_index"]
-    image_description = None
-    # Upload
-    if current_tab_index == 0:
-        if image is not None:
-            image_description = process_image(image)
-    # Sketch
-    elif current_tab_index == 1:
-        print(sketchpad)
-        if sketchpad and sketchpad["composite"]:
-            image_description = process_image(sketchpad["composite"], True)
-    yield from get_math_response(image_description, question)
+def math_chat_bot(images, sketchpad, question, chat_history):
+    global image_descriptions
+    
+    # Process new images
+    for image in images:
+        if image:
+            description = process_image(image)
+            image_descriptions.append(description)
+    
+    # Process sketchpad if present
+    if sketchpad and sketchpad["composite"]:
+        sketch_description = process_image(sketchpad["composite"], True)
+        image_descriptions.append(sketch_description)
+    
+    # Generate response
+    response = ""
+    for chunk in get_math_response(image_descriptions, question):
+        response += chunk
+        yield chat_history + [(question, response)]
 
 css = """
 #qwen-md .katex-display { display: inline; }
@@ -102,11 +97,7 @@ css = """
 #qwen-md .katex-display>.katex>.katex-html { display: inline; }
 """
 
-def tabs_select(e: gr.SelectData, _state):
-    _state["tab_index"] = e.index
-
-
-# 创建Gradio接口
+# Create Gradio interface
 with gr.Blocks(css=css) as demo:
     gr.HTML("""\
 <p align="center"><img src="https://modelscope.oss-cn-beijing.aliyuncs.com/resource/qwen.png" style="height: 60px"/><p>"""
@@ -114,56 +105,23 @@ with gr.Blocks(css=css) as demo:
             """\
 <center><font size=3>This WebUI is based on Qwen2-VL for OCR and Qwen2.5-Math for mathematical reasoning. You can input either images or texts of mathematical or arithmetic problems.</center>"""
             )
-    state = gr.State({"tab_index": 0})
+    
     with gr.Row():
         with gr.Column():
-            with gr.Tabs() as input_tabs:
-                with gr.Tab("Upload"):
-                    input_image = gr.Image(type="pil", label="Upload"),
-                with gr.Tab("Sketch"):
-                    input_sketchpad = gr.Sketchpad(type="pil", label="Sketch", layers=False)
-            input_tabs.select(fn=tabs_select, inputs=[state])
-            input_text = gr.Textbox(label="input your question")
+            input_images = gr.File(file_count="multiple", label="Upload Images")
+            input_sketchpad = gr.Sketchpad(type="pil", label="Sketch", layers=False)
+            input_text = gr.Textbox(label="Input your question")
             with gr.Row():
-                with gr.Column():
-                    clear_btn = gr.ClearButton(
-                        [*input_image, input_sketchpad, input_text])
-                with gr.Column():
-                    submit_btn = gr.Button("Submit", variant="primary")
+                clear_btn = gr.ClearButton([input_images, input_sketchpad, input_text])
+                submit_btn = gr.Button("Submit", variant="primary")
+        
         with gr.Column():
-            output_md = gr.Markdown(label="answer",
-                                    latex_delimiters=[{
-                                        "left": "\\(",
-                                        "right": "\\)",
-                                        "display": True
-                                    }, {
-                                        "left": "\\begin\{equation\}",
-                                        "right": "\\end\{equation\}",
-                                        "display": True
-                                    }, {
-                                        "left": "\\begin\{align\}",
-                                        "right": "\\end\{align\}",
-                                        "display": True
-                                    }, {
-                                        "left": "\\begin\{alignat\}",
-                                        "right": "\\end\{alignat\}",
-                                        "display": True
-                                    }, {
-                                        "left": "\\begin\{gather\}",
-                                        "right": "\\end\{gather\}",
-                                        "display": True
-                                    }, {
-                                        "left": "\\begin\{CD\}",
-                                        "right": "\\end\{CD\}",
-                                        "display": True
-                                    }, {
-                                        "left": "\\[",
-                                        "right": "\\]",
-                                        "display": True
-                                    }],
-                                    elem_id="qwen-md")
-        submit_btn.click(
-            fn=math_chat_bot,
-            inputs=[*input_image, input_sketchpad, input_text, state],
-            outputs=output_md)
+            chat_output = gr.Chatbot(label="Chat History", elem_id="qwen-md")
+    
+    submit_btn.click(
+        fn=math_chat_bot,
+        inputs=[input_images, input_sketchpad, input_text, chat_output],
+        outputs=chat_output
+    )
+
 demo.launch()