Display images before answering

app.py

@@ -31,6 +31,7 @@ from langchain_chroma import Chroma
 from collections import defaultdict
 from gradio_modal import Modal
 
+from PIL import Image
 
 
 # ClimateQ&A imports
@@ -134,6 +135,7 @@ async def chat(query,history,audience,sources,reports,current_graphs):
 
     docs = []
     docs_used = True
+    used_figures=[]
     docs_html = ""
     output_query = ""
     output_language = ""
@@ -174,6 +176,11 @@ async def chat(query,history,audience,sources,reports,current_graphs):
                     except Exception as e:
                         print(f"Error getting documents: {e}")
                         print(event)
+                        
+                    
+                        
+                    used_figures, figures, gallery = process_figures(docs, figures, gallery, used_figures)
+
  
                 elif event["name"] in steps_display.keys() and event["event"] == "on_chain_start": #display steps
                     event_description,display_output = steps_display[node]
@@ -267,32 +274,48 @@ async def chat(query,history,audience,sources,reports,current_graphs):
         raise gr.Error(f"ClimateQ&A Error: {str(e)[:100]} - The error has been noted, try another question and if the error remains, you can contact us :)")
 
         
-    docs_figures = [d for d in docs if d.metadata["chunk_type"] == "image"]
-    for i, doc in enumerate(docs_figures):
-        if doc.metadata["chunk_type"] == "image":
-            try:
-                key = f"Image {i+1}"
-
-                image_path = doc.metadata["image_path"].split("documents/")[1]
-                img = get_image_from_azure_blob_storage(image_path)
-
-                # Convert the image to a byte buffer
-                buffered = BytesIO()
-                img.save(buffered, format="PNG")
-                img_str = base64.b64encode(buffered.getvalue()).decode()
-                
-                figures = figures + make_html_figure_sources(doc, i, img_str)  
-                
-                gallery.append(img)
+        
+    # figures, gallery = process_figures(docs, figures, gallery)
 
-            except Exception as e:
-                print(f"Skipped adding image {i} because of {e}")
    
         
     
 
     yield history, docs_html, output_query, output_language, gallery, figures, graphs_html#,output_query,output_keywords
 
+def process_figures(docs, figures, gallery, used_figures):
+    
+    docs_figures = [d for d in docs if d.metadata["chunk_type"] == "image"]
+    for i, doc in enumerate(docs_figures):
+        if doc.metadata["chunk_type"] == "image":
+            if doc.metadata["figure_code"] != "N/A":
+                title = f"{doc.metadata['figure_code']} - {doc.metadata['short_name']}"
+            else:
+                title = f"{doc.metadata['short_name']}"
+            
+            
+            if title not in used_figures:
+                used_figures.append(title)
+                try:
+                    key = f"Image {i+1}"
+
+                    image_path = doc.metadata["image_path"].split("documents/")[1]
+                    img = get_image_from_azure_blob_storage(image_path)
+
+                    # Convert the image to a byte buffer
+                    buffered = BytesIO()
+                    max_image_length = 500
+                    img_resized = img.resize((max_image_length, int(max_image_length * img.size[1]/img.size[0])))
+                    img_resized.save(buffered, format="PNG")
+
+                    img_str = base64.b64encode(buffered.getvalue()).decode()
+                    
+                    figures = figures + make_html_figure_sources(doc, i, img_str)  
+                    gallery.append(img)
+                except Exception as e:
+                    print(f"Skipped adding image {i} because of {e}")
+
+    return used_figures, figures, gallery
 
 def save_feedback(feed: str, user_id):
     if len(feed) > 1:
@@ -619,16 +642,12 @@ with gr.Blocks(title="Climate Q&A", css_paths=os.getcwd()+ "/style.css", theme=t
                 
 
 
-    # config_button = gr.Button("Configuration",elem_id="config-button",interactive=True)
-
     def start_chat(query,history):
-        # history = history + [(query,None)]
-        # history = [tuple(x) for x in history]
         history = history + [ChatMessage(role="user", content=query)]
         return (gr.update(interactive = False),gr.update(selected=1),history)
     
     def finish_chat():
-        return (gr.update(interactive = True,value = ""),gr.update(selected=3))
+        return gr.update(interactive = True,value = "")
 
 
     def change_completion_status(current_state):
@@ -651,7 +670,7 @@ with gr.Blocks(title="Climate Q&A", css_paths=os.getcwd()+ "/style.css", theme=t
     (textbox
         .submit(start_chat, [textbox,chatbot], [textbox,tabs,chatbot],queue = False,api_name = "start_chat_textbox")
         .then(chat, [textbox,chatbot,dropdown_audience, dropdown_sources,dropdown_reports, current_graphs], [chatbot,sources_textbox,output_query,output_language,gallery_component, figures_cards, current_graphs],concurrency_limit = 8,api_name = "chat_textbox")
-        .then(finish_chat, None, [textbox,tabs],api_name = "finish_chat_textbox")
+        .then(finish_chat, None, [textbox],api_name = "finish_chat_textbox")
         .then(update_sources_number_display, [sources_textbox, figures_cards, current_graphs],[tab_sources, tab_figures, tab_recommended_content] )
     )
 

front/utils.py

@@ -216,8 +216,9 @@ def make_html_figure_sources(source,i,img_str):
     <div class="card card-image">
         <div class="card-content">
             <h2>Image {i} - {title} - Page {int(meta['page_number'])}</h2>
-            <p class='ai-generated'>AI-generated description</p>
             <img src="data:image/png;base64, { img_str }" alt="Alt text" />
+            <p class='ai-generated'>AI-generated description</p>
+
 
             <p>{content}</p>
 

style.css

@@ -11,6 +11,10 @@
     margin: 0px;
 }
 
+.gradio-container {
+    width: 100%!important;
+    max-width: 100% !important;
+}
 
 /* fix for huggingface infinite growth*/
 main.flex.flex-1.flex-col { 
@@ -93,7 +97,12 @@ body.dark .tip-box * {
     font-size:14px !important;
     
 }
-
+.card-content img {
+    display: block;
+    margin: auto;
+    max-width: 100%; /* Ensures the image is responsive */
+    height: auto;
+}
 
 a {
     text-decoration: none;