app.py

import gradio as gr
# from models.vsa_model import VisionSearchAssistant
# from models.vsa_prompt import COCO_CLASSES


SAMPLES = {
    "images/iclr.jpg": ("What prize did this paper win in 2024?", ", ".join(COCO_CLASSES)),
    "images/tesla.jpg": ("What's the income of this company?", "car"),
    "images/xiaomi.jpg": ("Provide information about the new products of this brand.", ", ".join(COCO_CLASSES)),
    "images/leshi.jpg": ("Provide information about new products of this brand of potato chips in 2024.", ", ".join(COCO_CLASSES)),
}
SAMPLE_IMAGES = list(SAMPLES.keys())
SAMPLE_TEXTS = [e[0] for e in SAMPLES.values()]
SAMPLE_CLASSES = [e[1] for e in SAMPLES.values()]


def process_inputs(image, text, ground_classes):
    if len(ground_classes) == 0:
        ground_classes = None
    else:
        ground_classes = ground_classes.split(', ')

    ground_output, query_output, search_output, answer_output = None, None, None, None
    for output, output_type in vsa.app_run(image, text, ground_classes = ground_classes):
        if output_type == 'ground':
            ground_output = output
            yield ground_output, query_output, search_output, answer_output
        elif output_type == 'query':
            query_output = ''
            for qid, query in enumerate(output):
                query_output += '[Area {}] '.format(qid) + query + '\n'
            yield ground_output, query_output, search_output, answer_output
        elif output_type == 'search':
            search_output = ''
            for cid, context in enumerate(output):
                search_output += '[Context {}] '.format(cid) + context + '\n'
            yield ground_output, query_output, search_output, answer_output
        elif output_type == 'answer':
            answer_output = output
            yield ground_output, query_output, search_output, answer_output


def select_sample_inputs(sample):
    if sample == 'none':
        return None, None, None
    image = sample
    text, classes = SAMPLES[sample]
    return image, text, classes

def confirm_sample_inputs(image, text, classes):
    return image, text, classes


# Create a Blocks interface
with gr.Blocks() as app:
    with gr.Tab("Run"):
        with gr.Row():
            with gr.Column():
                with gr.Row():
                    image_input = gr.Image(label="Input Image", height=300, width=300)
                    ground_output = gr.Image(label="Grounding Output", height=300, width=300, interactive=False)
                prompt_input = gr.Textbox(label="Input Text Prompt", lines=1, max_lines=1)
                ground_class_input = gr.Textbox(
                    label="Ground Classes", 
                    placeholder="Defaultly, the model will use COCO classes.",
                    lines=1, max_lines=1
                )
                submit_button = gr.Button("Submit")
                answer_output = gr.Textbox(label="Answer Output", lines=4, max_lines=4, interactive=False)
            with gr.Column():
                query_output = gr.Textbox(label='Query Output', lines=14, max_lines=14, interactive=False)
                search_output = gr.Textbox(label="Search Output", lines=14, max_lines=14, interactive=False)
    with gr.Tab("Samples"):
        sample_input = gr.Dropdown(label="Select One Sample", choices=SAMPLE_IMAGES)
        with gr.Row():
            sample_image = gr.Image(label="Sample Input Image", height=300, interactive=False, value=SAMPLE_IMAGES[0])
            with gr.Column():
                sample_text = gr.Textbox(label="Sample Input Text", lines=4, max_lines=4, interactive=False, value=SAMPLE_TEXTS[0])
                sample_classes = gr.Textbox(label="Sample Input Classes", lines=4, max_lines=4, interactive=False, value=SAMPLE_CLASSES[0])
        sample_button = gr.Button("Select This Sample")
        
    
    # Processing action
    submit_button.click(
        fn=process_inputs,
        inputs=[image_input, prompt_input, ground_class_input],
        outputs=[ground_output, query_output, search_output, answer_output],
        show_progress=True,
    )
    sample_input.change(
        fn=select_sample_inputs,
        inputs=[sample_input],
        outputs=[sample_image, sample_text, sample_classes]
    )
    sample_button.click(
        fn=confirm_sample_inputs,
        inputs=[sample_image, sample_text, sample_classes],
        outputs=[image_input, prompt_input, ground_class_input],
    )


# vsa = VisionSearchAssistant(
#     ground_device = "cuda:0",
#     vlm_device="cuda:0",
#     vlm_load_4bit=True,
# )
# Launch the app
app.launch()