Upload 4 files

README.md

@@ -1,12 +1,16 @@
 ---
-title: LMM For Better Captioning
-emoji: 🖼
+title: Share Captioner
+emoji: 🏃
 colorFrom: purple
-colorTo: red
+colorTo: green
 sdk: gradio
-sdk_version: 5.0.1
+sdk_version: 4.36.1
 app_file: app.py
 pinned: false
+license: apache-2.0
 ---
 
+**Paper or resources for more information:**
+[[Project](https://ShareGPT4V.github.io/)] [[Paper](https://huggingface.co/papers/2311.12793)] [[Code](https://github.com/ShareGPT4Omni/ShareGPT4V)]
+
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,154 +1,111 @@
 import gradio as gr
-import numpy as np
-import random
-
-# import spaces #[uncomment to use ZeroGPU]
-from diffusers import DiffusionPipeline
 import torch
-
-device = "cuda" if torch.cuda.is_available() else "cpu"
-model_repo_id = "stabilityai/sdxl-turbo"  # Replace to the model you would like to use
-
-if torch.cuda.is_available():
-    torch_dtype = torch.float16
-else:
-    torch_dtype = torch.float32
-
-pipe = DiffusionPipeline.from_pretrained(model_repo_id, torch_dtype=torch_dtype)
-pipe = pipe.to(device)
-
-MAX_SEED = np.iinfo(np.int32).max
-MAX_IMAGE_SIZE = 1024
-
-
-# @spaces.GPU #[uncomment to use ZeroGPU]
-def infer(
-    prompt,
-    negative_prompt,
-    seed,
-    randomize_seed,
-    width,
-    height,
-    guidance_scale,
-    num_inference_steps,
-    progress=gr.Progress(track_tqdm=True),
-):
-    if randomize_seed:
-        seed = random.randint(0, MAX_SEED)
-
-    generator = torch.Generator().manual_seed(seed)
-
-    image = pipe(
-        prompt=prompt,
-        negative_prompt=negative_prompt,
-        guidance_scale=guidance_scale,
-        num_inference_steps=num_inference_steps,
-        width=width,
-        height=height,
-        generator=generator,
-    ).images[0]
-
-    return image, seed
-
-
-examples = [
-    "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k",
-    "An astronaut riding a green horse",
-    "A delicious ceviche cheesecake slice",
-]
-
-css = """
-#col-container {
-    margin: 0 auto;
-    max-width: 640px;
+import spaces
+from PIL import Image
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_name = "Lin-Chen/ShareCaptioner"
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    model_name, device_map="cpu", torch_dtype=torch.float16, trust_remote_code=True).eval()
+model.tokenizer = tokenizer
+
+model.cuda()
+
+seg1 = '<|User|>:'
+seg2 = f'Analyze the image in a comprehensive and detailed manner.{model.eoh}\n<|Bot|>:'
+seg_emb1 = model.encode_text(seg1, add_special_tokens=True).cuda()
+seg_emb2 = model.encode_text(seg2, add_special_tokens=False).cuda()
+
+
+@spaces.GPU
+def detailed_caption(img_path):
+    subs = []
+    image = Image.open(img_path).convert("RGB")
+    subs.append(model.vis_processor(image).unsqueeze(0))
+
+    subs = torch.cat(subs, dim=0).cuda()
+    tmp_bs = subs.shape[0]
+    tmp_seg_emb1 = seg_emb1.repeat(tmp_bs, 1, 1)
+    tmp_seg_emb2 = seg_emb2.repeat(tmp_bs, 1, 1)
+    with torch.cuda.amp.autocast():
+        with torch.no_grad():
+            subs = model.encode_img(subs)
+            input_emb = torch.cat([tmp_seg_emb1, subs, tmp_seg_emb2], dim=1)
+            out_embeds = model.internlm_model.generate(inputs_embeds=input_emb,
+                                                       max_length=500,
+                                                       num_beams=3,
+                                                       min_length=1,
+                                                       do_sample=True,
+                                                       repetition_penalty=1.5,
+                                                       length_penalty=1.0,
+                                                       temperature=1.,
+                                                       eos_token_id=model.tokenizer.eos_token_id,
+                                                       num_return_sequences=1,
+                                                       )
+
+    return model.decode_text([out_embeds[0]])
+
+
+block_css = """
+#buttons button {
+    min-width: min(120px,100%);
 }
 """
-
-with gr.Blocks(css=css) as demo:
-    with gr.Column(elem_id="col-container"):
-        gr.Markdown(" # Text-to-Image Gradio Template")
+title_markdown = ("""
+# 🐬 ShareGPT4V: Improving Large Multi-modal Models with Better Captions
+[[Project Page](https://sharegpt4v.github.io/)] [[Code](https://github.com/ShareGPT4Omni/ShareGPT4V)] | [[Paper](https://github.com/InternLM/InternLM-XComposer/blob/main/projects/ShareGPT4V/ShareGPT4V.pdf)]
+""")
+tos_markdown = ("""
+### Terms of use
+By using this service, users are required to agree to the following terms:
+The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes.
+For an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.
+""")
+learn_more_markdown = ("""
+### License
+The service is a research preview intended for non-commercial use only, subject to the model [License](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) of LLaMA, [Terms of Use](https://openai.com/policies/terms-of-use) of the data generated by OpenAI, and [Privacy Practices](https://chrome.google.com/webstore/detail/sharegpt-share-your-chatg/daiacboceoaocpibfodeljbdfacokfjb) of ShareGPT. Please contact us if you find any potential violation.
+""")
+ack_markdown = ("""
+### Acknowledgement
+The template for this web demo is from [LLaVA](https://github.com/haotian-liu/LLaVA), and we are very grateful to LLaVA for their open source contributions to the community!
+""")
+
+
+def build_demo():
+    with gr.Blocks(title="Share-Captioner", theme=gr.themes.Default(), css=block_css) as demo:
+        gr.Markdown(title_markdown)
 
         with gr.Row():
-            prompt = gr.Text(
-                label="Prompt",
-                show_label=False,
-                max_lines=1,
-                placeholder="Enter your prompt",
-                container=False,
-            )
-
-            run_button = gr.Button("Run", scale=0, variant="primary")
-
-        result = gr.Image(label="Result", show_label=False)
-
-        with gr.Accordion("Advanced Settings", open=False):
-            negative_prompt = gr.Text(
-                label="Negative prompt",
-                max_lines=1,
-                placeholder="Enter a negative prompt",
-                visible=False,
-            )
-
-            seed = gr.Slider(
-                label="Seed",
-                minimum=0,
-                maximum=MAX_SEED,
-                step=1,
-                value=0,
-            )
-
-            randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
-
-            with gr.Row():
-                width = gr.Slider(
-                    label="Width",
-                    minimum=256,
-                    maximum=MAX_IMAGE_SIZE,
-                    step=32,
-                    value=1024,  # Replace with defaults that work for your model
-                )
-
-                height = gr.Slider(
-                    label="Height",
-                    minimum=256,
-                    maximum=MAX_IMAGE_SIZE,
-                    step=32,
-                    value=1024,  # Replace with defaults that work for your model
-                )
-
-            with gr.Row():
-                guidance_scale = gr.Slider(
-                    label="Guidance scale",
-                    minimum=0.0,
-                    maximum=10.0,
-                    step=0.1,
-                    value=0.0,  # Replace with defaults that work for your model
-                )
-
-                num_inference_steps = gr.Slider(
-                    label="Number of inference steps",
-                    minimum=1,
-                    maximum=50,
-                    step=1,
-                    value=2,  # Replace with defaults that work for your model
-                )
-
-        gr.Examples(examples=examples, inputs=[prompt])
-    gr.on(
-        triggers=[run_button.click, prompt.submit],
-        fn=infer,
-        inputs=[
-            prompt,
-            negative_prompt,
-            seed,
-            randomize_seed,
-            width,
-            height,
-            guidance_scale,
-            num_inference_steps,
-        ],
-        outputs=[result, seed],
-    )
-
-if __name__ == "__main__":
+            with gr.Column(scale=5):
+                with gr.Row(elem_id="Model ID"):
+                    gr.Dropdown(
+                        choices=['Share-Captioner'],
+                        value='Share-Captioner',
+                        interactive=True,
+                        label='Model ID',
+                        container=False)
+                img_path = gr.Image(label="Image", type="filepath")
+            with gr.Column(scale=8):
+                with gr.Row():
+                    caption = gr.Textbox(label='Caption')
+                with gr.Row():
+                    submit_btn = gr.Button(
+                        value="🚀 Generate", variant="primary")
+                    regenerate_btn = gr.Button(value="🔄 Regenerate")
+
+        gr.Markdown(tos_markdown)
+        gr.Markdown(learn_more_markdown)
+        gr.Markdown(ack_markdown)
+
+        submit_btn.click(detailed_caption, inputs=[
+                         img_path], outputs=[caption])
+        regenerate_btn.click(detailed_caption, inputs=[
+                             img_path], outputs=[caption])
+
+    return demo
+
+
+if __name__ == '__main__':
+    demo = build_demo()
     demo.launch()

gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

requirements.txt

@@ -1,6 +1,14 @@
-accelerate
-diffusers
-invisible_watermark
-torch
-transformers
-xformers
+transformers==4.32.0
+accelerate==0.24.0
+tiktoken==0.5.1
+einops==0.7.0
+transformers_stream_generator==0.0.4
+scipy==1.11.3
+torch==2.1.2
+torchvision==0.16.2
+pillow==10.0.1
+matplotlib==3.8.0
+sentencepiece
+urllib3==1.26.18
+timm==1.0.3
+spaces