initial commit

.gitattributes

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,4 @@
+__py_cache__/
+*.pyc
+.gradio/
+.ipynb_checkpoints/

Dockerfile

@@ -0,0 +1,38 @@
+FROM nvidia/cuda:12.2.2-cudnn8-devel-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+    python3-pip \
+    python3-dev \
+    git \
+    build-essential \
+    ninja-build \
+    && useradd -m user \
+    && rm -rf /var/lib/apt/lists/*
+
+USER user
+WORKDIR /home/user/app
+
+COPY --chown=user:user . .
+
+RUN pip install --upgrade pip setuptools wheel packaging
+
+RUN pip install torch==2.6.0 torchvision==0.21.0 --index-url https://download.pytorch.org/whl/cu124
+
+RUN pip install \
+    gradio \
+    pillow \
+    matplotlib \
+    numpy \
+    requests \
+    peft==0.15.2 \
+    accelerate==1.4.0 \
+    'transformers @ git+https://github.com/huggingface/transformers@ccf2ca162e33f381e454cdb74bf4b41a51ab976d'
+
+# Install flash-attn matching dev environment
+RUN MAX_JOBS=1 pip install flash-attn==2.7.4.post1 --no-build-isolation
+
+EXPOSE 7860
+
+CMD ["python3", "app.py"]

README.md

@@ -0,0 +1,11 @@
+---
+title: Jina Embeddings V4 Retrieval Visual
+emoji: 👀
+colorFrom: green
+colorTo: purple
+sdk: docker
+app_port: 7860
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,181 @@
+"""
+app.py – Token‑wise heatmaps with JinaV4SimilarityMapper (similarity4)
+
+───────── Feature checklist (for future edits) ─────────
+1. Prompt + Image‑URL inputs (placeholders).                       ✅
+2. Run → downloads image (≥512 h), gets tokens+heatmaps,           ✅
+   auto‑selects first token, shows overlay, clears inputs.
+3. Output widgets hidden until results are ready.                  ✅
+4. Every run saved to examples/auto_<timestamp>/ with:
+     • prompt.txt, img_url.txt, image.jpg,
+     • heatmaps.json, per‑token PNGs, preview_first_token.jpg.     ✅
+5. On startup, first 3 example folders rendered below output with
+   layout: Prompt → Image URL → Tokens → Image+Heatmap.            ✅
+6. Margins: 40 px before “Examples” heading, 25 px between         ✅
+   successive examples (no separators, no extra HTML).
+7. Works on gradio==5.35.0 (no gr.Box, no Button.style, etc.).     ✅
+"""
+
+import sys, signal, base64, re, io, json, time
+from io import BytesIO
+from pathlib import Path
+from typing import Dict
+import subprocess
+
+import requests
+import gradio as gr
+from PIL import Image
+from similarity import JinaV4SimilarityMapper
+
+EX_DIR = Path("examples"); EX_DIR.mkdir(exist_ok=True)
+MIN_H  = 512
+
+# ───────── component helper (Buttons vs Radio) ─────────
+if hasattr(gr, "Buttons"):
+    ButtonsLike = gr.Buttons
+    def buttons_update(toks):
+        first = toks[0] if toks else None
+        return ButtonsLike.update(choices=toks, value=first, visible=True)
+else:
+    ButtonsLike = gr.Radio
+    def buttons_update(toks):
+        first = toks[0] if toks else None
+        return gr.update(choices=toks, value=first, visible=True)
+
+# ───────── util functions ─────────
+def _slug(t: str, n: int = 60) -> str:
+    return re.sub(r"[^\w\-]+", "_", t.lower())[:n] or "x"
+
+def _ensure_h(img: Image.Image, h: int = MIN_H) -> Image.Image:
+    if img.height >= h:
+        return img
+    sc = h / img.height
+    return img.resize((int(img.width * sc), h), Image.BICUBIC)
+
+def overlay(tok: str, maps: Dict[str, str], base: Image.Image) -> Image.Image:
+    if tok not in maps:
+        return base
+    hm = Image.open(BytesIO(base64.b64decode(maps[tok]))).convert("RGBA")
+    if hm.size != base.size:
+        hm = hm.resize(base.size, Image.BILINEAR)
+    return Image.alpha_composite(base.convert("RGBA"), hm)
+
+def save_run(prompt: str, url: str, img: Image.Image, maps: Dict[str, str]) -> None:
+    ts   = time.strftime("%Y%m%d_%H%M%S")
+    fldr = EX_DIR / f"auto_{_slug(prompt,30)}_{ts}"
+    fldr.mkdir(parents=True, exist_ok=True)
+
+    (fldr / "prompt.txt").write_text(prompt)
+    (fldr / "img_url.txt").write_text(url)
+    img.convert("RGB").save(fldr / "image.jpg", "JPEG")
+
+    with (fldr / "heatmaps.json").open("w") as f:
+        json.dump(maps, f)
+
+    for tok, b64png in maps.items():
+        (fldr / f"heatmap_{_slug(tok,30)}.png").write_bytes(base64.b64decode(b64png))
+
+    first = next(iter(maps))
+    overlay(first, maps, img).convert("RGB").save(fldr / "preview_first_token.jpg", "JPEG")
+    print(f"✨ Saved run to {fldr}", flush=True)
+
+# ───────── load mapper ─────────
+print("⏳ Loading JinaV4SimilarityMapper …", flush=True)
+MAPPER = JinaV4SimilarityMapper(client_type="local")
+print("✅ Mapper ready.", flush=True)
+
+# ───────── load up to 3 example folders ─────────
+def load_examples(n: int = 3):
+    ex = []
+    for fld in sorted(EX_DIR.iterdir())[:n]:
+        p_txt, p_url, p_img, p_map = fld/"prompt.txt", fld/"img_url.txt", None, fld/"heatmaps.json"
+        for c in fld.glob("image.*"): p_img = c; break
+        if not (p_txt.exists() and p_url.exists() and p_img and p_map.exists()): continue
+        ex.append(dict(
+            prompt=p_txt.read_text().strip(),
+            url   =p_url.read_text().strip(),
+            base  =Image.open(p_img).convert("RGB"),
+            maps  =json.load(open(p_map))
+        ))
+    return ex
+
+static_examples = load_examples()
+
+# ───────── backend for user Run ─────────
+def run_mapper(prompt: str, img_url: str):
+    if not img_url:
+        raise gr.Error("Please provide an image URL.")
+    try:
+        r = requests.get(img_url, stream=True, timeout=10); r.raise_for_status()
+        img = _ensure_h(Image.open(io.BytesIO(r.content)).convert("RGB"))
+    except Exception as e:
+        raise gr.Error(f"Image load failed: {e}")
+
+    img_proc = img.copy(); MAPPER.process_image(img_proc)
+    toks, maps = MAPPER.get_token_similarity_maps(prompt, img_proc)
+    if not toks:
+        raise gr.Error("Mapper returned no tokens.")
+
+    save_run(prompt, img_url, img, maps)
+
+    first_tok = toks[0]
+    info      = f"**Prompt:** {prompt}\n\n**Image URL:** {img_url}"
+    return (
+        buttons_update(toks), maps, img,
+        gr.update(value=overlay(first_tok, maps, img), visible=True),
+        gr.update(value=info, visible=True),
+        "", "")
+
+# ───────── UI ─────────
+css = """
+#main-title { margin-bottom: 40px; }
+#run-btn { margin: 20px 0; }
+#examples-title { margin: 40px 0; }
+.example-space { margin: 20px 0; }
+"""
+
+with gr.Blocks(css=css) as demo:
+    gr.Markdown("# Jina Embeddings v4", elem_id="main-title")
+
+    # User input
+    prompt_in = gr.Textbox(label="Prompt", placeholder="Describe what to query…")
+    url_in    = gr.Textbox(label="Image URL", placeholder="https://example.com/image.jpg")
+    run_btn   = gr.Button("Run", elem_id="run-btn")
+
+    # Output area
+    info_md   = gr.Markdown(visible=False)
+    token_sel = ButtonsLike(choices=[], label="Tokens", interactive=True, visible=False)
+    maps_st   = gr.State({})
+    img_st    = gr.State(None)
+    img_out   = gr.Image(label="Image + Heatmap", visible=False)
+
+    run_btn.click(run_mapper,
+        [prompt_in, url_in],
+        [token_sel, maps_st, img_st, img_out, info_md, prompt_in, url_in])
+
+    (token_sel.select if hasattr(token_sel,"select") else token_sel.change)(
+        overlay, [token_sel, maps_st, img_st], [img_out])
+
+    # Margin before examples heading
+    gr.Markdown("## Examples", elem_id="examples-title")
+
+    # Render examples
+    for ex in static_examples:
+        gr.Markdown(f"**Prompt:** {ex['prompt']}")
+        gr.Markdown(f"**Image URL:** {ex['url']}")
+        ex_img_st  = gr.State(ex["base"])
+        ex_map_st  = gr.State(ex["maps"])
+        first      = next(iter(ex["maps"]))
+        ex_btns    = ButtonsLike(choices=list(ex["maps"].keys()), value=first, interactive=True)
+        ex_disp    = gr.Image(value=overlay(first, ex["maps"], ex["base"]))
+        (ex_btns.select if hasattr(ex_btns,"select") else ex_btns.change)(
+            overlay, [ex_btns, ex_map_st, ex_img_st], [ex_disp])
+        # vertical margin after each example
+        gr.Markdown("", elem_classes=["example-space"])
+
+# ───────── graceful shutdown ─────────
+def _shutdown(*_): print("🛑 Shutting down …", flush=True); demo.close(); sys.exit(0)
+signal.signal(signal.SIGINT, _shutdown); signal.signal(signal.SIGTERM, _shutdown)
+
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0", server_port=7860, show_error=True)

examples/auto_a_person_riding_a_bicycle_near_20250702_150805/img_url.txt

@@ -0,0 +1 @@
+https://cdn.duvine.com/wp-content/uploads/2016/04/17095703/Slides_mallorca_FOR-WEB.jpg

examples/auto_a_person_riding_a_bicycle_near_20250702_150805/prompt.txt

@@ -0,0 +1 @@
+A person riding a bicycle near the ocean

examples/auto_a_person_walking_down_the_stre_20250702_152111/img_url.txt

@@ -0,0 +1 @@
+https://www.dogingtonpost.com/wp-content/uploads/2014/04/walkingthedog2.jpg

examples/auto_a_person_walking_down_the_stre_20250702_152111/prompt.txt

@@ -0,0 +1 @@
+A person walking down the street with a dog

examples/auto_computing_jobs_in_the_us_20250702_150954/img_url.txt

@@ -0,0 +1 @@
+https://www.ayresassociates.com/wp-content/uploads/2019/02/Career-Expo-Pie-Chart-Crop.jpg

examples/auto_computing_jobs_in_the_us_20250702_150954/prompt.txt

@@ -0,0 +1 @@
+Computing jobs in the US

examples/auto_simple_column_chart_with_index_20250702_151028/img_url.txt

@@ -0,0 +1 @@
+https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSf3wwnclk5MltzC6R_ghESRmlR2jlOf2REsw&s