Update app.py

app.py

@@ -1,470 +1,305 @@
+# /// script
+# requires-python = ">=3.12"
+# dependencies = [
+#     "chromadb==1.0.4",
+#     "datasets==3.5.0",
+#     "marimo",
+#     "matplotlib==3.10.1",
+#     "numpy==2.2.4",
+#     "open-clip-torch==2.32.0",
+#     "pillow==11.1.0",
+# ]
+# ///
+
 import marimo
 
-__generated_with = "0.9.2"
-app = marimo.App()
+__generated_with = "0.12.8"
+app = marimo.App(width="medium")
 
 
 @app.cell
-def __():
+def _():
     import marimo as mo
-
-    mo.md("# Welcome to marimo! 🌊🍃")
     return (mo,)
 
 
-@app.cell
-def __(mo):
-    slider = mo.ui.slider(1, 22)
-    return (slider,)
-
-
-@app.cell
-def __(mo, slider):
+@app.cell(hide_code=True)
+def _(mo):
     mo.md(
-        f"""
-        marimo is a **reactive** Python notebook.
+        r"""
+        # Multimodal Retrieval
 
-        This means that unlike traditional notebooks, marimo notebooks **run
-        automatically** when you modify them or
-        interact with UI elements, like this slider: {slider}.
+        Chroma supports multimodal collections, i.e. collections which contain, and can be queried by, multiple modalities of data.
 
-        {"##" + "🍃" * slider.value}
+        This notebook shows an example of how to create and query a collection with both text and images, using Chroma's built-in features.
         """
     )
     return
 
 
 @app.cell(hide_code=True)
-def __(mo):
-    mo.accordion(
-        {
-            "Tip: disabling automatic execution": mo.md(
-                rf"""
-            marimo lets you disable automatic execution: just go into the
-            notebook settings and set
-
-            "Runtime > On Cell Change" to "lazy".
-
-            When the runtime is lazy, after running a cell, marimo marks its
-            descendants as stale instead of automatically running them. The
-            lazy runtime puts you in control over when cells are run, while
-            still giving guarantees about the notebook state.
-            """
-            )
-        }
-    )
-    return
-
-
-@app.cell(hide_code=True)
-def __(mo):
+def _(mo):
     mo.md(
-        """
-        Tip: This is a tutorial notebook. You can create your own notebooks
-        by entering `marimo edit` at the command line.
-        """
-    ).callout()
-    return
+        r"""
+        ## Dataset
 
+        We us a small subset of the [coco object detection dataset](https://huggingface.co/datasets/detection-datasets/coco), hosted on HuggingFace.
 
-@app.cell(hide_code=True)
-def __(mo):
-    mo.md(
+        We download a small fraction of all the images in the dataset locally, and use it to create a multimodal collection.
         """
-        ## 1. Reactive execution
+    )
+    return
 
-        A marimo notebook is made up of small blocks of Python code called
-        cells.
 
-        marimo reads your cells and models the dependencies among them: whenever
-        a cell that defines a global variable  is run, marimo
-        **automatically runs** all cells that reference that variable.
+@app.cell
+def _():
+    import os
 
-        Reactivity keeps your program state and outputs in sync with your code,
-        making for a dynamic programming environment that prevents bugs before they
-        happen.
-        """
-    )
-    return
+    from datasets import load_dataset
+    from matplotlib import pyplot as plt
+    return load_dataset, os
 
 
-@app.cell(hide_code=True)
-def __(changed, mo):
-    (
-        mo.md(
-            f"""
-            **✨ Nice!** The value of `changed` is now {changed}.
-
-            When you updated the value of the variable `changed`, marimo
-            **reacted** by running this cell automatically, because this cell
-            references the global variable `changed`.
-
-            Reactivity ensures that your notebook state is always
-            consistent, which is crucial for doing good science; it's also what
-            enables marimo notebooks to double as tools and  apps.
-            """
-        )
-        if changed
-        else mo.md(
-            """
-            **🌊 See it in action.** In the next cell, change the value of the
-            variable  `changed` to `True`, then click the run button.
-            """
+@app.cell
+def _(load_dataset, mo):
+    with mo.status.spinner(title="Loading dataset"):
+        dataset = load_dataset(
+            path="detection-datasets/coco",
+            name="default",
+            split="train",
+            streaming=True,
         )
-    )
-    return
+
+    N_IMAGES = 20
+    return N_IMAGES, dataset
 
 
 @app.cell
-def __():
-    changed = False
-    return (changed,)
+def _(N_IMAGES, dataset, mo, os):
+    # Write the images to a folder
+    IMAGE_FOLDER = "images"
+    os.makedirs(IMAGE_FOLDER, exist_ok=True)
+    i = 0
+    all_images = []
+    with mo.status.spinner(title="Loading images"):
+        for row in dataset.take(N_IMAGES):
+            image = row["image"]
+            all_images.append(image)
+            image.save(f"images/{i}.jpg")
+            i += 1
+    return IMAGE_FOLDER, all_images
 
 
 @app.cell(hide_code=True)
-def __(mo):
-    mo.accordion(
-        {
-            "Tip: execution order": (
-                """
-                The order of cells on the page has no bearing on
-                the order in which cells are executed: marimo knows that a cell
-                reading a variable must run after the cell that  defines it. This
-                frees you to organize your code in the way that makes the most
-                sense for you.
-                """
-            )
-        }
+def _(mo):
+    img_width = mo.ui.slider(
+        label="Image width", start=100, stop=300, step=10, debounce=True
     )
-    return
+    img_width
+    return (img_width,)
 
 
 @app.cell(hide_code=True)
-def __(mo):
-    mo.md(
-        """
-        **Global names must be unique.** To enable reactivity, marimo imposes a
-        constraint on how names appear in cells: no two cells may define the same
-        variable.
-        """
-    )
-    return
+def _(all_images, img_width, mo):
+    import io
 
 
-@app.cell(hide_code=True)
-def __(mo):
-    mo.accordion(
-        {
-            "Tip: encapsulation": (
-                """
-                By encapsulating logic in functions, classes, or Python modules,
-                you can minimize the number of global variables in your notebook.
-                """
-            )
-        }
-    )
-    return
+    def as_image(src):
+        img_byte_arr = io.BytesIO()
+        src.save(img_byte_arr, format=src.format or "PNG")
+        img_byte_arr.seek(0)
+        return mo.image(img_byte_arr, width=img_width.value)
 
 
-@app.cell(hide_code=True)
-def __(mo):
-    mo.accordion(
-        {
-            "Tip: private variables": (
-                """
-                Variables prefixed with an underscore are "private" to a cell, so
-                they can be defined by multiple cells.
-                """
-            )
-        }
+    mo.hstack(
+        [as_image(_img) for _img in all_images[10:]],
+        wrap=True,
     )
     return
 
 
 @app.cell(hide_code=True)
-def __(mo):
+def _(mo):
     mo.md(
-        """
-        ## 2. UI elements
+        r"""
+        ## Ingesting multimodal data
 
-        Cells can output interactive UI elements. Interacting with a UI
-        element **automatically triggers notebook execution**: when
-        you interact with a UI element, its value is sent back to Python, and
-        every cell that references that element is re-run.
+        Chroma supports multimodal collections by referencing external URIs for data types other than text.
+        All you have to do is specify a data loader when creating the collection, and then provide the URI for each entry.
 
-        marimo provides a library of UI elements to choose from under
-        `marimo.ui`.
+        For this example, we are only adding images, though you can also add text.
         """
     )
     return
 
 
-@app.cell
-def __(mo):
-    mo.md("""**🌊 Some UI elements.** Try interacting with the below elements.""")
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ### Creating a multi-modal collection
+
+        First we create the default Chroma client.
+        """
+    )
     return
 
 
 @app.cell
-def __(mo):
-    icon = mo.ui.dropdown(["🍃", "🌊", "✨"], value="🍃")
-    return (icon,)
+def _():
+    import chromadb
 
+    client = chromadb.Client()
+    return (client,)
 
-@app.cell
-def __(icon, mo):
-    repetitions = mo.ui.slider(1, 16, label=f"number of {icon.value}: ")
-    return (repetitions,)
 
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        Next we specify an embedding function and a data loader.
 
-@app.cell
-def __(icon, repetitions):
-    icon, repetitions
+        The built-in `OpenCLIPEmbeddingFunction` works with both text and image data. The `ImageLoader` is a simple data loader that loads images from a local directory.
+        """
+    )
     return
 
 
 @app.cell
-def __(icon, mo, repetitions):
-    mo.md("# " + icon.value * repetitions.value)
-    return
+def _():
+    from chromadb.utils.data_loaders import ImageLoader
+    from chromadb.utils.embedding_functions import OpenCLIPEmbeddingFunction
+
+    embedding_function = OpenCLIPEmbeddingFunction()
+    image_loader = ImageLoader()
+    return embedding_function, image_loader
 
 
 @app.cell(hide_code=True)
-def __(mo):
-    mo.md(
-        """
-        ## 3. marimo is just Python
+def _(mo):
+    mo.md(r"""We create a collection with the embedding function and data loader.""")
+    return
 
-        marimo cells parse Python (and only Python), and marimo notebooks are
-        stored as pure Python files — outputs are _not_ included. There's no
-        magical syntax.
 
-        The Python files generated by marimo are:
+@app.cell
+def _(IMAGE_FOLDER, client, embedding_function, image_loader, os):
+    collection = client.create_collection(
+        name="multimodal_collection",
+        embedding_function=embedding_function,
+        data_loader=image_loader,
+        get_or_create=True,
+    )
 
-        - easily versioned with git, yielding minimal diffs
-        - legible for both humans and machines
-        - formattable using your tool of choice,
-        - usable as Python  scripts, with UI  elements taking their default
-        values, and
-        - importable by other modules (more on that in the future).
-        """
+    # Get the uris to the images
+    image_uris = sorted(
+        [
+            os.path.join(IMAGE_FOLDER, image_name)
+            for image_name in os.listdir(IMAGE_FOLDER)
+        ]
     )
-    return
+    ids = [str(i) for i in range(len(image_uris))]
+
+    collection.add(ids=ids, uris=image_uris)
+    return (collection,)
 
 
 @app.cell(hide_code=True)
-def __(mo):
+def _(mo):
     mo.md(
-        """
-        ## 4. Running notebooks as apps
-
-        marimo notebooks can double as apps. Click the app window icon in the
-        bottom-right to see this notebook in "app view."
+        r"""
+        ### Adding multi-modal data
 
-        Serve a notebook as an app with `marimo run` at the command-line.
-        Of course, you can use marimo just to level-up your
-        notebooking, without ever making apps.
+        We add image data to the collection using the image URIs. The data loader and embedding functions we specified earlier will ingest data from the provided URIs automatically.
         """
     )
     return
 
 
 @app.cell(hide_code=True)
-def __(mo):
+def _(mo):
     mo.md(
-        """
-        ## 5. The `marimo` command-line tool
-
-        **Creating and editing notebooks.** Use
-
-        ```
-        marimo edit
-        ```
-
-        in a terminal to start the marimo notebook server. From here
-        you can create a new notebook or edit existing ones.
-
-
-        **Running as apps.** Use
+        r"""
+        ## Querying a multi-modal collection
 
-        ```
-        marimo run notebook.py
-        ```
-
-        to start a webserver that serves your notebook as an app in read-only mode,
-        with code cells hidden.
-
-        **Convert a Jupyter notebook.** Convert a Jupyter notebook to a marimo
-        notebook using `marimo convert`:
-
-        ```
-        marimo convert your_notebook.ipynb > your_app.py
-        ```
-
-        **Tutorials.** marimo comes packaged with tutorials:
+        We can query the collection using text as normal, since the `OpenCLIPEmbeddingFunction` works with both text and images.
+        """
+    )
+    return
 
-        - `dataflow`: more on marimo's automatic execution
-        - `ui`: how to use UI elements
-        - `markdown`: how to write markdown, with interpolated values and
-           LaTeX
-        - `plots`: how plotting works in marimo
-        - `sql`: how to use SQL
-        - `layout`: layout elements in marimo
-        - `fileformat`: how marimo's file format works
-        - `markdown-format`: for using `.md` files in marimo
-        - `for-jupyter-users`: if you are coming from Jupyter
 
-        Start a tutorial with `marimo tutorial`; for example,
+@app.cell(hide_code=True)
+def _(mo):
+    query = mo.ui.text_area(label="Query with text", full_width=True).form(
+        bordered=False
+    )
+    mo.vstack([query, mo.md("Try: *animal* or *vehicle*")])
+    return (query,)
 
-        ```
-        marimo tutorial dataflow
-        ```
 
-        In addition to tutorials, we have examples in our
-        [our GitHub repo](https://www.github.com/marimo-team/marimo/tree/main/examples).
-        """
+@app.cell
+def _(collection, mo, query):
+    mo.stop(not query.value)
+    _retrieved = collection.query(
+        query_texts=[query.value], include=["data"], n_results=3
     )
+
+    [mo.image(img, height=200) for img in _retrieved["data"][0]]
     return
 
 
 @app.cell(hide_code=True)
-def __(mo):
+def _(mo):
     mo.md(
-        """
-        ## 6. The marimo editor
-
-        Here are some tips to help you get started with the marimo editor.
+        r"""
+        /// admonition | One more thing!
+        We can also query by images directly, by using the `query_images` field in the `collection.query` method.
+        ///
         """
     )
     return
 
 
 @app.cell
-def __(mo, tips):
-    mo.accordion(tips)
-    return
+def _(collection, mo, selected_image):
+    mo.stop(not selected_image.value)
+    import numpy as np
+    from PIL import Image
+
+    query_image = np.array(Image.open(selected_image.path()))
+    selected = mo.as_html(mo.image(query_image))
+
+    _retrieved = collection.query(
+        query_images=[query_image], include=["data"], n_results=5
+    )
+    results = [mo.image(_img) for _img in _retrieved["data"][0][1:]]
+    return results, selected
 
 
 @app.cell(hide_code=True)
-def __(mo):
-    mo.md("""## Finally, a fun fact""")
-    return
+def _(IMAGE_FOLDER, mo):
+    selected_image = mo.ui.file_browser(IMAGE_FOLDER, multiple=False)
+    selected_image
+    return (selected_image,)
 
 
 @app.cell(hide_code=True)
-def __(mo):
-    mo.md(
-        """
-        The name "marimo" is a reference to a type of algae that, under
-        the right conditions, clumps together to form a small sphere
-        called a "marimo moss ball". Made of just strands of algae, these
-        beloved assemblages are greater than the sum of their parts.
-        """
+def _(mo, results, selected):
+    mo.hstack(
+        [
+            mo.vstack([mo.md("## Selected"), selected]),
+            mo.vstack([mo.md("## Similar"), *results]),
+        ],
+        widths="equal",
+        gap=4,
     )
     return
 
 
 @app.cell(hide_code=True)
-def __():
-    tips = {
-        "Saving": (
-            """
-            **Saving**
-
-            - _Name_ your app using the box at the top of the screen, or
-              with `Ctrl/Cmd+s`. You can also create a named app at the
-              command line, e.g., `marimo edit app_name.py`.
-
-            - _Save_ by clicking the save icon on the bottom right, or by
-              inputting `Ctrl/Cmd+s`. By default marimo is configured
-              to autosave.
-            """
-        ),
-        "Running": (
-            """
-            1. _Run a cell_ by clicking the play ( ▷ ) button on the top
-            right of a cell, or by inputting `Ctrl/Cmd+Enter`.
-
-            2. _Run a stale cell_  by clicking the yellow run button on the
-            right of the cell, or by inputting `Ctrl/Cmd+Enter`. A cell is
-            stale when its code has been modified but not run.
-
-            3. _Run all stale cells_ by clicking the play ( ▷ ) button on
-            the bottom right of the screen, or input `Ctrl/Cmd+Shift+r`.
-            """
-        ),
-        "Console Output": (
-            """
-            Console output (e.g., `print()` statements) is shown below a
-            cell.
-            """
-        ),
-        "Creating, Moving, and Deleting Cells": (
-            """
-            1. _Create_ a new cell above or below a given one by clicking
-                the plus button to the left of the cell, which appears on
-                mouse hover.
-
-            2. _Move_ a cell up or down by dragging on the handle to the 
-                right of the cell, which appears on mouse hover.
-
-            3. _Delete_ a cell by clicking the trash bin icon. Bring it
-                back by clicking the undo button on the bottom right of the
-                screen, or with `Ctrl/Cmd+Shift+z`.
-            """
-        ),
-        "Disabling Automatic Execution": (
-            """
-            Via the notebook settings (gear icon) or footer panel, you
-            can disable automatic execution. This is helpful when
-            working with expensive notebooks or notebooks that have
-            side-effects like database transactions.
-            """
-        ),
-        "Disabling Cells": (
-            """
-            You can disable a cell via the cell context menu.
-            marimo will never run a disabled cell or any cells that depend on it.
-            This can help prevent accidental execution of expensive computations
-            when editing a notebook.
-            """
-        ),
-        "Code Folding": (
-            """
-            You can collapse or fold the code in a cell by clicking the arrow
-            icons in the line number column to the left, or by using keyboard
-            shortcuts.
-
-            Use the command palette (`Ctrl/Cmd+k`) or a keyboard shortcut to
-            quickly fold or unfold all cells.
-            """
-        ),
-        "Code Formatting": (
-            """
-            If you have [ruff](https://github.com/astral-sh/ruff) installed,
-            you can format a cell with the keyboard shortcut `Ctrl/Cmd+b`.
-            """
-        ),
-        "Command Palette": (
-            """
-            Use `Ctrl/Cmd+k` to open the command palette.
-            """
-        ),
-        "Keyboard Shortcuts": (
-            """
-            Open the notebook menu (top-right) or input `Ctrl/Cmd+Shift+h` to
-            view a list of all keyboard shortcuts.
-            """
-        ),
-        "Configuration": (
-            """
-           Configure the editor by clicking the gears icon near the top-right
-           of the screen.
-           """
-        ),
-    }
-    return (tips,)
+def _(mo):
+    mo.md(r"""This example was adapted from [multimodal_retrieval.ipynb](https://github.com/chroma-core/chroma/blob/main/examples/multimodal/multimodal_retrieval.ipynb), using `marimo convert`.""")
+    return
 
 
 if __name__ == "__main__":
-    app.run()
+    app.run()