Upload 4 files

app.py

@@ -0,0 +1,111 @@
+from pathlib import Path
+
+import gradio as gr
+
+from app_1M_image import get_demo as get_demo_1M_image
+from app_image import get_demo as get_demo_image
+from app_json import get_demo as get_demo_json
+from huggingface_hub import logging
+
+
+logging.set_verbosity_debug()
+
+
+def _get_demo_code(path: str) -> str:
+    code = Path(path).read_text()
+    code = code.replace("def get_demo():", "with gr.Blocks() as demo:")
+    code += "\n\ndemo.launch()"
+    return code
+
+
+DEMO_EXPLANATION = """
+<h1 style='text-align: center; margin-bottom: 1rem'> How to persist data from a Space to a Dataset? </h1>
+
+This demo shows how to leverage both `gradio` and `huggingface_hub` to save data from a Space to a Dataset on the Hub.
+When doing so, a few things must be taken care of: file formats, concurrent writes, name collision, number of commits,
+number of files,... The tabs below shows different ways of implementing a "save to dataset" feature. Depending on the
+complexity and usage of your app, you might want to use one or the other.
+
+This Space demo comes as a pair with this guide. If you need more technical details, please refer to it.
+"""
+
+JSON_DEMO_EXPLANATION = """
+## Use case
+
+- Save inputs and outputs
+- Build an annotation platform
+
+## Data
+
+Json-able only: text and numeric but no binaries.
+
+## Robustness
+
+Works with concurrent users and replicas.
+
+## Limitations
+
+if you expect millions of lines, you will need to split the local JSON file into multiple files to avoid getting your file tracked as LFS (5MB) on the Hub.
+
+## Demo
+"""
+
+IMAGE_DEMO_EXPLANATION = """
+## Use case
+
+Save images with metadata (caption, parameters, datetime,...).
+
+## Robustness
+
+Works with concurrent users and replicas.
+
+## Limitations
+
+  - only 10k images/folder supported on the Hub. If you expect more usage, you must save data in subfolders.
+  - only 1M images/repo supported on the Hub. If you expect more usage, you can zip your data before upload. See the _1M images Dataset_ demo.
+
+## Demo
+"""
+
+IMAGE_1M_DEMO_EXPLANATION = """
+## Use case:
+
+Same as _Image Dataset_ example, but with very high usage expected.
+
+## Robustness
+
+Works with concurrent users and replicas.
+
+## Limitations
+
+None.
+
+## Demo
+"""
+
+with gr.Blocks() as demo:
+    gr.Markdown(DEMO_EXPLANATION)
+
+    with gr.Tab("JSON Dataset"):
+        gr.Markdown(JSON_DEMO_EXPLANATION)
+        get_demo_json()
+        gr.Markdown("## Result\n\nhttps://huggingface.co/datasets/Wauplin/example-commit-scheduler-json\n\n## Code")
+        with gr.Accordion("Source code", open=True):
+            gr.Code(_get_demo_code("app_json.py"), language="python")
+
+    with gr.Tab("Image Dataset"):
+        gr.Markdown(IMAGE_DEMO_EXPLANATION)
+        get_demo_image()
+        gr.Markdown("## Result\n\nhttps://huggingface.co/datasets/Wauplin/example-commit-scheduler-image\n\n## Code")
+        with gr.Accordion("Source code", open=True):
+            gr.Code(_get_demo_code("app_image.py"), language="python")
+
+    with gr.Tab("1M images Dataset"):
+        gr.Markdown(IMAGE_1M_DEMO_EXPLANATION)
+        get_demo_1M_image()
+        gr.Markdown(
+            "## Result\n\nhttps://huggingface.co/datasets/Wauplin/example-commit-scheduler-image-zip\n\n## Code"
+        )
+        with gr.Accordion("Source code", open=True):
+            gr.Code(_get_demo_code("app_1M_image.py"), language="python")
+demo.launch()

app_1M_image.py

@@ -0,0 +1,112 @@
+import json
+import tempfile
+import zipfile
+from datetime import datetime
+from pathlib import Path
+from uuid import uuid4
+
+import gradio as gr
+import numpy as np
+from PIL import Image
+
+from huggingface_hub import CommitScheduler, InferenceClient
+
+
+IMAGE_DATASET_DIR = Path("image_dataset_1M") / f"train-{uuid4()}"
+
+IMAGE_DATASET_DIR.mkdir(parents=True, exist_ok=True)
+IMAGE_JSONL_PATH = IMAGE_DATASET_DIR / "metadata.jsonl"
+
+
+class ZipScheduler(CommitScheduler):
+    """
+    Example of a custom CommitScheduler with overwritten `push_to_hub` to zip images before pushing them to the Hub.
+
+    Workflow:
+    1. Read metadata + list PNG files.
+    2. Zip png files in a single archive.
+    3. Create commit (metadata + archive).
+    4. Delete local png files to avoid re-uploading them later.
+
+    Only step 1 requires to activate the lock. Once the metadata is read, the lock is released and the rest of the
+    process can be done without blocking the Gradio app.
+    """
+
+    def push_to_hub(self):
+        # 1. Read metadata + list PNG files
+        with self.lock:
+            png_files = list(self.folder_path.glob("*.png"))
+            if len(png_files) == 0:
+                return None  # return early if nothing to commit
+
+            # Read and delete metadata file
+            metadata = IMAGE_JSONL_PATH.read_text()
+            try:
+                IMAGE_JSONL_PATH.unlink()
+            except Exception:
+                pass
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # 2. Zip png files + metadata in a single archive
+            archive_path = Path(tmpdir) / "train.zip"
+            with zipfile.ZipFile(archive_path, "w", zipfile.ZIP_DEFLATED) as zip:
+                # PNG files
+                for png_file in png_files:
+                    zip.write(filename=png_file, arcname=png_file.name)
+
+                # Metadata
+                tmp_metadata = Path(tmpdir) / "metadata.jsonl"
+                tmp_metadata.write_text(metadata)
+                zip.write(filename=tmp_metadata, arcname="metadata.jsonl")
+
+            # 3. Create commit
+            self.api.upload_file(
+                repo_id=self.repo_id,
+                repo_type=self.repo_type,
+                revision=self.revision,
+                path_in_repo=f"train-{uuid4()}.zip",
+                path_or_fileobj=archive_path,
+            )
+
+        # 4. Delete local png files to avoid re-uploading them later
+        for png_file in png_files:
+            try:
+                png_file.unlink()
+            except Exception:
+                pass
+
+
+scheduler = ZipScheduler(
+    repo_id="example-commit-scheduler-image-zip",
+    repo_type="dataset",
+    folder_path=IMAGE_DATASET_DIR,
+)
+
+client = InferenceClient()
+
+
+def generate_image(prompt: str) -> Image:
+    return client.text_to_image(prompt)
+
+
+def save_image(prompt: str, image_array: np.ndarray) -> None:
+    print("Saving: " + prompt)
+    image_path = IMAGE_DATASET_DIR / f"{uuid4()}.png"
+
+    with scheduler.lock:
+        Image.fromarray(image_array).save(image_path)
+        with IMAGE_JSONL_PATH.open("a") as f:
+            json.dump({"prompt": prompt, "file_name": image_path.name, "datetime": datetime.now().isoformat()}, f)
+            f.write("\n")
+
+
+def get_demo():
+    with gr.Row():
+        prompt_value = gr.Textbox(label="Prompt")
+        image_value = gr.Image(label="Generated image")
+    text_to_image_btn = gr.Button("Generate")
+    text_to_image_btn.click(fn=generate_image, inputs=prompt_value, outputs=image_value).success(
+        fn=save_image,
+        inputs=[prompt_value, image_value],
+        outputs=None,
+    )

app_image.py

@@ -0,0 +1,50 @@
+import json
+from datetime import datetime
+from pathlib import Path
+from uuid import uuid4
+
+import gradio as gr
+import numpy as np
+from PIL import Image
+
+from huggingface_hub import CommitScheduler, InferenceClient
+
+
+IMAGE_DATASET_DIR = Path("image_dataset") / f"train-{uuid4()}"
+IMAGE_DATASET_DIR.mkdir(parents=True, exist_ok=True)
+IMAGE_JSONL_PATH = IMAGE_DATASET_DIR / "metadata.jsonl"
+
+scheduler = CommitScheduler(
+    repo_id="example-commit-scheduler-image",
+    repo_type="dataset",
+    folder_path=IMAGE_DATASET_DIR,
+    path_in_repo=IMAGE_DATASET_DIR.name,
+)
+
+client = InferenceClient()
+
+
+def generate_image(prompt: str) -> Image:
+    return client.text_to_image(prompt)
+
+
+def save_image(prompt: str, image_array: np.ndarray) -> None:
+    image_path = IMAGE_DATASET_DIR / f"{uuid4()}.png"
+
+    with scheduler.lock:
+        Image.fromarray(image_array).save(image_path)
+        with IMAGE_JSONL_PATH.open("a") as f:
+            json.dump({"prompt": prompt, "file_name": image_path.name, "datetime": datetime.now().isoformat()}, f)
+            f.write("\n")
+
+
+def get_demo():
+    with gr.Row():
+        prompt_value = gr.Textbox(label="Prompt")
+        image_value = gr.Image(label="Generated image")
+    text_to_image_btn = gr.Button("Generate")
+    text_to_image_btn.click(fn=generate_image, inputs=prompt_value, outputs=image_value).success(
+        fn=save_image,
+        inputs=[prompt_value, image_value],
+        outputs=None,
+    )

app_json.py

@@ -0,0 +1,44 @@
+import json
+from datetime import datetime
+from pathlib import Path
+from uuid import uuid4
+
+import gradio as gr
+
+from huggingface_hub import CommitScheduler
+
+
+JSON_DATASET_DIR = Path("json_dataset")
+JSON_DATASET_DIR.mkdir(parents=True, exist_ok=True)
+
+JSON_DATASET_PATH = JSON_DATASET_DIR / f"train-{uuid4()}.json"
+
+scheduler = CommitScheduler(
+    repo_id="example-commit-scheduler-json",
+    repo_type="dataset",
+    folder_path=JSON_DATASET_DIR,
+    path_in_repo="data",
+)
+
+
+def greet(name: str) -> str:
+    return "Hello " + name + "!"
+
+
+def save_json(name: str, greetings: str) -> None:
+    with scheduler.lock:
+        with JSON_DATASET_PATH.open("a") as f:
+            json.dump({"name": name, "greetings": greetings, "datetime": datetime.now().isoformat()}, f)
+            f.write("\n")
+
+
+def get_demo():
+    with gr.Row():
+        greet_name = gr.Textbox(label="Name")
+        greet_output = gr.Textbox(label="Greetings")
+    greet_btn = gr.Button("Greet")
+    greet_btn.click(fn=greet, inputs=greet_name, outputs=greet_output).success(
+        fn=save_json,
+        inputs=[greet_name, greet_output],
+        outputs=None,
+    )