Refactor dataset migration tool for GitHub and Kaggle datasets

app.py

@@ -2,13 +2,15 @@ import contextlib
 import re
 import tempfile
 from functools import lru_cache
+from typing import Optional
 
 import gradio as gr
 from git import Repo
 from httpx import Client
-from typing import Optional
 from huggingface_hub import create_repo, upload_folder
 from toolz import groupby
+import kagglehub
+from kagglehub import KaggleDatasetAdapter
 
 client = Client()
 
@@ -46,7 +48,7 @@ def upload_directory_to_hf(
         commit_message="Migrated from GitHub",
         ignore_patterns=[
             "*.git*",
-           # "*README.md*",
+            # "*README.md*",
             "*.DS_Store",
             "*.env",
         ],  # ignore git files and .env files
@@ -132,6 +134,34 @@ def show_files_and_directories(url: str):
         )
 
 
+def push_kaggle_to_hf(
+    source_kaggle_dataset: str,
+    destination_hf_hub_repository: str,
+    file_path: str,
+    oauth_token: gr.OAuthToken,
+):
+    """Pushes a Kaggle dataset to HuggingFace Hub using the HF dataset adapter"""
+    if not file_path:
+        raise ValueError("File path must be specified for Kaggle datasets")
+
+    gr.Info("Loading Kaggle dataset...")
+    dataset = kagglehub.load_dataset(
+        KaggleDatasetAdapter.HUGGING_FACE,
+        source_kaggle_dataset,
+        file_path,
+    )
+    gr.Info("Loading Kaggle dataset...Done")
+
+    gr.Info("Pushing to Hugging Face Hub...")
+    dataset.push_to_hub(
+        destination_hf_hub_repository,
+        token=oauth_token.token,
+    )
+    gr.Info("Pushing to Hugging Face Hub...Done")
+
+    return f"Pushed the dataset to [{destination_hf_hub_repository}](https://huggingface.co/datasets/{destination_hf_hub_repository})"
+
+
 html_text_app_description = """
 Whilst GitHub is great for hosting code the Hugging Face Datasets Hub is a better place to host datasets. 
 Some of the benefits of hosting datasets on the Hugging Face Datasets Hub are:
@@ -148,71 +178,101 @@ This app will help you migrate a dataset currently hosted on GitHub to the Huggi
 
 with gr.Blocks(theme=gr.themes.Base()) as demo:
     gr.HTML(
-        """<h1 style='text-align: center;'> GitHub to Hugging Face Hub Dataset Migration Tool</h1>
-        <center><i> &#x2728; Migrate a dataset in a few steps &#x2728;</i></center>"""
-    )
-    gr.HTML(
-        """<center> GitHub is a great place for sharing code but the Hugging Face Hub has many advantages for sharing datasets. 
-        <br> This Space will guide you through the process of migrating a dataset from GitHub to the Hugging Face Hub. </center>"""
+        """<h1 style='text-align: center;'> Dataset Migration Tool</h1>
+        <center><i> &#x2728; Migrate datasets to Hugging Face Hub in a few steps &#x2728;</i></center>"""
     )
+
     with gr.Row():
         gr.LoginButton(size="sm")
-         
-    gr.Markdown("### Location of existing dataset")
-    gr.Markdown("URL for the GitHub repository where the dataset is currently hosted")
-    source_github_repository = gr.Textbox(lines=1, label="Source GitHub Repository URL")
-    gr.Markdown(
-        "Use advanced options to select specific files and folders to migrate. Currently this app supports migrating specific subfolder(s) or top level files. If this is not sufficient for your use case please open a discussion!"
-    )
-    with gr.Accordion("Advanced Options", open=False):
-        gr.Markdown("### Select files and folder to migrate")
-        gr.Markdown(
-            "(Optional): select a specific folder and/or files to migrate from the GitHub repository. If you select a folder all the files in that folder will be migrated."
-        )
-        folder_in_github_repo = gr.Dropdown(
-            None,
-            label="Folder in the GitHub Repository to migrate",
-            allow_custom_value=True,
-            visible=True,
-        )
-        files_in_github_repo = gr.Dropdown(
-            None,
-            label="Files in GitHub Repository to migrate",
-            allow_custom_value=True,
-            visible=True,
-        )
-        source_github_repository.change(
-            show_files_and_directories,
-            [source_github_repository],
-            [folder_in_github_repo, files_in_github_repo],
-        )
-    gr.Markdown("### Destination for your migrated dataset")
-    gr.Markdown("Destination repository for your dataset on the Hugging Face Hub")
-    destination_hf_hub_repository = gr.Textbox(
-        label="Destination Hugging Face Repository",
-        placeholder="i.e. <hugging face username>/<repository_name>",
-    )
-    # gr.Markdown("## Authentication")
-    # gr.Markdown(
-    #     """You need to provide a token with write access to the namespace you want to upload to.
-    #             You can generate/access your Hugging FAce token from [here](https://huggingface.co/settings/token)."""
-    # )
-    # hf_token = gr.Textbox(label="Hugging Face Token", type="password")
-    summit_btn = gr.Button("Migrate Dataset")
-    result = gr.Markdown(label="Summary", visible=True)
-    summit_btn.click(
-        push_to_hf,
-        [
-            source_github_repository,
-            destination_hf_hub_repository,
-            folder_in_github_repo,
-        ],
-        [result],
-    )
+
+    with gr.Tabs() as tabs:
+        with gr.Tab("GitHub"):
+            gr.Markdown("### Location of existing dataset")
+            gr.Markdown(
+                "URL for the GitHub repository where the dataset is currently hosted"
+            )
+            source_github_repository = gr.Textbox(
+                lines=1, label="Source GitHub Repository URL"
+            )
+
+            with gr.Accordion("Advanced Options", open=False):
+                gr.Markdown("### Select files and folder to migrate")
+                gr.Markdown(
+                    "(Optional): select a specific folder and/or files to migrate from the GitHub repository. If you select a folder all the files in that folder will be migrated."
+                )
+                folder_in_github_repo = gr.Dropdown(
+                    None,
+                    label="Folder in the GitHub Repository to migrate",
+                    allow_custom_value=True,
+                    visible=True,
+                )
+                files_in_github_repo = gr.Dropdown(
+                    None,
+                    label="Files in GitHub Repository to migrate",
+                    allow_custom_value=True,
+                    visible=True,
+                )
+                source_github_repository.change(
+                    show_files_and_directories,
+                    [source_github_repository],
+                    [folder_in_github_repo, files_in_github_repo],
+                )
+
+            gr.Markdown("### Destination for your migrated dataset")
+            destination_hf_hub_repository = gr.Textbox(
+                label="Destination Hugging Face Repository",
+                placeholder="i.e. <hugging face username>/<repository_name>",
+            )
+
+            github_submit_btn = gr.Button("Migrate GitHub Dataset")
+            github_result = gr.Markdown(label="Summary", visible=True)
+
+            github_submit_btn.click(
+                push_to_hf,
+                [
+                    source_github_repository,
+                    destination_hf_hub_repository,
+                    folder_in_github_repo,
+                ],
+                [github_result],
+            )
+
+        with gr.Tab("Kaggle"):
+            gr.Markdown("### Source Kaggle Dataset")
+            gr.Markdown("Enter the Kaggle dataset name and file path")
+            source_kaggle_dataset = gr.Textbox(
+                lines=1,
+                label="Source Kaggle Dataset",
+                placeholder="username/dataset-name",
+            )
+            kaggle_file_path = gr.Textbox(
+                label="File path in dataset",
+                placeholder="e.g., train.csv",
+                info="Specify the file to migrate from the dataset",
+            )
+
+            gr.Markdown("### Destination for your migrated dataset")
+            kaggle_destination_hf_hub = gr.Textbox(
+                label="Destination Hugging Face Repository",
+                placeholder="i.e. <hugging face username>/<repository_name>",
+            )
+
+            kaggle_submit_btn = gr.Button("Migrate Kaggle Dataset")
+            kaggle_result = gr.Markdown(label="Summary", visible=True)
+
+            kaggle_submit_btn.click(
+                push_kaggle_to_hf,
+                [
+                    source_kaggle_dataset,
+                    kaggle_destination_hf_hub,
+                    kaggle_file_path,
+                ],
+                [kaggle_result],
+            )
+
     gr.Markdown(
         """You should add a dataset card for your dataset to help people discover and understand your dataset. You can find instructions for creating a dataset card [here](https://huggingface.co/docs/datasets/dataset_card). 
         If you have any questions or feedback feel free to reach out to us on using the [Discussion tab](https://huggingface.co/spaces/librarian-bots/github-to-huggingface-dataset-migration-tool/discussions/1)"""
     )
 
-
 demo.launch()