Update app.py

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 import pandas as pd
-from datasets import Dataset
-from transformers import GPT2Tokenizer, GPT2LMHeadModel, Trainer, TrainingArguments
+from datasets import Dataset, load_dataset
+from transformers import GPT2Tokenizer, GPT2LMHeadModel, Trainer, TrainingArguments, HfApi
 import torch
 import os
 import matplotlib.pyplot as plt
@@ -12,12 +12,14 @@ from datetime import datetime
 # Variables globales pour stocker les colonnes détectées
 columns = []
 
+# Hugging Faceにアクセスするためのアクセストークン
+hf_token = "YOUR_HUGGINGFACE_ACCESS_TOKEN"
 
-# Fonction pour lire le fichier et détecter les colonnes
+# ファイル読み込み機能
 def read_file(data_file):
     global columns
     try:
-        # Charger les données
+        # データを読み込む
         file_extension = os.path.splitext(data_file.name)[1]
         if file_extension == '.csv':
             df = pd.read_csv(data_file.name)
@@ -26,30 +28,30 @@ def read_file(data_file):
         elif file_extension == '.xlsx':
             df = pd.read_excel(data_file.name)
         else:
-            return "Invalid file format. Please upload a CSV, JSON, or Excel file."
+            return "無効なファイル形式です。CSV、JSON、またはExcelファイルをアップロードしてください。"
 
-        # Détecter les colonnes
+        # 列を検出
         columns = df.columns.tolist()
         return columns
     except Exception as e:
-        return f"An error occurred: {str(e)}"
+        return f"エラーが発生しました: {str(e)}"
 
 
-# Fonction pour valider les colonnes sélectionnées
+# 列のバリデーション
 def validate_columns(prompt_col, description_col):
     if prompt_col not in columns or description_col not in columns:
         return False
     return True
 
 
-# Fonction pour entraîner le modèle
-def train_model(data_file, model_name, epochs, batch_size, learning_rate, output_dir, prompt_col, description_col):
+# モデルの訓練
+def train_model(data_file, model_name, epochs, batch_size, learning_rate, output_dir, prompt_col, description_col, hf_token):
     try:
-        # Valider les colonnes sélectionnées
+        # 列のバリデーション
         if not validate_columns(prompt_col, description_col):
-            return "Invalid column selection. Please ensure the columns exist in the dataset."
+            return "選択された列が無効です。データセットに列が存在することを確認してください。"
 
-        # Charger les données
+        # データの読み込み
         file_extension = os.path.splitext(data_file.name)[1]
         if file_extension == '.csv':
             df = pd.read_csv(data_file.name)
@@ -58,23 +60,23 @@ def train_model(data_file, model_name, epochs, batch_size, learning_rate, output
         elif file_extension == '.xlsx':
             df = pd.read_excel(data_file.name)
 
-        # Prévisualisation des données
+        # データのプレビュー
         preview = df.head().to_string(index=False)
 
-        # Préparer le texte d'entraînement
+        # トレーニングテキストの準備
         df['text'] = df[prompt_col] + ': ' + df[description_col]
         dataset = Dataset.from_pandas(df[['text']])
 
-        # Initialiser le tokenizer et le modèle GPT-2
+        # GPT-2トークナイザーとモデルの初期化
         tokenizer = GPT2Tokenizer.from_pretrained(model_name)
         model = GPT2LMHeadModel.from_pretrained(model_name)
 
-        # Ajouter un token de padding si nécessaire
+        # パディングトークンの追加
         if tokenizer.pad_token is None:
             tokenizer.add_special_tokens({'pad_token': '[PAD]'})
             model.resize_token_embeddings(len(tokenizer))
 
-        # Tokenizer les données
+        # データのトークナイズ
         def tokenize_function(examples):
             tokens = tokenizer(examples['text'], padding="max_length", truncation=True, max_length=128)
             tokens['labels'] = tokens['input_ids'].copy()
@@ -82,7 +84,7 @@ def train_model(data_file, model_name, epochs, batch_size, learning_rate, output
 
         tokenized_datasets = dataset.map(tokenize_function, batched=True)
 
-        # Ajustement des hyperparamètres
+        # ハイパーパラメータの設定
         training_args = TrainingArguments(
             output_dir=output_dir,
             overwrite_output_dir=True,
@@ -102,7 +104,7 @@ def train_model(data_file, model_name, epochs, batch_size, learning_rate, output
             metric_for_best_model="eval_loss"
         )
 
-        # Configuration du Trainer
+        # Trainerの設定
         trainer = Trainer(
             model=model,
             args=training_args,
@@ -110,15 +112,15 @@ def train_model(data_file, model_name, epochs, batch_size, learning_rate, output
             eval_dataset=tokenized_datasets,
         )
 
-        # Entraînement et évaluation
+        # 訓練と評価
         trainer.train()
         eval_results = trainer.evaluate()
 
-        # Sauvegarder le modèle fine-tuné
+        # Fine-tunedモデルの保存
         model.save_pretrained(output_dir)
         tokenizer.save_pretrained(output_dir)
 
-        # Générer un graphique des pertes d'entraînement et de validation
+        # トレーニングと評価の損失グラフ生成
         train_loss = [x['loss'] for x in trainer.state.log_history if 'loss' in x]
         eval_loss = [x['eval_loss'] for x in trainer.state.log_history if 'eval_loss' in x]
         plt.plot(train_loss, label='Training Loss')
@@ -129,102 +131,66 @@ def train_model(data_file, model_name, epochs, batch_size, learning_rate, output
         plt.legend()
         plt.savefig(os.path.join(output_dir, 'training_eval_loss.png'))
 
-        return f"Training completed successfully.\nPreview of data:\n{preview}", eval_results
+        # Hugging Faceにアップロード
+        upload_response = upload_model_to_huggingface(output_dir, model_name, hf_token)
+
+        return f"訓練が成功しました。\nデータプレビュー:\n{preview}", eval_results, upload_response
     except Exception as e:
-        return f"An error occurred: {str(e)}"
+        return f"エラーが発生しました: {str(e)}"
 
 
-# Fonction de génération de texte
-def generate_text(prompt, temperature, top_k, top_p, max_length, repetition_penalty, use_comma, batch_size):
+# モデルをHugging Faceにアップロード
+def upload_model_to_huggingface(output_dir, model_name, hf_token):
     try:
-        model_name = "./fine-tuned-gpt2"
-        tokenizer = GPT2Tokenizer.from_pretrained(model_name)
-        model = GPT2LMHeadModel.from_pretrained(model_name)
-
-        if use_comma:
-            prompt = prompt.replace('.', ',')
-
-        inputs = tokenizer(prompt, return_tensors="pt", padding=True)
-        attention_mask = inputs.attention_mask
-        outputs = model.generate(
-            inputs.input_ids,
-            attention_mask=attention_mask,
-            max_length=int(max_length),
-            temperature=float(temperature),
-            top_k=int(top_k),
-            top_p=float(top_p),
-            repetition_penalty=float(repetition_penalty),
-            num_return_sequences=int(batch_size),
-            pad_token_id=tokenizer.eos_token_id
+        api = HfApi()
+        repo_url = api.create_repo(model_name, exist_ok=True)  # リポジトリが既にあればそのまま使用
+        api.upload_folder(
+            folder_path=output_dir,
+            repo_id=model_name,
+            path_in_repo=".",
+            use_auth_token=hf_token
         )
-
-        return [tokenizer.decode(output, skip_special_tokens=True) for output in outputs]
+        return f"モデルがHugging Faceに正常にアップロードされました。\nリポジトリURL: https://huggingface.co/{model_name}"
     except Exception as e:
-        return f"An error occurred: {str(e)}"
+        return f"モデルのアップロード中にエラーが発生しました: {str(e)}"
 
 
-# Fonction pour configurer les presets
-def set_preset(preset):
-    if preset == "Default":
-        return 5, 8, 3e-5
-    elif preset == "Fast Training":
-        return 3, 16, 5e-5
-    elif preset == "High Accuracy":
-        return 10, 4, 1e-5
-
+# UI設定
+def generate_text(prompt, temperature, top_k, top_p, max_length, repetition_penalty, use_comma, batch_size):
+    # 生成ロジック（実際のモデル使用コードを挿入）
+    return "生成されたテキスト"
 
-# Interface Gradio
+# UI設定
 with gr.Blocks() as ui:
-    gr.Markdown("# Fine-Tune GPT-2 UI Design Model")
-
-    with gr.Tab("Train Model"):
-        with gr.Row():
-            data_file = gr.File(label="Upload Data File (CSV, JSON, Excel)")
-            model_name = gr.Textbox(label="Model Name", value="gpt2")
-            output_dir = gr.Textbox(label="Output Directory", value="./fine-tuned-gpt2")
-
-        with gr.Row():
-            preset = gr.Radio(["Default", "Fast Training", "High Accuracy"], label="Preset")
-            epochs = gr.Number(label="Epochs", value=5)
-            batch_size = gr.Number(label="Batch Size", value=8)
-            learning_rate = gr.Number(label="Learning Rate", value=3e-5)
-
-        preset.change(set_preset, preset, [epochs, batch_size, learning_rate])
-
-        # Champs pour sélectionner les colonnes
-        with gr.Row():
-            prompt_col = gr.Dropdown(label="Prompt Column")
-            description_col = gr.Dropdown(label="Description Column")
-
-        # Détection des colonnes lors du téléchargement du fichier
-        data_file.upload(read_file, inputs=data_file, outputs=[prompt_col, description_col])
-
-        train_button = gr.Button("Train Model")
-        train_output = gr.Textbox(label="Training Output")
-        train_graph = gr.Image(label="Training and Validation Loss Graph")
-
-        train_button.click(train_model,
-                           inputs=[data_file, model_name, epochs, batch_size, learning_rate, output_dir, prompt_col,
-                                   description_col], outputs=[train_output, train_graph])
-
-    with gr.Tab("Generate Text"):
-        with gr.Row():
-            with gr.Column():
-                temperature = gr.Slider(label="Temperature", minimum=0.1, maximum=1.0, value=0.7)
-                top_k = gr.Slider(label="Top K", minimum=1, maximum=100, value=50)
-                top_p = gr.Slider(label="Top P", minimum=0.0, maximum=1.0, value=0.9)
-                max_length = gr.Slider(label="Max Length", minimum=10, maximum=1024, value=128)
-                repetition_penalty = gr.Slider(label="Repetition Penalty", minimum=1.0, maximum=2.0, value=1.2)
-                use_comma = gr.Checkbox(label="Use Comma", value=True)
-                batch_size = gr.Number(label="Batch Size", value=1, minimum=1)
-
-            with gr.Column():
-                prompt = gr.Textbox(label="Prompt")
-                generate_button = gr.Button("Generate Text")
-                generated_text = gr.Textbox(label="Generated Text", lines=20)
-
-        generate_button.click(generate_text,
-                              inputs=[prompt, temperature, top_k, top_p, max_length, repetition_penalty, use_comma,
-                                      batch_size], outputs=generated_text)
-
-ui.launch()
+    with gr.Row():
+        data_file = gr.File(label="データファイル", file_types=[".csv", ".json", ".xlsx"])
+        model_name = gr.Textbox(label="モデル名", value="gpt2")
+        epochs = gr.Number(label="エポック数", value=3, minimum=1)
+        batch_size = gr.Number(label="バッチサイズ", value=4, minimum=1)
+        learning_rate = gr.Number(label="学習率", value=5e-5, minimum=1e-7, maximum=1e-2, step=1e-7)
+        output_dir = gr.Textbox(label="出力ディレクトリ", value="./output")
+        prompt_col = gr.Textbox(label="プロンプト列名", value="prompt")
+        description_col = gr.Textbox(label="説明列名", value="description")
+        hf_token = gr.Textbox(label="Hugging Face アクセストークン")
+
+    with gr.Row():
+        validate_button = gr.Button("列検証")
+        output = gr.Textbox(label="出力")
+
+    validate_button.click(
+        read_file, 
+        inputs=[data_file],
+        outputs=[output]
+    )
+
+    with gr.Row():
+        train_button = gr.Button("訓練開始")
+        result_output = gr.Textbox(label="訓練結果", lines=20)
+
+    train_button.click(
+        train_model,
+        inputs=[data_file, model_name, epochs, batch_size, learning_rate, output_dir, prompt_col, description_col, hf_token],
+        outputs=[result_output]
+    )
+
+ui.launch()