eval fine-tuned checkpoints

llm_toolkit/eval_epochs.py

@@ -3,6 +3,20 @@ import sys
 import subprocess
 from dotenv import find_dotenv, load_dotenv
 
+found_dotenv = find_dotenv(".env")
+
+if len(found_dotenv) == 0:
+    found_dotenv = find_dotenv(".env.example")
+print(f"loading env vars from: {found_dotenv}")
+load_dotenv(found_dotenv, override=False)
+
+workding_dir = os.path.dirname(found_dotenv)
+os.chdir(workding_dir)
+sys.path.append(workding_dir)
+print("workding dir:", workding_dir)
+print(f"adding {workding_dir} to sys.path")
+sys.path.append(workding_dir)
+
 from llm_toolkit.llm_utils import *
 from llm_toolkit.translation_utils import *
 
@@ -12,90 +26,80 @@ def evaluate_model_all_epochs(
     tokenizer,
     model_name,
     adapter_path_base,
-    num_of_entries=-1,
-    result_file=None,
+    dataset,
+    results_path,
     start_epoch=0,
     end_epoch=-1,
+    batch_size=1,
+    max_new_tokens=300,
+    device="cuda",
 ):
-    new_env = os.environ.copy()
-    new_env["MODEL_NAME"] = model_name
-    model = model_name.split("/")[-1]
-
-    new_env["LOAD_IN_4BIT"] = "true" if load_in_4bit else "false"
-    if result_file is not None:
-        new_env["RESULTS_PATH"] = result_file
-
     if adapter_path_base is None:
         num_train_epochs = 0
         print(f"No adapter path provided. Running with base model:{model_name}")
     else:
-        if end_epoch >= 0:
-            num_train_epochs = end_epoch
-            print(f"Running from epoch {start_epoch} to {end_epoch}")
-        else:
-            # find subdirectories in adapter_path_base
-            # and sort them by epoch number
-            subdirs = [
-                d
-                for d in os.listdir(adapter_path_base)
-                if os.path.isdir(os.path.join(adapter_path_base, d))
-            ]
-
-            subdirs = sorted(subdirs, key=lambda x: int(x.split("-")[-1]))
-            num_train_epochs = len(subdirs)
-            print(f"found {num_train_epochs} checkpoints: {subdirs}")
-
-    for i in range(start_epoch, num_train_epochs + 1):
-        print(f"Epoch {i}")
-        if i == 0:
-            os.unsetenv("ADAPTER_NAME_OR_PATH")
-        else:
-            adapter_path = adapter_path_base + "/" + subdirs[i - 1]
-            new_env["ADAPTER_NAME_OR_PATH"] = adapter_path
+        # find subdirectories in adapter_path_base
+        # and sort them by epoch number
+        subdirs = [
+            d
+            for d in os.listdir(adapter_path_base)
+            if os.path.isdir(os.path.join(adapter_path_base, d))
+        ]
 
-        print(f"adapter path: {new_env.get('ADAPTER_NAME_OR_PATH')}")
+        subdirs = sorted(subdirs, key=lambda x: int(x.split("-")[-1]))
+        num_train_epochs = len(subdirs)
+        print(f"found {num_train_epochs} checkpoints: {subdirs}")
 
-        log_file = "./logs/{}_epoch_{}.txt".format(model, i)
-        with open(log_file, "w") as f_obj:
-            subprocess.run(
-                f"python llm_toolkit/eval_shots.py {num_of_entries}",
-                shell=True,
-                env=new_env,
-                stdout=f_obj,
-                text=True,
-            )
+        if end_epoch < 0 or end_epoch > num_train_epochs:
+            end_epoch = num_train_epochs
 
+        print(f"Running from epoch {start_epoch} to {end_epoch}")
 
-if __name__ == "__main__":
-    found_dotenv = find_dotenv(".env")
+    for i in range(start_epoch, end_epoch + 1):
+        print(f"Epoch {i}")
+        if i > 0:
+            adapter_path = adapter_path_base + "/" + subdirs[i - 1]
+            print(f"loading adapter: {adapter_path}")
+            adapter_name = model.load_adapter(adapter_path)
+            model.active_adapters = adapter_name
+
+        predictions = eval_model(
+            model,
+            tokenizer,
+            dataset,
+            device=device,
+            batch_size=batch_size,
+            max_new_tokens=max_new_tokens,
+        )
+
+        model_name_with_epochs = f"{model_name}/epochs-{i:02d}"
 
-    if len(found_dotenv) == 0:
-        found_dotenv = find_dotenv(".env.example")
-    print(f"loading env vars from: {found_dotenv}")
-    load_dotenv(found_dotenv, override=False)
+        save_results(
+            model_name_with_epochs,
+            results_path,
+            dataset,
+            predictions,
+        )
+
+        metrics = calc_metrics(dataset["english"], predictions, debug=True)
+        print(f"{model_name_with_epochs} metrics: {metrics}")
 
-    workding_dir = os.path.dirname(found_dotenv)
-    os.chdir(workding_dir)
-    sys.path.append(workding_dir)
-    print("workding dir:", workding_dir)
-    print(f"adding {workding_dir} to sys.path")
-    sys.path.append(workding_dir)
 
+if __name__ == "__main__":
     model_name = os.getenv("MODEL_NAME")
     adapter_path_base = os.getenv("ADAPTER_PATH_BASE")
-    start_epoch = int(os.getenv("START_EPOCH", 0))
+    start_epoch = int(os.getenv("START_EPOCH", 1))
     end_epoch = os.getenv("END_EPOCH", -1)
     load_in_4bit = os.getenv("LOAD_IN_4BIT", "true").lower() == "true"
-    result_file = os.getenv("RESULTS_PATH", None)
-
-    num_of_entries = int(sys.argv[1]) if len(sys.argv) > 1 else -1
+    results_path = os.getenv("RESULTS_PATH", None)
+    data_path = os.getenv("DATA_PATH")
 
     print(
         model_name,
         adapter_path_base,
         load_in_4bit,
         start_epoch,
-        result_file,
+        results_path,
     )
 
     device = check_gpu()
@@ -132,11 +136,11 @@ if __name__ == "__main__":
         tokenizer,
         model_name,
         adapter_path_base,
+        datasets["test"],
+        results_path,
         start_epoch=start_epoch,
         end_epoch=end_epoch,
-        load_in_4bit=load_in_4bit,
-        num_of_entries=num_of_entries,
-        result_file=result_file,
+        device=device,
     )
 
     if is_cuda:

scripts/eval-epochs.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+BASEDIR=$(dirname "$0")
+cd $BASEDIR/..
+echo Current Directory:
+pwd
+
+export ORG_NAME=$1
+export MODEL=$2
+
+export MODEL_NAME=$ORG_NAME/$MODEL
+export ADAPTER_PATH_BASE=llama-factory/saves/$MODEL
+
+echo Evaluating $MODEL_NAME
+python llm_toolkit/eval_epochs.py

scripts/eval-mac.sh

@@ -11,16 +11,23 @@ cat /etc/os-release
 lscpu
 grep MemTotal /proc/meminfo
 
-pip install torch torchvision torchaudio
-pip install -r requirements.txt
+# pip install torch torchvision torchaudio
+# pip install -r requirements.txt
 
-export START_NUM_SHOTS=50
+# export START_NUM_SHOTS=50
 
-#./scripts/eval-model.sh internlm/internlm2_5-7b-chat
+# ./scripts/eval-model.sh internlm/internlm2_5-7b-chat
 
-#./scripts/eval-model.sh Qwen/Qwen2-7B-Instruct
+# ./scripts/eval-model.sh Qwen/Qwen2-7B-Instruct
 
-#./scripts/eval-model.sh shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat
+# ./scripts/eval-model.sh shenzhi-wang/Mistral-7B-v0.3-Chinese-Chat
 
-./scripts/eval-model.sh shenzhi-wang/Llama3.1-8B-Chinese-Chat
+# ./scripts/eval-model.sh shenzhi-wang/Llama3.1-8B-Chinese-Chat
 
+./scripts/eval-epochs.sh internlm internlm2_5-7b-chat
+
+./scripts/eval-epochs.sh Qwen Qwen2-7B-Instruct
+
+./scripts/eval-epochs.sh shenzhi-wang Mistral-7B-v0.3-Chinese-Chat
+
+./scripts/eval-epochs.sh shenzhi-wang Llama3.1-8B-Chinese-Chat