update loading for new huggingface_hub

mgr_biases.py

@@ -6,6 +6,7 @@ import re
 import pandas as pd
 import numpy as np
 import glob
+from pathlib import Path
 import huggingface_hub
 print("hfh", huggingface_hub.__version__)
 from huggingface_hub import hf_hub_download, upload_file, delete_file, snapshot_download, list_repo_files, dataset_info
@@ -181,6 +182,19 @@ def retrievePredefinedBiases():
 
   return flt_files
 
+def list_folders_sorted_by_date(path):
+  # Convert string path to a Path object
+  directory = Path(path)
+  
+  # Get all folders in the given directory
+  folders = [f for f in directory.iterdir() if f.is_dir()]
+  
+  # Sort folders by modification time, most recent first
+  sorted_folders = sorted(folders, key=lambda x: x.stat().st_mtime, reverse=True)
+  
+  # Return folder names
+  return [folder.name for folder in sorted_folders]
+
 # https://huggingface.co/spaces/elonmuskceo/persistent-data/blob/main/app.py
 def get_bias_json(filepath: str):
   filename = os.path.basename(filepath)
@@ -198,6 +212,13 @@ def get_bias_json(filepath: str):
     # file not found
     print(f"file not found, probably: {e}")
   
+  ds_local_path = os.path.join(LOCAL_DATA_DIRNAME,
+                               "datasets--AnimaLab--bias-test-gpt-biases",
+                               "snapshots")
+  folders_sorted = list_folders_sorted_by_date(ds_local_path)
+  print("---FOLDERS---")
+  print(folders_sorted[0])
+    
   with open(os.path.join(LOCAL_DATA_DIRNAME, filename)) as f:
     bias_json = json.load(f)