Upload 2 files

civitai_to_hf.py

@@ -13,6 +13,7 @@ from PIL import Image
 import json
 import pandas as pd
 import tempfile
+import hashlib
 
 
 TEMP_DIR = tempfile.mkdtemp()
@@ -64,6 +65,19 @@ def upload_safetensors_to_repo(filename, repo_id, repo_type, is_private, progres
     return url
 
 
+def is_same_file(filename: str, cmp_sha256: str, cmp_size: int):
+    if cmp_sha256:
+        sha256_hash = hashlib.sha256()
+        with open(filename, "rb") as f:
+            for byte_block in iter(lambda: f.read(4096), b""):
+                sha256_hash.update(byte_block)
+        sha256 = sha256_hash.hexdigest()
+    else: sha256 = ""
+    size = os.path.getsize(filename)
+    if size == cmp_size and sha256 == cmp_sha256: return True
+    else: return False
+
+
 def get_safe_filename(filename, repo_id, repo_type):
     hf_token = get_token()
     api = HfApi(token=hf_token)
@@ -71,10 +85,15 @@ def get_safe_filename(filename, repo_id, repo_type):
     try:
         i = 1
         while api.file_exists(repo_id=repo_id, filename=Path(new_filename).name, repo_type=repo_type, token=hf_token):
+            infos = api.get_paths_info(repo_id=repo_id, paths=[Path(new_filename).name], repo_type=repo_type, token=hf_token)
+            if infos and len(infos) == 1:
+                repo_fs = infos[0].size
+                repo_sha256 = infos[0].lfs.sha256 if infos[0].lfs is not None else ""
+                if is_same_file(filename, repo_sha256, repo_fs): break
             new_filename = str(Path(Path(filename).parent, f"{Path(filename).stem}_{i}{Path(filename).suffix}"))
             i += 1
         if filename != new_filename:
-            print(f"{Path(filename).name} is already exists. renaming to {Path(new_filename).name}.")
+            print(f"{Path(filename).name} is already exists but file content is different. renaming to {Path(new_filename).name}.")
             Path(filename).rename(new_filename)
     except Exception as e:
         print(f"Error occured when renaming {filename}. {e}")