Update utils.py

utils.py

@@ -17,6 +17,38 @@ from constants import (
 api = HfApi(token=HF_TOKEN)
 
 
+OPEN_LICENSE_KEYWORDS = {
+    "mit", "apache", "apache-2", "apache-2.0",
+    "bsd", "bsd-2", "bsd-3", "bsd-2-clause", "bsd-3-clause",
+    "isc", "mpl", "mpl-2.0",
+    "lgpl", "lgpl-2.1", "lgpl-3.0",
+    "gpl", "gpl-2.0", "gpl-3.0", "agpl", "agpl-3.0",
+    "epl", "epl-2.0", "cddl", "cddl-1.0", "cddl-1.1",
+    "bsl", "bsl-1.0", "boost", "zlib", "unlicense", "artistic-2.0",
+    "cc0", "cc0-1.0",
+    "cc-by", "cc-by-3.0", "cc-by-4.0",
+    "cc-by-sa", "cc-by-sa-3.0", "cc-by-sa-4.0",
+    "openrail", "openrail-m", "bigscience openrail", "bigscience openrail-m",
+    "open-source", "opensource", "open source"
+}
+
+RESTRICTIVE_LICENSE_KEYWORDS = {
+    "cc-by-nc", "cc-by-nc-sa", "cc-nc", "nc-sa", "nc-nd",
+    "cc-by-nd", "cc-nd", "no-derivatives", "no derivatives",
+    "non-commercial", "noncommercial", "research-only", "research only",
+    "llama", "llama-2", "community license",
+    "proprietary", "closed", "unknown", "custom"
+}
+
+def is_open_license(license_str: str) -> bool:
+    s = (str(license_str) if license_str is not None else "").strip().lower()
+    if not s:
+        return False
+    if any(pat in s for pat in RESTRICTIVE_LICENSE_KEYWORDS):
+        return False
+    return any(pat in s for pat in OPEN_LICENSE_KEYWORDS)
+
+
 def init_repo():
     try:
         api.repo_info(REPO_ID, repo_type="dataset")
@@ -62,13 +94,7 @@ def load_data():
         )
         df = df.drop(columns=["link"])
 
-        df["license"] = df["license"].apply(
-            lambda x: "Открытая"
-            if any(
-                term in x.lower() for term in ["mit", "apache", "bsd", "gpl", "open"]
-            )
-            else "Закрытая"
-        )
+        df["license"] = df["license"].apply(lambda x: "Открытая" if is_open_license(x) else "Закрытая")
 
         df["rank"] = df["rank"].apply(
             lambda r: "🥇" if r == 1 else "🥈" if r == 2 else "🥉" if r == 3 else str(r)