Fix test

3gpp_bm25_maker.py

@@ -1,6 +1,14 @@
-import os, warnings
+import os, warnings, requests
 os.environ["CURL_CA_BUNDLE"] = ''
 from dotenv import load_dotenv
+from huggingface_hub import configure_http_backend
+def backend_factory() -> requests.Session:
+    session = requests.Session()
+    session.verify = False
+    return session
+
+configure_http_backend(backend_factory=backend_factory)
+
 warnings.filterwarnings("ignore")
 load_dotenv()
 import bm25s
@@ -8,8 +16,8 @@ from bm25s.hf import BM25HF
 from datasets import load_dataset
 unique_specs = set()
 
-dataset_text = load_dataset("OrganizedProgrammers/3GPPSpecContent", token=os.environ.get("HF_TOKEN"))
-dataset_metadata = load_dataset("OrganizedProgrammers/3GPPSpecMetadata", token=os.environ.get("HF_TOKEN"))
+dataset_text = load_dataset("OrganizedProgrammers/3GPPSpecContent")
+dataset_metadata = load_dataset("OrganizedProgrammers/3GPPSpecMetadata")
 
 dataset_text = dataset_text["train"].to_list()
 dataset_metadata = dataset_metadata["train"].to_list()

3gpp_spec_indexer.py

@@ -1,10 +1,20 @@
 import os
 import time
 import warnings
+import requests
 from dotenv import load_dotenv
 import numpy as np
 import pandas as pd
 
+from huggingface_hub import configure_http_backend
+def backend_factory() -> requests.Session:
+    session = requests.Session()
+    session.verify = False
+    return session
+
+configure_http_backend(backend_factory=backend_factory)
+
+
 warnings.filterwarnings("ignore")
 os.environ["CURL_CA_BUNDLE"] = ""
 load_dotenv()
@@ -14,7 +24,6 @@ import threading
 import zipfile
 import sys
 import subprocess
-import requests
 import re
 import traceback
 import io
@@ -26,7 +35,7 @@ DICT_LOCK = threading.Lock()
 DOCUMENT_LOCK = threading.Lock()
 STOP_EVENT = threading.Event()
 
-spec_contents = load_dataset("OrganizedProgrammers/3GPPSpecContent", token=os.environ["HF_TOKEN"])
+spec_contents = load_dataset("OrganizedProgrammers/3GPPSpecContent")
 spec_contents = spec_contents["train"].to_list()
 documents_by_spec_num = {}
 for section in spec_contents:

3gpp_tdoc_indexer.py

@@ -10,6 +10,13 @@ import re
 import concurrent.futures
 import threading
 from typing import List, Dict, Any
+from huggingface_hub import configure_http_backend
+def backend_factory() -> requests.Session:
+    session = requests.Session()
+    session.verify = False
+    return session
+
+configure_http_backend(backend_factory=backend_factory)
 
 warnings.filterwarnings("ignore")
 

etsi_bm25_maker.py

@@ -1,6 +1,13 @@
 from typing import Optional
-import os, warnings
+import os, warnings, requests
 os.environ["CURL_CA_BUNDLE"] = ''
+from huggingface_hub import configure_http_backend
+def backend_factory() -> requests.Session:
+    session = requests.Session()
+    session.verify = False
+    return session
+
+configure_http_backend(backend_factory=backend_factory)
 from dotenv import load_dotenv
 warnings.filterwarnings("ignore")
 load_dotenv()
@@ -9,8 +16,8 @@ from bm25s.hf import BM25HF
 from datasets import load_dataset
 unique_specs = set()
 
-dataset_text = load_dataset("OrganizedProgrammers/ETSISpecContent", token=os.environ.get("HF_TOKEN"))
-dataset_metadata = load_dataset("OrganizedProgrammers/ETSISpecMetadata", token=os.environ.get("HF_TOKEN"))
+dataset_text = load_dataset("OrganizedProgrammers/ETSISpecContent")
+dataset_metadata = load_dataset("OrganizedProgrammers/ETSISpecMetadata")
 
 dataset_text = dataset_text["train"].to_list()
 dataset_metadata = dataset_metadata["train"].to_list()

etsi_spec_indexer.py

@@ -3,18 +3,25 @@ import time
 import warnings
 from dotenv import load_dotenv
 import numpy as np
+import requests
 import pandas as pd
 
 warnings.filterwarnings("ignore")
 os.environ["CURL_CA_BUNDLE"] = ""
 load_dotenv()
+from huggingface_hub import configure_http_backend
+def backend_factory() -> requests.Session:
+    session = requests.Session()
+    session.verify = False
+    return session
+
+configure_http_backend(backend_factory=backend_factory)
 
 from datasets import load_dataset, Dataset
 from datasets.data_files import EmptyDatasetError
 import threading
 import zipfile
 import sys
-import requests
 import fitz
 import re
 import json
@@ -23,6 +30,7 @@ import io
 import concurrent.futures
 import hashlib
 
+
 CHARS = "0123456789abcdefghijklmnopqrstuvwxyz"
 DICT_LOCK = threading.Lock()
 DOCUMENT_LOCK = threading.Lock()