[BETA] Add GlobalPlatorm support

app.py

@@ -6,10 +6,22 @@ from nltk.stem import WordNetLemmatizer
 from dotenv import load_dotenv
 from sklearn.preprocessing import MinMaxScaler
 
-os.environ['CURL_CA_BUNDLE'] = ""
+from bs4 import BeautifulSoup
+import requests
+from urllib.parse import parse_qs, urlparse
+
 warnings.filterwarnings('ignore')
 nltk.download('wordnet')
 load_dotenv()
+os.environ['CURL_CA_BUNDLE'] = ""
+
+from huggingface_hub import configure_http_backend
+def backend_factory() -> requests.Session:
+    session = requests.Session()
+    session.verify = False
+    return session
+
+configure_http_backend(backend_factory=backend_factory)
 
 from datasets import load_dataset
 import bm25s
@@ -23,9 +35,6 @@ from fastapi.staticfiles import StaticFiles
 from schemas import *
 from classes import *
 
-from bs4 import BeautifulSoup
-import requests
-
 lemmatizer = WordNetLemmatizer()
 
 spec_metadatas_3gpp = load_dataset("OrganizedProgrammers/3GPPSpecMetadata")
@@ -41,9 +50,55 @@ spec_contents_etsi = spec_contents_etsi["train"].to_list()
 spec_metadatas_etsi = spec_metadatas_etsi["train"].to_list()
 tdoc_locations = tdoc_locations_3gpp["train"].to_list()
 
-bm25_index_3gpp = BM25HF.load_from_hub("OrganizedProgrammers/3GPPBM25IndexSingle", load_corpus=True, token=os.environ["HF_TOKEN"])
+bm25_index_3gpp = BM25HF.load_from_hub("OrganizedProgrammers/3GPPBM25IndexSingle", load_corpus=True, token=os.environ["HF_TOKEN"], )
 bm25_index_etsi = BM25HF.load_from_hub("OrganizedProgrammers/ETSIBM25IndexSingle", load_corpus=True, token=os.environ["HF_TOKEN"])
 
+def extract_args_and_map(href):
+    if not href or not href.lower().startswith('javascript:'):
+        return None
+    js = href[len('javascript:'):].strip()
+    m = re.match(r'\w+\((.*)\)', js)
+    if not m:
+        return None
+    args_str = m.group(1).strip()
+    parts = [part.strip() for part in args_str.split(',', 1)]
+    if len(parts) != 2:
+        return None
+    try:
+        media_id = int(parts[0])
+    except ValueError:
+        return None
+    spec_type = parts[1].strip()
+    if (spec_type.startswith("'") and spec_type.endswith("'")) or (spec_type.startswith('"') and spec_type.endswith('"')):
+        spec_type = spec_type[1:-1]
+
+    return media_id, spec_type
+
+url = "https://globalplatform.org/wp-content/themes/globalplatform/ajax/specs-library.php"
+headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"}
+resp = requests.post(url, verify=False, headers=headers)
+soup = BeautifulSoup(resp.text, 'html.parser')
+
+panels = soup.find_all('div', class_='panel panel-default')
+gp_spec_locations = {}
+for panel in panels:
+  header = ''.join([t for t in panel.find('a').children if t.name is None]).strip()
+  try:
+    title, doc_id = header.split(' | ')
+    panel_body = panel.find('div', class_='panel-body')
+
+    download_btn_href = panel_body.find_all('a', href=lambda href: href and href.strip().lower().startswith('javascript:'))[0]
+    media_id, spec_type = extract_args_and_map(download_btn_href['href'])
+
+    changes_history = panel.find_all('div', class_="row")
+    paragraphs_ch = [version.find('p').text for version in changes_history][::-1]
+    document_commits = []
+    for version in range(len(paragraphs_ch)):
+       document_commits.append(f"Version {version + 1} : {paragraphs_ch[version]}")
+    gp_spec_locations[doc_id] = {"title": title, "file_id": media_id, "committee": spec_type, "summary": "\n".join(document_commits)}
+  except:
+    continue
+
 def get_docs_from_url(url):
     """Get list of documents/directories from a URL"""
     try:
@@ -74,6 +129,22 @@ def get_document(spec_id: str, spec_title: str, source: str):
             text.extend([section['section'], section['content']])
     return text
 
+def get_gp_spec_url(data):
+    file_id = data['file_id']
+    spec_type = data['committee']
+
+    url = "https://globalplatform.org/wp-content/themes/globalplatform/ajax/download-spec-submit.php"
+    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"}
+    resp = requests.post(url, verify=False, headers=headers, data={"first_name": "", "last_name": "", "company": "", "email": "", "media_id": file_id, "spec_type": spec_type, "agree": "true"})
+
+    r = resp.text
+    mat = re.search(r"window\.location\.href\s*=\s*'([^']+)'", r)
+    if mat:
+        full_url = mat.group(1)
+        parsed_url = urlparse(full_url)
+        query_params = parse_qs(parsed_url.query)
+        return query_params.get('f')[0]
+
 tags_metadata = [
     {
         "name": "Document Retrieval",
@@ -153,11 +224,21 @@ def frontend():
 def find_document(request: DocRequest):
     start_time = time.time()
     document = request.doc_id
+    if valid_3gpp_doc_format.match(document):
+        url = get_tdoc_url(document)
+    elif valid_3gpp_spec_format.match(document):
+        url = get_spec_url(document)
+    elif valid_etsi_doc_format.match(document):
+        etsi_doc_finder.search_document(document)
+    elif valid_etsi_spec_format.match(document):
+        etsi_spec_finder.search_document(document)
+    elif document.startswith("GP"):
+        for sp in gp_spec_locations:
+            if document.lower() in sp.lower():
+                url = get_gp_spec_url(gp_spec_locations[sp])
+    else:
+        url = "Document ID not supported"
 
-    url = get_tdoc_url(document) if valid_3gpp_doc_format.match(document) else \
-        get_spec_url(document) if valid_3gpp_spec_format.match(document) else \
-        etsi_doc_finder.search_document(document) if valid_etsi_doc_format.match(document) else \
-        etsi_spec_finder.search_document(document) if valid_etsi_spec_format.match(document) else "Document ID not supported"
     if "Specification" in url or "Document" in url:
         raise HTTPException(status_code=404, detail=url)
 
@@ -203,11 +284,20 @@ def find_document_batch(request: BatchDocRequest):
     missing = []
 
     for document in documents:
-        url = get_tdoc_url(document) if valid_3gpp_doc_format.match(document) else \
-            get_spec_url(document) if valid_3gpp_spec_format.match(document) else \
-            etsi_doc_finder.search_document(document) if valid_etsi_doc_format.match(document) else \
-            etsi_spec_finder.search_document(document) if valid_etsi_spec_format.match(document) else "Document ID not supported"
-    
+        if valid_3gpp_doc_format.match(document):
+            url = get_tdoc_url(document)
+        elif valid_3gpp_spec_format.match(document):
+            url = get_spec_url(document)
+        elif valid_etsi_doc_format.match(document):
+            etsi_doc_finder.search_document(document)
+        elif valid_etsi_spec_format.match(document):
+            etsi_spec_finder.search_document(document)
+        elif document.startswith("GP"):
+            for sp in gp_spec_locations:
+                if document.lower() in sp.lower():
+                    url = get_gp_spec_url(gp_spec_locations[sp])
+        else:
+            url = "Document ID not supported"
         if "Specification" in url or "Document" in url:
             missing.append(document)
         else: