Final version

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,11 +1,14 @@
 ---
-title: 3GPPindexers
-emoji: 💻
-colorFrom: purple
-colorTo: green
-sdk: docker
-pinned: false
+title: 3GPPIndexers
+emoji: ⚙️
+colorFrom: blue
+colorTo: red
+sdk: gradio
+sdk_version: 5.29.0
+app_file: app.py
+pinned: true
 license: gpl-3.0
+short_description: A Gradio app for indexing documents and specifications
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,264 @@
+from datetime import datetime
+import os
+import warnings
+import traceback
+import gradio as gr
+import subprocess
+from huggingface_hub import Repository
+from git import Repo
+import requests
+
+warnings.filterwarnings('ignore')
+
+DOC_INDEXER = "indexer_multi.py"
+SPEC_INDEXER = "spec_indexer_multi.py"
+SPEC_DOC_INDEXER = "spec_doc_indexer_multi.py"
+BM25_INDEXER = "bm25_maker.py"
+
+DOC_INDEX_FILE = "indexed_docs.json"
+SPEC_INDEX_FILE = "indexed_specifications.json"
+SPEC_DOC_INDEX_FILE = "indexed_docs_content.zip"
+BM25_INDEX_FILE = "bm25s.zip"
+
+HF_SEARCH_REPO = "OrganizedProgrammers/3GPPDocFinder"
+REPO_DIR = os.path.dirname(os.path.abspath(__file__))
+
+def get_docs_stats():
+    if os.path.exists(DOC_INDEX_FILE):
+        import json
+        with open(DOC_INDEX_FILE, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+            return len(data["docs"])
+    return 0
+
+def get_specs_stats():
+    if os.path.exists(SPEC_INDEX_FILE):
+        import json
+        with open(SPEC_INDEX_FILE, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+            return len(data["specs"])
+    return 0
+
+def get_scopes_stats():
+    if os.path.exists(SPEC_INDEX_FILE):
+        import json
+        with open(SPEC_INDEX_FILE, 'r', encoding="utf-8") as f:
+            data = json.load(f)
+            return len(data['scopes'])
+    return 0
+
+def check_permissions(user: str, token: str):
+    try:
+        req = requests.get("https://huggingface.co/api/whoami-v2", verify=False, headers={"Accept": "application/json", "Authorization": f"Bearer {token}"})
+        if req.status_code != 200:
+            return False
+        reqJson: dict = req.json()
+        if not reqJson.get("name") or reqJson['name'] != user:
+            return False
+        if not reqJson.get("orgs") or len(reqJson['orgs']) == 0:
+            return False
+        for org in reqJson['orgs']:
+            if "645cfa1b5ebf379fd6d8a339" == org['id']:
+                return True
+        if not reqJson.get('auth') or reqJson['auth'] == {}:
+            return False
+        if reqJson['auth']['accessToken']['role'] != "fineGrained":
+            return False
+        for scope in reqJson['auth']['accessToken']['fineGrained']['scoped']:
+            if scope['entity']['type'] == "org" and scope['entity']['_id'] == "645cfa1b5ebf379fd6d8a339" and all(perm in scope['permissions'] for perm in ['repo.write', 'repo.content.read']):
+                return True
+        return False
+    except Exception as e:
+        traceback.print_exception(e)
+        return False
+
+def update_logged(user: str, token: str):
+    if check_permissions(user, token):
+        return gr.update(visible=False), gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)
+    else:
+        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)
+
+def commit_and_push_3gppindexers(user, token, files, message, current_log=""):
+    log = current_log + "\n"
+    repo = Repo(REPO_DIR)
+    origin = repo.remotes.origin
+    repo.config_writer().set_value("user", "name", "3GPP Indexer Automatic Git Tool").release()
+    repo.config_writer().set_value("user", "email", "example@mail.org").release()
+    origin.pull()
+    log += "Git pull succeed !\n"
+    yield log
+    
+    repo.git.add(files)
+    repo.index.commit(message)
+    
+    try:
+        repo.git.push(f"https://{user}:{token}@huggingface.co/spaces/OrganizedProgrammers/3GPPIndexers")
+        log += "Git push succeed !\n"
+        yield log
+        log += "Wait for Huggingface to restart the Space\n"
+        yield log
+    except Exception as e:
+        log += f"Git push failed: {e}\n"
+        yield log
+
+def commit_and_push_3gppdocfinder(token, files, message, current_log=""):
+    log = current_log + "\n"
+    if not token:
+        log += "No token provided. Skipping HuggingFace push.\n"
+        yield log
+        return
+        
+    hf_repo_dir = os.path.join(REPO_DIR, "hf_spaces")
+    repo = None
+    
+    if not os.path.exists(hf_repo_dir):
+        repo = Repository(
+            local_dir=hf_repo_dir,
+            repo_type="space",
+            clone_from=HF_SEARCH_REPO,
+            git_user="3GPP Indexer Automatic Git Tool",
+            git_email="example@mail.org",
+            token=token,
+            skip_lfs_files=True
+        )
+    else:
+        repo = Repository(
+            local_dir=hf_repo_dir,
+            repo_type="space",
+            git_user="3GPP Indexer Automatic Git Tool",
+            git_email="example@mail.org",
+            token=token,
+            skip_lfs_files=True
+        )
+    
+    repo.git_pull()
+    
+    # Copy artifact files to huggingface space
+    for f in files:
+        import shutil
+        shutil.copy2(f, os.path.join(hf_repo_dir, f))
+    
+    repo.git_add(auto_lfs_track=True)
+    repo.git_commit(message)
+    repo.git_push()
+    
+    log += "Pushed to HuggingFace.\n"
+    yield log
+
+def refresh_stats():
+    return str(get_docs_stats()), str(get_specs_stats()), str(get_scopes_stats())
+
+def stream_script_output(script_path, current_log=""):
+    accumulated_output = current_log
+    
+    process = subprocess.Popen(
+        ["python", script_path],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        bufsize=1,
+        universal_newlines=True,
+    )
+    
+    for line in process.stdout:
+        accumulated_output += line
+        yield accumulated_output
+    
+    process.stdout.close()
+    process.wait()
+    
+    yield accumulated_output
+
+def index_documents(user, token):
+    log_output = "⏳ Indexation en cours...\n"
+    # Désactiver tous les boutons
+    yield gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), log_output
+    
+    # Lancer l'indexation
+    if not check_permissions(user, token):
+        log_output += "❌ Identifiants invalides\n"
+        yield gr.update(interactive=True), gr.update(interactive=True), gr.update(interactive=True), log_output
+        return
+    
+    for log in stream_script_output(DOC_INDEXER, log_output):
+        yield gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), log
+        log_output = log
+    
+    d = datetime.today().strftime("%d/%m/%Y-%H:%M:%S")
+    
+    for log in commit_and_push_3gppdocfinder(token, [DOC_INDEX_FILE], f"Update documents indexer via Indexer: {d}", log_output):
+        yield gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), log
+        log_output = log
+    
+    for log in commit_and_push_3gppindexers(user, token, [DOC_INDEX_FILE], f"Update documents indexer via Indexer: {d}", log_output):
+        yield gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), log
+        log_output = log
+    
+    # Réactiver les boutons à la fin
+    log_output += "✅ Terminé.\n"
+    yield gr.update(interactive=True), gr.update(interactive=True), gr.update(interactive=True), log_output
+
+def index_specifications(user, token):
+    log_output = "⏳ Indexation en cours...\n"
+    # Désactiver tous les boutons
+    yield gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), log_output
+    
+    # Lancer l'indexation
+    if not check_permissions(user, token):
+        log_output += "❌ Identifiants invalides\n"
+        yield gr.update(interactive=True), gr.update(interactive=True), gr.update(interactive=True), log_output
+        return
+    
+    for log in stream_script_output(SPEC_INDEXER, log_output):
+        yield gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), log
+        log_output = log
+
+    for log in stream_script_output(SPEC_DOC_INDEXER, log_output):
+        yield gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), log
+        log_output = log
+
+    for log in stream_script_output(BM25_INDEXER, log_output):
+        yield gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), log
+        log_output = log
+    
+    d = datetime.today().strftime("%d/%m/%Y-%H:%M:%S")
+    
+    for log in commit_and_push_3gppdocfinder(token, [SPEC_DOC_INDEX_FILE, BM25_INDEX_FILE, SPEC_INDEX_FILE], f"Update specifications indexer via Indexer: {d}", log_output):
+        yield gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), log
+        log_output = log
+    
+    for log in commit_and_push_3gppindexers(user, token, [SPEC_DOC_INDEX_FILE, BM25_INDEX_FILE, SPEC_INDEX_FILE], f"Update specifications indexer via Indexer: {d}", log_output):
+        yield gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), log
+        log_output = log
+    
+    # Réactiver les boutons à la fin
+    log_output += "✅ Terminé.\n"
+    yield gr.update(interactive=True), gr.update(interactive=True), gr.update(interactive=True), log_output
+
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    gr.Markdown("## 📄 3GPP Indexers")
+    
+    with gr.Row() as r1:
+        with gr.Column():
+            git_user = gr.Textbox(label="Git user (for push/pull indexes)")
+            git_pass = gr.Textbox(label="Git Token", type="password")
+            btn_login = gr.Button("Login", variant="primary")
+    
+    with gr.Row(visible=False) as r2:
+        with gr.Column():
+            doc_count = gr.Textbox(label="Docs Indexed", value=str(get_docs_stats()), interactive=False)
+            btn_docs = gr.Button("Re-index Documents", variant="primary")
+        with gr.Column():
+            spec_count = gr.Textbox(label="Specs Indexed", value=str(get_specs_stats()), interactive=False)
+            btn_specs = gr.Button("Re-index Specifications", variant="primary")
+        with gr.Column():
+            scope_count = gr.Textbox(label="Scopes Indexed", value=str(get_scopes_stats()), interactive=False)
+    
+    out = gr.Textbox(label="Output/Log", lines=13, autoscroll=True, visible=False)
+    refresh = gr.Button(value="🔄 Refresh Stats", visible=False)
+    
+    btn_login.click(update_logged, inputs=[git_user, git_pass], outputs=[r1, r2, out, refresh])
+    btn_docs.click(index_documents, inputs=[git_user, git_pass], outputs=[btn_docs, btn_specs, refresh, out])
+    btn_specs.click(index_specifications, inputs=[git_user, git_pass], outputs=[btn_docs, btn_specs, refresh, out])
+    refresh.click(refresh_stats, outputs=[doc_count, spec_count, scope_count])
+
+demo.launch()

bm25_maker.py

@@ -0,0 +1,59 @@
+import shutil
+import zipfile
+import json
+import bm25s
+
+import nltk
+from nltk.stem import WordNetLemmatizer
+
+nltk.download("wordnet")
+lemmatizer = WordNetLemmatizer()
+indexer_id = "3gpp_bm25_docs"
+unique_specs = set()
+
+with open("indexed_specifications.json", "r") as f:
+    spec_data = json.load(f)["specs"]
+with zipfile.ZipFile(open("indexed_docs_content.zip", "rb")) as zf:
+    for file_name in zf.namelist():
+        if file_name.endswith(".json"):
+            doc_bytes = zf.read(file_name)
+            try:
+                doc_data = json.loads(doc_bytes.decode("utf-8"))
+                print("Documents loaded successfully !")
+            except json.JSONDecodeError as e:
+                print(f"Error while decoding the JSON file {file_name}: {e}")
+
+corpus_json = []
+
+for _, specification in spec_data.items():
+    full_text = f"{specification['id']} - {specification['title']}\n\n\n"
+    if specification['id'] in unique_specs:
+        continue
+    document = doc_data.get(specification['id'], None)
+    if document is None: continue
+    if not isinstance(document, str):
+        full_text += "\n".join([f"{title}\n\n{document[title]}" for title in document.keys()])
+        corpus_json.append({"text": lemmatizer.lemmatize(full_text), "metadata": {
+            "id": specification['id'],
+            "title": specification['title'],
+            "version": specification['version'],
+            "release": specification['release'],
+            "type": specification['type'],
+            "working_group": specification['working_group'],
+            "url": specification['url'],
+            "scope": specification['scope']
+        }})
+        unique_specs.add(specification['id'])
+    else:
+        print(f"Skipping {specification['id']}")
+        unique_specs.add(specification['id'])
+        
+corpus_text = [doc["text"] for doc in corpus_json]
+corpus_tokens = bm25s.tokenize(corpus_text, stopwords="en")
+
+retriever = bm25s.BM25(corpus=corpus_json)
+retriever.index(corpus_tokens)
+
+retriever.save(indexer_id)
+
+shutil.make_archive("bm25s", 'zip', '.', indexer_id)

indexer_multi.py

@@ -0,0 +1,286 @@
+from datetime import datetime
+import requests
+from bs4 import BeautifulSoup
+import json
+import os
+import time
+import warnings
+import re
+import concurrent.futures
+import threading
+from typing import List, Dict, Any
+
+warnings.filterwarnings("ignore")
+
+class TsgDocIndexer:
+    def __init__(self, max_workers=10):
+        self.main_ftp_url = "https://www.3gpp.org/ftp"
+        self.indexer_file = "indexed_docs.json"
+        self.indexer, self.latest_date = self.load_indexer()
+        self.valid_doc_pattern = re.compile(r'^(S[1-6P]|C[1-6P])-\d+', flags=re.IGNORECASE)
+        self.max_workers = max_workers
+        
+        # Verrous pour les opérations thread-safe
+        self.print_lock = threading.Lock()
+        self.indexer_lock = threading.Lock()
+        
+        # Compteurs pour le suivi
+        self.total_indexed = 0
+        self.processed_count = 0
+        self.total_count = 0
+
+    def load_indexer(self):
+        """Load existing index if available"""
+        if os.path.exists(self.indexer_file):
+            with open(self.indexer_file, "r", encoding="utf-8") as f:
+                x = json.load(f)
+                return x["docs"], x["last_indexed_date"]
+        return {}, None
+    
+    def save_indexer(self):
+        """Save the updated index"""
+        with open(self.indexer_file, "w", encoding="utf-8") as f:
+            today = datetime.today()
+            self.latest_date = today.strftime("%d/%m/%Y-%H:%M:%S")
+            output = {"docs": self.indexer, "last_indexed_date": self.latest_date}
+            json.dump(output, f, indent=4, ensure_ascii=False)
+
+    def get_docs_from_url(self, url):
+        """Récupérer la liste des documents/répertoires depuis une URL"""
+        try:
+            response = requests.get(url, verify=False, timeout=10)
+            soup = BeautifulSoup(response.text, "html.parser")
+            return [item.get_text() for item in soup.select("tr td a")]
+        except Exception as e:
+            with self.print_lock:
+                print(f"Erreur lors de l'accès à {url}: {e}")
+            return []
+
+    def is_valid_document_pattern(self, filename):
+        """Vérifier si le nom de fichier correspond au motif requis"""
+        return bool(self.valid_doc_pattern.match(filename))
+
+    def is_zip_file(self, filename):
+        """Vérifier si le fichier est un ZIP"""
+        return filename.lower().endswith('.zip')
+
+    def extract_doc_id(self, filename):
+        """Extraire l'identifiant du document à partir du nom de fichier s'il correspond au motif"""
+        if self.is_valid_document_pattern(filename):
+            match = self.valid_doc_pattern.match(filename)
+            if match:
+                # Retourner le motif complet (comme S1-12345)
+                full_id = filename.split('.')[0]  # Enlever l'extension si présente
+                return full_id.split('_')[0]  # Enlever les suffixes après underscore si présents
+        return None
+
+    def process_zip_files(self, files_list, base_url, workshop=False):
+        """Traiter une liste de fichiers pour trouver et indexer les ZIP valides"""
+        indexed_count = 0
+        
+        for file in files_list:
+            if file in ['./', '../', 'ZIP/', 'zip/']:
+                continue
+                
+            # Vérifier si c'est un fichier ZIP et s'il correspond au motif
+            if self.is_zip_file(file) and (self.is_valid_document_pattern(file) or workshop):
+                file_url = f"{base_url}/{file}"
+                
+                # Extraire l'ID du document
+                doc_id = self.extract_doc_id(file)
+                if doc_id is None:
+                    doc_id = file.split('.')[0]
+                if doc_id:
+                    # Vérifier si ce fichier est déjà indexé
+                    with self.indexer_lock:
+                        if doc_id in self.indexer and self.indexer[doc_id] == file_url:
+                            continue
+                            
+                        # Ajouter ou mettre à jour l'index
+                        self.indexer[doc_id] = file_url
+                        indexed_count += 1
+                        self.total_indexed += 1
+                        
+        return indexed_count
+
+    def process_meeting(self, meeting, wg_url, workshop=False):
+        """Traiter une réunion individuelle avec multithreading"""
+        try:
+            if meeting in ['./', '../']:
+                return 0
+                
+            meeting_url = f"{wg_url}/{meeting}"
+            
+            with self.print_lock:
+                print(f" Vérification de la réunion: {meeting}")
+                
+            # Vérifier le contenu de la réunion
+            meeting_contents = self.get_docs_from_url(meeting_url)
+            
+            key = None
+            if "docs" in [x.lower() for x in meeting_contents]:
+                key = "docs"
+            elif "tdocs" in [x.lower() for x in meeting_contents]:
+                key = "tdocs"
+                
+            if key is not None:
+                docs_url = f"{meeting_url}/{key}"
+                
+                with self.print_lock:
+                    print(f" Vérification des documents dans {docs_url}")
+                    
+                # Récupérer la liste des fichiers dans le dossier Docs
+                docs_files = self.get_docs_from_url(docs_url)
+                
+                # 1. Indexer les fichiers ZIP directement dans le dossier Docs
+                docs_indexed_count = self.process_zip_files(docs_files, docs_url, workshop)
+                
+                if docs_indexed_count > 0:
+                    with self.print_lock:
+                        print(f" {docs_indexed_count} nouveaux fichiers ZIP indexés dans Docs")
+                
+                # 2. Vérifier le sous-dossier ZIP s'il existe
+                if "zip" in [x.lower() for x in docs_files]:
+                    zip_url = f"{docs_url}/zip"
+                    
+                    with self.print_lock:
+                        print(f" Vérification du dossier ZIP: {zip_url}")
+                        
+                    # Récupérer les fichiers dans le sous-dossier ZIP
+                    zip_files = self.get_docs_from_url(zip_url)
+                    
+                    # Indexer les fichiers ZIP dans le sous-dossier ZIP
+                    zip_indexed_count = self.process_zip_files(zip_files, zip_url, workshop)
+                    
+                    if zip_indexed_count > 0:
+                        with self.print_lock:
+                            print(f" {zip_indexed_count} nouveaux fichiers ZIP indexés dans le dossier ZIP")
+            
+            # Mise à jour du compteur de progression
+            with self.indexer_lock:
+                self.processed_count += 1
+                
+            # Affichage de la progression
+            with self.print_lock:
+                progress = (self.processed_count / self.total_count) * 100 if self.total_count > 0 else 0
+                print(f"\rProgression: {self.processed_count}/{self.total_count} réunions traitées ({progress:.1f}%)", end="")
+                
+            return 1  # Réunion traitée avec succès
+            
+        except Exception as e:
+            with self.print_lock:
+                print(f"\nErreur lors du traitement de la réunion {meeting}: {str(e)}")
+            return 0
+
+    def process_workgroup(self, wg, main_url):
+        """Traiter un groupe de travail avec multithreading pour ses réunions"""
+        if wg in ['./', '../']:
+            return
+            
+        wg_url = f"{main_url}/{wg}"
+        
+        with self.print_lock:
+            print(f" Vérification du groupe de travail: {wg}")
+            
+        # Récupérer les dossiers de réunion
+        meeting_folders = self.get_docs_from_url(wg_url)
+        
+        # Ajouter au compteur total
+        self.total_count += len([m for m in meeting_folders if m not in ['./', '../']])
+        
+        # Utiliser ThreadPoolExecutor pour traiter les réunions en parallèle
+        with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            futures = [executor.submit(self.process_meeting, meeting, wg_url) 
+                      for meeting in meeting_folders if meeting not in ['./', '../']]
+            
+            # Attendre que toutes les tâches soient terminées
+            concurrent.futures.wait(futures)
+        
+        # Sauvegarder après chaque groupe de travail
+        with self.indexer_lock:
+            self.save_indexer()
+
+    def index_all_tdocs(self):
+        """Indexer tous les documents ZIP dans la structure FTP 3GPP avec multithreading"""
+        print("Démarrage de l'indexation des documents ZIP 3GPP (S1-S6, SP, C1-C6, CP)...")
+        
+        start_time = time.time()
+        docs_count_before = len(self.indexer)
+        
+        # Principaux groupes TSG
+        main_groups = ["tsg_sa", "tsg_ct"]  # Ajouter d'autres si nécessaire
+        
+        for main_tsg in main_groups:
+            print(f"\nIndexation de {main_tsg.upper()}...")
+            
+            main_url = f"{self.main_ftp_url}/{main_tsg}"
+            
+            # Récupérer les groupes de travail
+            workgroups = self.get_docs_from_url(main_url)
+            
+            # Traiter chaque groupe de travail séquentiellement
+            # (mais les réunions à l'intérieur seront traitées en parallèle)
+            for wg in workgroups:
+                self.process_workgroup(wg, main_url)
+        
+        docs_count_after = len(self.indexer)
+        new_docs_count = docs_count_after - docs_count_before
+        
+        print(f"\nIndexation terminée en {time.time() - start_time:.2f} secondes")
+        print(f"Nouveaux documents ZIP indexés: {new_docs_count}")
+        print(f"Total des documents dans l'index: {docs_count_after}")
+        
+        return self.indexer
+
+    def index_all_workshops(self):
+        print("Démarrage de l'indexation des workshops ZIP 3GPP...")
+        start_time = time.time()
+        docs_count_before = len(self.indexer)
+
+        print("\nIndexation du dossier 'workshop'")
+        main_url = f"{self.main_ftp_url}/workshop"
+            
+        # Récupérer les dossiers de réunion
+        meeting_folders = self.get_docs_from_url(main_url)
+        
+        # Ajouter au compteur total
+        self.total_count += len([m for m in meeting_folders if m not in ['./', '../']])
+        
+        # Utiliser ThreadPoolExecutor pour traiter les réunions en parallèle
+        with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            futures = [executor.submit(self.process_meeting, meeting, main_url, workshop=True) 
+                      for meeting in meeting_folders if meeting not in ['./', '../']]
+            concurrent.futures.wait(futures)
+        
+        # Sauvegarder après chaque groupe de travail
+        with self.indexer_lock:
+            self.save_indexer()
+        
+        docs_count_after = len(self.indexer)
+        new_docs_count = docs_count_after - docs_count_before
+        
+        print(f"\nIndexation terminée en {time.time() - start_time:.2f} secondes")
+        print(f"Nouveaux documents ZIP indexés: {new_docs_count}")
+        print(f"Total des documents dans l'index: {docs_count_after}")
+        
+        return self.indexer
+            
+
+
+def main():
+    # Nombre de workers pour le multithreading (ajustable selon les ressources disponibles)
+    max_workers = 64
+    
+    indexer = TsgDocIndexer(max_workers=max_workers)
+    
+    try:
+        indexer.index_all_tdocs()
+        indexer.index_all_workshops()
+    except Exception as e:
+        print(f"Erreur lors de l'indexation: {e}")
+    finally:
+        indexer.save_indexer()
+        print("Index sauvegardé.")
+
+if __name__ == "__main__":
+    main()

packages.txt

@@ -0,0 +1,2 @@
+libreoffice
+libreoffice-writer

requirements.txt

@@ -0,0 +1,9 @@
+gradio
+requests
+beautifulsoup4
+gitpython
+huggingface_hub
+lxml
+scikit-learn
+bm25s[full]
+nltk

spec_doc_indexer_multi.py

@@ -0,0 +1,291 @@
+import datetime
+import time
+import sys
+import json
+import traceback
+import requests
+import zipfile
+import uuid
+import os
+import re
+import subprocess
+import concurrent.futures
+import threading
+from io import StringIO, BytesIO
+from typing import List, Dict, Any
+
+import pandas as pd
+import numpy as np
+import warnings
+
+warnings.filterwarnings("ignore")
+
+# Caractères pour le formatage des versions
+chars = "0123456789abcdefghijklmnopqrstuvwxyz"
+
+# Verrous pour les opérations thread-safe
+print_lock = threading.Lock()
+dict_lock = threading.Lock()
+scope_lock = threading.Lock()
+
+# Dictionnaires globaux
+indexed_specifications = {}
+documents_by_spec_num = {}
+processed_count = 0
+total_count = 0
+
+def get_text(specification: str, version: str):
+    """Récupère les bytes du PDF à partir d'une spécification et d'une version."""
+    doc_id = specification
+    series = doc_id.split(".")[0]
+    
+    response = requests.get(
+        f"https://www.3gpp.org/ftp/Specs/archive/{series}_series/{doc_id}/{doc_id.replace('.', '')}-{version}.zip", 
+        verify=False,
+        headers={"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
+    )
+    
+    if response.status_code != 200:
+        raise Exception(f"Téléchargement du ZIP échoué pour {specification}-{version}")
+    
+    zip_bytes = BytesIO(response.content)
+    
+    with zipfile.ZipFile(zip_bytes) as zf:
+        for file_name in zf.namelist():
+            if file_name.endswith("zip"):
+                print("Another ZIP !")
+                zip_bytes = BytesIO(zf.read(file_name))
+                zf = zipfile.ZipFile(zip_bytes)
+                for file_name2 in zf.namelist():
+                    if file_name2.endswith("doc") or file_name2.endswith("docx"):
+                        if "cover" in file_name2.lower():
+                            print("COVER !")
+                            continue
+                        ext = file_name2.split(".")[-1]
+                        doc_bytes = zf.read(file_name2)
+                        temp_id = str(uuid.uuid4())
+                        input_path = f"/tmp/{temp_id}.{ext}"
+                        output_path = f"/tmp/{temp_id}.txt"
+                        
+                        with open(input_path, "wb") as f:
+                            f.write(doc_bytes)
+                        
+                        subprocess.run([
+                            "libreoffice",
+                            "--headless",
+                            "--convert-to", "txt",
+                            "--outdir", "/tmp",
+                            input_path
+                        ], check=True)
+                        
+                        with open(output_path, "r") as f:
+                            txt_data = [line.strip() for line in f if line.strip()]
+                        
+                        os.remove(input_path)
+                        os.remove(output_path)
+                        return txt_data
+            elif file_name.endswith("doc") or file_name.endswith("docx"):
+                if "cover" in file_name.lower():
+                    print("COVER !")
+                    continue
+                ext = file_name.split(".")[-1]
+                doc_bytes = zf.read(file_name)
+                temp_id = str(uuid.uuid4())
+                input_path = f"/tmp/{temp_id}.{ext}"
+                output_path = f"/tmp/{temp_id}.txt"
+                
+                print("Ecriture")
+                with open(input_path, "wb") as f:
+                    f.write(doc_bytes)
+
+                print("Convertissement")                
+                subprocess.run([
+                    "libreoffice",
+                    "--headless",
+                    "--convert-to", "txt",
+                    "--outdir", "/tmp",
+                    input_path
+                ], check=True)
+                
+                print("Ecriture TXT")
+                with open(output_path, "r", encoding="utf-8") as f:
+                    txt_data = [line.strip() for line in f if line.strip()]
+                
+                os.remove(input_path)
+                os.remove(output_path)
+                return txt_data
+    
+    raise Exception(f"Aucun fichier .doc/.docx trouvé dans le ZIP pour {specification}-{version}")
+
+def get_spec_content(specification: str, version: str):
+    text = get_text(specification, version)
+    forewords = []
+    for x in range(len(text)):
+        line = text[x]
+        if "Foreword" in line:
+            forewords.append(x)
+        if len(forewords) >= 2:
+            break
+
+    toc_brut = text[forewords[0]:forewords[1]]
+    chapters = []
+    for line in toc_brut:
+        x = line.split("\t")
+        if re.search(r"^\d+\t[\ \S]+", line):
+            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))
+        if re.search(r"^\d+\.\d+\t[\ \S]+", line):
+            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))
+        if re.search(r"^\d+\.\d+\.\d+\t[\ \S]+", line):
+            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))
+        if re.search(r"^\d+\.\d+\.\d+.\d+\t[\ \S]+", line):
+            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))
+        if re.search(r"^\d+\.\d+\.\d+.\d+.\d+\t[\ \S]+", line):
+            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))
+
+    real_toc_indexes = {}
+
+    for chapter in chapters:
+        try:
+            x = text.index(chapter)
+            real_toc_indexes[chapter] = x
+        except ValueError as e:
+            try:
+                number = chapter.split("\t")[0] + "\t"
+                for line in text[forewords[1]:]:
+                    if number in line:
+                        x = text.index(line)
+                        real_toc_indexes[line] = x
+                        break
+            except:
+                real_toc_indexes[chapter] = -float("inf")
+
+    document = {}
+    toc = list(real_toc_indexes.keys())
+    index_toc = list(real_toc_indexes.values())
+    curr_index = 0
+    for x in range(1, len(toc)):
+        document[toc[curr_index].replace("\t", " ")] = re.sub(r"[\ \t]+", " ", "\n".join(text[index_toc[curr_index]+1:index_toc[x]]))
+        curr_index = x
+
+    document[toc[curr_index].replace("\t"," ")] = re.sub(r"\s+", " ", " ".join(text[index_toc[curr_index]+1:]))
+    return document
+
+def process_specification(spec: Dict[str, Any], columns: List[str]) -> None:
+    """Traite une spécification individuelle avec multithreading."""
+    global processed_count, indexed_specifications, documents_by_spec_num
+    
+    try:
+        if spec.get('vers', None) is None:
+            return
+            
+        doc_id = str(spec["spec_num"])
+        series = doc_id.split(".")[0]
+        
+        a, b, c = str(spec["vers"]).split(".")
+        
+        # Formatage de l'URL selon la version
+        if not (int(a) > 35 or int(b) > 35 or int(c) > 35):
+            version_code = f"{chars[int(a)]}{chars[int(b)]}{chars[int(c)]}"
+            spec_url = f"https://www.3gpp.org/ftp/Specs/archive/{series}_series/{doc_id}/{doc_id.replace('.', '')}-{version_code}.zip"
+        else:
+            x, y, z = str(a), str(b), str(c)
+            while len(x) < 2:
+                x = "0" + x
+            while len(y) < 2:
+                y = "0" + y
+            while len(z) < 2:
+                z = "0" + z
+            version_code = f"{x}{y}{z}"
+            spec_url = f"https://www.3gpp.org/ftp/Specs/archive/{series}_series/{doc_id}/{doc_id.replace('.', '')}-{version_code}.zip"
+        
+        string = f"{spec['spec_num']}+-+{spec['title']}+-+{spec['type']}+-+{spec['vers']}+-+{spec['WG']}+-+Rel-{spec['vers'].split('.')[0]}"
+        
+        metadata = {
+            "id": str(spec["spec_num"]),
+            "title": spec["title"],
+            "type": spec["type"],
+            "release": str(spec["vers"].split(".")[0]),
+            "version": str(spec["vers"]),
+            "working_group": spec["WG"],
+            "url": spec_url
+        }
+        
+        # Mise à jour du dictionnaire global avec verrou
+        with dict_lock:
+            indexed_specifications[string] = metadata
+            processed_count += 1
+        
+        # Affichage de la progression avec verrou
+        with print_lock:
+            sys.stdout.write(f"\rTraitement: {processed_count}/{total_count} spécifications")
+            sys.stdout.flush()
+            
+    except Exception as e:
+        with print_lock:
+            print(f"\nErreur lors du traitement de {spec.get('spec_num', 'inconnu')}: {str(e)}")
+
+def main():
+    global total_count
+    start_time = time.time()
+    
+    # Récupération des spécifications depuis le site 3GPP
+    print("Récupération des spécifications depuis 3GPP...")
+    response = requests.get(
+        f'https://www.3gpp.org/dynareport?code=status-report.htm', 
+        headers={"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}, 
+        verify=False
+    )
+    
+    # Analyse des tableaux HTML
+    dfs = pd.read_html(
+        StringIO(response.text), 
+        storage_options={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}, 
+        encoding="utf-8"
+    )
+    
+    for x in range(len(dfs)):
+        dfs[x] = dfs[x].replace({np.nan: None})
+    
+    # Extraction des colonnes nécessaires
+    columns_needed = [0, 1, 2, 3, 4]
+    extracted_dfs = [df.iloc[:, columns_needed] for df in dfs]
+    columns = [x.replace("\xa0", "_") for x in extracted_dfs[0].columns]
+    
+    # Préparation des spécifications
+    specifications = []
+    for df in extracted_dfs:
+        for index, row in df.iterrows():
+            doc = row.to_list()
+            doc_dict = dict(zip(columns, doc))
+            specifications.append(doc_dict)
+    
+    total_count = len(specifications)
+    print(f"Traitement de {total_count} spécifications avec multithreading...")
+    
+    try:
+        # Vérification si un fichier de documents existe déjà
+        if os.path.exists("indexed_docs_content.zip"):
+            with zipfile.ZipFile(open("indexed_docs_content.zip", "rb")) as zf:
+                for file_name in zf.namelist():
+                    if file_name.endswith(".json"):
+                        doc_bytes = zf.read(file_name)
+                        global documents_by_spec_num
+                        documents_by_spec_num = json.loads(doc_bytes.decode("utf-8"))
+                        print(f"Chargement de {len(documents_by_spec_num)} documents depuis le cache.")
+        
+        # Utilisation de ThreadPoolExecutor pour le multithreading
+        with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
+            futures = [executor.submit(process_specification, spec, columns) for spec in specifications]
+            concurrent.futures.wait(futures)
+    
+    finally:
+        json_str = json.dumps(documents_by_spec_num, indent=4, ensure_ascii=False)
+        json_bytes = json_str.encode("utf-8")
+        with zipfile.ZipFile("indexed_docs_content.zip", "w", compression=zipfile.ZIP_DEFLATED) as archive:
+            archive.writestr("indexed_documents.json", json_bytes)
+        elapsed_time = time.time() - start_time
+        print(f"\nTraitement terminé en {elapsed_time:.2f} secondes")
+        print(f"Résultats sauvegardés dans l'archive ZIP")
+
+if __name__ == "__main__":
+    main()

spec_indexer_multi.py

@@ -0,0 +1,294 @@
+import datetime
+import time
+import sys
+import json
+import traceback
+import requests
+import zipfile
+import uuid
+import os
+import re
+import subprocess
+import concurrent.futures
+import threading
+from io import StringIO, BytesIO
+from typing import List, Dict, Any
+
+import pandas as pd
+import numpy as np
+import warnings
+
+warnings.filterwarnings("ignore")
+
+# Caractères pour le formatage des versions
+chars = "0123456789abcdefghijklmnopqrstuvwxyz"
+
+# Verrous pour les opérations thread-safe
+print_lock = threading.Lock()
+dict_lock = threading.Lock()
+scope_lock = threading.Lock()
+
+# Dictionnaires globaux
+indexed_specifications = {}
+scopes_by_spec_num = {}
+processed_count = 0
+total_count = 0
+
+def get_text(specification: str, version: str):
+    """Récupère les bytes du PDF à partir d'une spécification et d'une version."""
+    doc_id = specification
+    series = doc_id.split(".")[0]
+    
+    response = requests.get(
+        f"https://www.3gpp.org/ftp/Specs/archive/{series}_series/{doc_id}/{doc_id.replace('.', '')}-{version}.zip", 
+        verify=False,
+        headers={"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
+    )
+    
+    if response.status_code != 200:
+        raise Exception(f"Téléchargement du ZIP échoué pour {specification}-{version}")
+    
+    zip_bytes = BytesIO(response.content)
+    
+    with zipfile.ZipFile(zip_bytes) as zf:
+        for file_name in zf.namelist():
+            if file_name.endswith("zip"):
+                print("Another ZIP !")
+                zip_bytes = BytesIO(zf.read(file_name))
+                zf = zipfile.ZipFile(zip_bytes)
+                for file_name2 in zf.namelist():
+                    if file_name2.endswith("doc") or file_name2.endswith("docx"):
+                        if "cover" in file_name2.lower():
+                            print("COVER !")
+                            continue
+                        ext = file_name2.split(".")[-1]
+                        doc_bytes = zf.read(file_name2)
+                        temp_id = str(uuid.uuid4())
+                        input_path = f"/tmp/{temp_id}.{ext}"
+                        output_path = f"/tmp/{temp_id}.txt"
+                        
+                        with open(input_path, "wb") as f:
+                            f.write(doc_bytes)
+                        
+                        subprocess.run([
+                            "libreoffice",
+                            "--headless",
+                            "--convert-to", "txt",
+                            "--outdir", "/tmp",
+                            input_path
+                        ], check=True)
+                        
+                        with open(output_path, "r") as f:
+                            txt_data = [line.strip() for line in f if line.strip()]
+                        
+                        os.remove(input_path)
+                        os.remove(output_path)
+                        return txt_data
+            elif file_name.endswith("doc") or file_name.endswith("docx"):
+                if "cover" in file_name.lower():
+                    print("COVER !")
+                    continue
+                ext = file_name.split(".")[-1]
+                doc_bytes = zf.read(file_name)
+                temp_id = str(uuid.uuid4())
+                input_path = f"/tmp/{temp_id}.{ext}"
+                output_path = f"/tmp/{temp_id}.txt"
+                
+                print("Ecriture")
+                with open(input_path, "wb") as f:
+                    f.write(doc_bytes)
+
+                print("Convertissement")                
+                subprocess.run([
+                    "libreoffice",
+                    "--headless",
+                    "--convert-to", "txt",
+                    "--outdir", "/tmp",
+                    input_path
+                ], check=True)
+                
+                print("Ecriture TXT")
+                with open(output_path, "r", encoding="utf-8") as f:
+                    txt_data = [line.strip() for line in f if line.strip()]
+                
+                os.remove(input_path)
+                os.remove(output_path)
+                return txt_data
+    
+    raise Exception(f"Aucun fichier .doc/.docx trouvé dans le ZIP pour {specification}-{version}")
+
+def get_scope(specification: str, version: str):
+    try:
+        spec_text = get_text(specification, version)
+        scp_i = 0
+        nxt_i = 0
+        for x in range(len(spec_text)):
+            text = spec_text[x]
+            if re.search(r"scope$", text, flags=re.IGNORECASE):
+                scp_i = x
+                nxt_i = scp_i + 10
+            if re.search(r"references$", text, flags=re.IGNORECASE):
+                nxt_i = x
+
+        return re.sub(r"\s+", " ", " ".join(spec_text[scp_i+1:nxt_i])) if len(spec_text[scp_i+1:nxt_i]) < 2 else "Not found"
+    except Exception as e:
+        traceback.print_exception(e)
+        return "Not found (error)"
+
+def process_specification(spec: Dict[str, Any], columns: List[str]) -> None:
+    """Traite une spécification individuelle avec multithreading."""
+    global processed_count, indexed_specifications, scopes_by_spec_num
+    
+    try:
+        if spec.get('vers', None) is None:
+            return
+            
+        doc_id = str(spec["spec_num"])
+        series = doc_id.split(".")[0]
+        
+        a, b, c = str(spec["vers"]).split(".")
+        
+        # Formatage de l'URL selon la version
+        if not (int(a) > 35 or int(b) > 35 or int(c) > 35):
+            version_code = f"{chars[int(a)]}{chars[int(b)]}{chars[int(c)]}"
+            spec_url = f"https://www.3gpp.org/ftp/Specs/archive/{series}_series/{doc_id}/{doc_id.replace('.', '')}-{version_code}.zip"
+        else:
+            x, y, z = str(a), str(b), str(c)
+            while len(x) < 2:
+                x = "0" + x
+            while len(y) < 2:
+                y = "0" + y
+            while len(z) < 2:
+                z = "0" + z
+            version_code = f"{x}{y}{z}"
+            spec_url = f"https://www.3gpp.org/ftp/Specs/archive/{series}_series/{doc_id}/{doc_id.replace('.', '')}-{version_code}.zip"
+        
+        string = f"{spec['spec_num']}+-+{spec['title']}+-+{spec['type']}+-+{spec['vers']}+-+{spec['WG']}+-+Rel-{spec['vers'].split('.')[0]}"
+        
+        metadata = {
+            "id": str(spec["spec_num"]),
+            "title": spec["title"],
+            "type": spec["type"],
+            "release": str(spec["vers"].split(".")[0]),
+            "version": str(spec["vers"]),
+            "working_group": spec["WG"],
+            "url": spec_url
+        }
+        
+        # Vérification si le scope existe déjà pour ce numéro de spécification
+        spec_num = str(spec["spec_num"])
+        
+        with scope_lock:
+            if spec_num in scopes_by_spec_num:
+                # Réutilisation du scope existant
+                metadata["scope"] = scopes_by_spec_num[spec_num]
+                with print_lock:
+                    print(f"\nRéutilisation du scope pour {spec_num}")
+            else:
+                # Extraction du scope seulement si nécessaire
+                if not (int(a) > 35 or int(b) > 35 or int(c) > 35):
+                    version_for_scope = f"{chars[int(a)]}{chars[int(b)]}{chars[int(c)]}"
+                else:
+                    version_for_scope = version_code
+                
+                with print_lock:
+                    print(f"\nExtraction du scope pour {spec_num} (version {version_for_scope})")
+                
+                try:
+                    scope = get_scope(metadata["id"], version_for_scope)
+                    # Stockage du scope pour une utilisation future
+                    scopes_by_spec_num[spec_num] = scope
+                    metadata["scope"] = scope
+                except Exception as e:
+                    error_msg = f"Erreur lors de l'extraction du scope: {str(e)}"
+                    metadata["scope"] = error_msg
+                    scopes_by_spec_num[spec_num] = error_msg
+        
+        # Mise à jour du dictionnaire global avec verrou
+        with dict_lock:
+            string += f"+-+{metadata['scope']}" if metadata['scope'] != " " or metadata['scope'] != "" or "not found" not in metadata['scope'].lower() else ""
+            indexed_specifications[string] = metadata
+            processed_count += 1
+        
+        # Affichage de la progression avec verrou
+        with print_lock:
+            sys.stdout.write(f"\rTraitement: {processed_count}/{total_count} spécifications")
+            sys.stdout.flush()
+            
+    except Exception as e:
+        with print_lock:
+            print(f"\nErreur lors du traitement de {spec.get('spec_num', 'inconnu')}: {str(e)}")
+
+def main():
+    global total_count
+    old_length = 0
+    
+    start_time = time.time()
+    
+    # Récupération des spécifications depuis le site 3GPP
+    print("Récupération des spécifications depuis 3GPP...")
+    response = requests.get(
+        f'https://www.3gpp.org/dynareport?code=status-report.htm', 
+        headers={"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}, 
+        verify=False
+    )
+    
+    # Analyse des tableaux HTML
+    dfs = pd.read_html(
+        StringIO(response.text), 
+        storage_options={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}, 
+        encoding="utf-8"
+    )
+    
+    for x in range(len(dfs)):
+        dfs[x] = dfs[x].replace({np.nan: None})
+    
+    # Extraction des colonnes nécessaires
+    columns_needed = [0, 1, 2, 3, 4]
+    extracted_dfs = [df.iloc[:, columns_needed] for df in dfs]
+    columns = [x.replace("\xa0", "_") for x in extracted_dfs[0].columns]
+    
+    # Préparation des spécifications
+    specifications = []
+    for df in extracted_dfs:
+        for index, row in df.iterrows():
+            doc = row.to_list()
+            doc_dict = dict(zip(columns, doc))
+            specifications.append(doc_dict)
+    
+    total_count = len(specifications)
+    print(f"Traitement de {total_count} spécifications avec multithreading...")
+    
+    try:
+        # Vérification si un fichier de scopes existe déjà
+        if os.path.exists("indexed_specifications.json"):
+            with open("indexed_specifications.json", "r", encoding="utf-8") as f:
+                global scopes_by_spec_num
+                f_up = json.load(f)
+                scopes_by_spec_num = f_up['scopes']
+                before = len(f_up['specs'])
+                print(f"Chargement de {len(scopes_by_spec_num)} scopes depuis le cache.")
+        
+        # Utilisation de ThreadPoolExecutor pour le multithreading
+        with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
+            futures = [executor.submit(process_specification, spec, columns) for spec in specifications]
+            concurrent.futures.wait(futures)
+    
+    finally:
+        # Sauvegarde des résultats
+        result = {
+            "specs": indexed_specifications,
+            "scopes": scopes_by_spec_num,
+            "last_indexed_date": datetime.datetime.today().strftime("%d-%m-%Y")
+        }
+        
+        with open("indexed_specifications.json", "w", encoding="utf-8") as f:
+            json.dump(result, f, indent=4, ensure_ascii=False)
+        
+        elapsed_time = time.time() - start_time
+        print(f"\nTraitement terminé en {elapsed_time:.2f} secondes")
+        print(f"Nouveaux specifications : {len(indexed_specifications) - before}")
+        print(f"Résultats sauvegardés dans indexed_specifications.json")
+
+if __name__ == "__main__":
+    main()