added tools for github file listing and retrieval

app.py

@@ -1,6 +1,7 @@
 import asyncio
 import os
 import time
+import traceback
 from typing import Dict, List
 
 import gradio as gr
@@ -8,10 +9,14 @@ from dotenv import load_dotenv
 from llama_index.core import Settings
 from llama_index.core.text_splitter import SentenceSplitter
 
-from rag.config import embed_model, get_available_repos, llm
+from rag.config import (delete_repository_data, embed_model,
+                        get_available_repos, get_repo_details,
+                        get_repository_stats, llm)
 from rag.github_file_loader import \
-    fetch_markdown_files as fetch_files_with_loader, load_github_files
+    fetch_markdown_files as fetch_files_with_loader
+from rag.github_file_loader import fetch_repository_files, load_github_files
 from rag.ingest import ingest_documents_async
+from rag.query import QueryRetriever
 
 load_dotenv()
 
@@ -922,8 +927,6 @@ with gr.Blocks(title="Doc-MCP") as demo:
                     return {"error": "Please select a valid repository."}
 
                 try:
-                    # Import QueryRetriever here to avoid circular imports
-                    from rag.query import QueryRetriever
 
                     # Create query retriever for the selected repo
                     retriever = QueryRetriever(repo)
@@ -934,8 +937,6 @@ with gr.Blocks(title="Doc-MCP") as demo:
 
                 except Exception as e:
                     print(f"Query error: {e}")
-                    import traceback
-
                     traceback.print_exc()
                     return {"error": f"Query failed: {str(e)}"}
 
@@ -1069,7 +1070,7 @@ with gr.Blocks(title="Doc-MCP") as demo:
             def load_repository_stats():
                 """Load overall repository statistics"""
                 try:
-                    from rag.config import get_repository_stats
+
                     stats = get_repository_stats()
                     return stats
                 except Exception as e:
@@ -1078,7 +1079,7 @@ with gr.Blocks(title="Doc-MCP") as demo:
             def load_repository_details():
                 """Load detailed repository information as a table"""
                 try:
-                    from rag.config import get_repo_details
+                    
                     details = get_repo_details()
                     
                     if not details:
@@ -1129,7 +1130,7 @@ with gr.Blocks(title="Doc-MCP") as demo:
                     return "❌ Please confirm deletion by checking the checkbox.", gr.Dropdown(choices=[]), gr.Checkbox(value=False)
                 
                 try:
-                    from rag.config import delete_repository_data
+
 
                     # Perform deletion
                     result = delete_repository_data(repo_name)
@@ -1216,6 +1217,341 @@ with gr.Blocks(title="Doc-MCP") as demo:
                 show_api=False
             )
 
+        # ================================
+        # Tab 4: GitHub File Search (Hidden API)
+        # ================================
+        with gr.TabItem("🔍 GitHub File Search", visible=False):
+            gr.Markdown("### 🔧 GitHub Repository File Search API")
+            gr.Markdown("Pure API endpoints for GitHub file operations - all responses in JSON format")
+            
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("#### 📋 List Repository Files")
+                    
+                    # Repository input for file operations
+                    api_repo_input = gr.Textbox(
+                        label="Repository URL",
+                        placeholder="owner/repo or https://github.com/owner/repo",
+                        value="",
+                        info="GitHub repository to scan"
+                    )
+                    
+                    # Branch selection
+                    api_branch_input = gr.Textbox(
+                        label="Branch",
+                        value="main",
+                        placeholder="main",
+                        info="Branch to search (default: main)"
+                    )
+                    
+                    # File extensions
+                    api_extensions_input = gr.Textbox(
+                        label="File Extensions (comma-separated)",
+                        value=".md,.mdx",
+                        placeholder=".md,.mdx,.txt",
+                        info="File extensions to include"
+                    )
+                    
+                    # List files button
+                    list_files_btn = gr.Button("📋 List Files", variant="primary")
+                    
+                with gr.Column():
+                    gr.Markdown("#### 📄 Get Single File")
+                    
+                    # Single file inputs
+                    single_repo_input = gr.Textbox(
+                        label="Repository URL",
+                        placeholder="owner/repo or https://github.com/owner/repo",
+                        value="",
+                        info="GitHub repository"
+                    )
+                    
+                    single_file_input = gr.Textbox(
+                        label="File Path",
+                        placeholder="docs/README.md",
+                        value="",
+                        info="Path to specific file in repository"
+                    )
+                    
+                    single_branch_input = gr.Textbox(
+                        label="Branch",
+                        value="main",
+                        placeholder="main",
+                        info="Branch name (default: main)"
+                    )
+                    
+                    # Get single file button
+                    get_single_btn = gr.Button("📄 Get Single File", variant="secondary")
+
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("#### 📚 Get Multiple Files")
+                    
+                    # Multiple files inputs
+                    multiple_repo_input = gr.Textbox(
+                        label="Repository URL",
+                        placeholder="owner/repo or https://github.com/owner/repo",
+                        value="",
+                        info="GitHub repository"
+                    )
+                    
+                    multiple_files_input = gr.Textbox(
+                        label="File Paths (comma-separated)",
+                        placeholder="README.md,docs/guide.md,api/overview.md",
+                        value="",
+                        lines=3,
+                        info="Comma-separated list of file paths"
+                    )
+                    
+                    multiple_branch_input = gr.Textbox(
+                        label="Branch",
+                        value="main",
+                        placeholder="main",
+                        info="Branch name (default: main)"
+                    )
+                    
+                    # Get multiple files button
+                    get_multiple_btn = gr.Button("📚 Get Multiple Files", variant="secondary")
+
+            # Single JSON output for all operations
+            gr.Markdown("### 📊 API Response")
+            api_response_output = gr.JSON(
+                label="JSON Response",
+                value={
+                    "message": "API responses will appear here",
+                    "info": "Use the buttons above to interact with GitHub repositories"
+                }
+            )
 
+            # Pure API Functions (JSON only responses)
+            def list_repository_files(repo_url: str, branch: str = "main", extensions: str = ".md,.mdx"):
+                """
+                List all files in a GitHub repository with specified extensions
+                
+                Args:
+                    repo_url: GitHub repository URL or owner/repo format
+                    branch: Branch name to search (default: main)
+                    extensions: Comma-separated file extensions (default: .md,.mdx)
+                    
+                Returns:
+                    JSON response with file list and metadata
+                """
+                try:
+                    if not repo_url.strip():
+                        return {"success": False, "error": "Repository URL is required"}
+                    
+                    # Parse extensions list
+                    ext_list = [ext.strip() for ext in extensions.split(",") if ext.strip()]
+                    if not ext_list:
+                        ext_list = [".md", ".mdx"]
+                    
+
+                    # Get files list
+                    files, status_message = fetch_repository_files(
+                        repo_url=repo_url,
+                        file_extensions=ext_list,
+                        github_token=os.getenv("GITHUB_API_KEY"),
+                        branch=branch
+                    )
+                    
+                    if files:
+                        return {
+                            "success": True,
+                            "repository": repo_url,
+                            "branch": branch,
+                            "extensions": ext_list,
+                            "total_files": len(files),
+                            "files": files,
+                            "status": status_message
+                        }
+                    else:
+                        return {
+                            "success": False,
+                            "repository": repo_url,
+                            "branch": branch,
+                            "extensions": ext_list,
+                            "total_files": 0,
+                            "files": [],
+                            "error": status_message or "No files found"
+                        }
+                        
+                except Exception as e:
+                    return {
+                        "success": False,
+                        "error": f"Failed to list files: {str(e)}",
+                        "repository": repo_url,
+                        "branch": branch
+                    }
+
+            def get_single_file(repo_url: str, file_path: str, branch: str = "main"):
+                """
+                Retrieve a single file from GitHub repository
+                
+                Args:
+                    repo_url: GitHub repository URL or owner/repo format  
+                    file_path: Path to the file in the repository
+                    branch: Branch name (default: main)
+                    
+                Returns:
+                    JSON response with file content and metadata
+                """
+                try:
+                    if not repo_url.strip():
+                        return {"success": False, "error": "Repository URL is required"}
+                    
+                    if not file_path.strip():
+                        return {"success": False, "error": "File path is required"}
+                    
+                    # Parse repo name
+                    if "github.com" in repo_url:
+                        repo_name = (
+                            repo_url.replace("https://github.com/", "")
+                            .replace("http://github.com/", "")
+                            .strip("/")
+                        )
+                    else:
+                        repo_name = repo_url.strip()
+                    
+                    # Load single file
+                    documents, failed = load_github_files(
+                        repo_name=repo_name,
+                        file_paths=[file_path.strip()],
+                        branch=branch,
+                        github_token=os.getenv("GITHUB_API_KEY")
+                    )
+                    
+                    if documents and len(documents) > 0:
+                        doc = documents[0]
+                        return {
+                            "success": True,
+                            "repository": repo_name,
+                            "branch": branch,
+                            "file_path": file_path,
+                            "file_name": doc.metadata.get("file_name", ""),
+                            "file_size": len(doc.text),
+                            "content": doc.text,
+                            "metadata": doc.metadata,
+                            "url": doc.metadata.get("url", ""),
+                            "raw_url": doc.metadata.get("raw_url", "")
+                        }
+                    else:
+                        error_msg = f"Failed to retrieve file: {failed[0] if failed else 'File not found or access denied'}"
+                        return {
+                            "success": False,
+                            "repository": repo_name,
+                            "branch": branch,
+                            "file_path": file_path,
+                            "error": error_msg
+                        }
+                        
+                except Exception as e:
+                    return {
+                        "success": False,
+                        "error": f"Failed to get single file: {str(e)}",
+                        "repository": repo_url,
+                        "file_path": file_path,
+                        "branch": branch
+                    }
+
+            def get_multiple_files(repo_url: str, file_paths_str: str, branch: str = "main"):
+                """
+                Retrieve multiple files from GitHub repository
+                
+                Args:
+                    repo_url: GitHub repository URL or owner/repo format
+                    file_paths_str: Comma-separated string of file paths
+                    branch: Branch name (default: main)
+                    
+                Returns:
+                    JSON response with multiple file contents and metadata
+                """
+                try:
+                    if not repo_url.strip():
+                        return {"success": False, "error": "Repository URL is required"}
+                    
+                    if not file_paths_str.strip():
+                        return {"success": False, "error": "File paths are required"}
+                    
+                    # Parse file paths from comma-separated string
+                    file_paths = [path.strip() for path in file_paths_str.split(",") if path.strip()]
+                    
+                    if not file_paths:
+                        return {"success": False, "error": "No valid file paths provided"}
+                    
+                    # Parse repo name
+                    if "github.com" in repo_url:
+                        repo_name = (
+                            repo_url.replace("https://github.com/", "")
+                            .replace("http://github.com/", "")
+                            .strip("/")
+                        )
+                    else:
+                        repo_name = repo_url.strip()
+                    
+                    # Load multiple files
+                    documents, failed = load_github_files(
+                        repo_name=repo_name,
+                        file_paths=file_paths,
+                        branch=branch,
+                        github_token=os.getenv("GITHUB_API_KEY")
+                    )
+                    
+                    # Process successful documents
+                    successful_files = []
+                    for doc in documents:
+                        file_data = {
+                            "file_path": doc.metadata.get("file_path", ""),
+                            "file_name": doc.metadata.get("file_name", ""),
+                            "file_size": len(doc.text),
+                            "content": doc.text,
+                            "metadata": doc.metadata,
+                            "url": doc.metadata.get("url", ""),
+                            "raw_url": doc.metadata.get("raw_url", "")
+                        }
+                        successful_files.append(file_data)
+                    
+                    return {
+                        "success": True,
+                        "repository": repo_name,
+                        "branch": branch,
+                        "requested_files": len(file_paths),
+                        "successful_files": len(successful_files),
+                        "failed_files": len(failed),
+                        "files": successful_files,
+                        "failed_file_paths": failed,
+                        "total_content_size": sum(len(doc.text) for doc in documents),
+                        "requested_file_paths": file_paths
+                    }
+                        
+                except Exception as e:
+                    return {
+                        "success": False,
+                        "error": f"Failed to get multiple files: {str(e)}",
+                        "repository": repo_url,
+                        "file_paths": file_paths_str,
+                        "branch": branch
+                    }
+
+            # Wire up the GitHub file search events - all output to single JSON component
+            list_files_btn.click(
+                fn=list_repository_files,
+                inputs=[api_repo_input, api_branch_input, api_extensions_input],
+                outputs=[api_response_output],
+                api_name="list_repository_files"
+            )
+            
+            get_single_btn.click(
+                fn=get_single_file,
+                inputs=[single_repo_input, single_file_input, single_branch_input],
+                outputs=[api_response_output],
+                api_name="get_single_file"
+            )
+            
+            get_multiple_btn.click(
+                fn=get_multiple_files,
+                inputs=[multiple_repo_input, multiple_files_input, multiple_branch_input],
+                outputs=[api_response_output],
+                api_name="get_multiple_files"
+            )
 if __name__ == "__main__":
     demo.launch(mcp_server=True)