Update app.py

app.py

@@ -1,3 +1,65 @@
+from flask import Flask, render_template, request, jsonify, send_file
+from huggingface_hub import HfApi
+import requests
+import base64
+import markdown
+import json
+import mimetypes
+import os
+import io
+import uuid
+from pathlib import Path
+
+app = Flask(__name__)
+
+GITHUB_API = "https://api.github.com/repos/"
+#HF_API = "https://huggingface.co/api/spaces/"
+
+def generate_file_tree(paths):
+    """Generate a simple file tree from a list of paths."""
+    print("generating file tree") 
+    tree = ["📁 Root"]
+    sorted_paths = sorted(paths)
+    for path in sorted_paths:
+        parts = path.split('/')
+        indent = "  " * (len(parts) - 1)
+        tree.append(f"{indent}📄 {parts[-1]}")
+    print("generating file tree - Complete") 
+    
+    return "\n".join(tree) + "\n\n"
+
+def get_all_files(owner, repo, path="", is_hf=False):
+    """Recursively fetch all files from a repository."""
+    if is_hf:
+        # Attempt to fetch file list from Hugging Face Space (publicly accessible files)
+        api_url = f"https://huggingface.co/spaces/{owner}/{repo}/tree/main/{path}".rstrip('/')
+    else:
+        api_url = f"{GITHUB_API}{owner}/{repo}/contents/{path}".rstrip('/')
+    
+    try:
+        response = requests.get(api_url)
+        print(response.content)
+        response = requests.get(api_url, headers={"Accept": "application/json"})
+        print(str(response.json()))
+        response.raise_for_status()
+        items = response.json()
+        
+        # Hugging Face might not return JSON in the same format; adjust if HTML is returned
+        if isinstance(items, str):  # If response isn’t JSON, it’s likely HTML
+            return None  # Fallback to error handling
+        
+        files = []
+        for item in items:
+            if item['type'] == 'file':
+                files.append(item)
+            elif item['type'] == 'dir':
+                files.extend(get_all_files(owner, repo, item['path'], is_hf))
+        print(files)
+        return files
+        
+    except Exception as e:
+        return None
+
 def get_hf_files(repo, name, path=""):
     api = HfApi()
     file_list = api.list_repo_files(repo_id=f'{repo}/{name}', repo_type="space")
@@ -46,4 +108,226 @@ def get_hf_files(repo, name, path=""):
         processed_files.append({"path": file_path})
 
     print(f"Processed files: {processed_files}")
-    return processed_files
+    return processed_files
+
+def get_repo_contents(url):
+    """Parse URL and fetch repository contents."""
+    try:
+        if "huggingface.co" in url:
+            parts = url.rstrip('/').split('/')
+            owner, repo = parts[-2], parts[-1]
+            # Fallback approach: manually fetch known files or use a simpler file list
+            # For now, assume a flat structure and fetch known files directly
+            # This is a workaround until a proper API token or endpoint is confirmed
+            known_files = [
+                {'path': 'app.py', 'type': 'file'},
+                {'path': 'README.md', 'type': 'file'}
+                # Add more known paths or implement HTML scraping if needed
+            ]
+            #files = get_all_files(owner, repo, "", True) or known_files
+            files = get_hf_files(owner, repo)
+            return owner, repo, files, True
+        else:  # Assume GitHub URL
+            parts = url.rstrip('/').split('/')
+            owner, repo = parts[-2], parts[-1]
+            files = get_all_files(owner, repo, "", False)
+            if files is None:
+                raise Exception("Failed to fetch GitHub repository contents")
+            return owner, repo, files, False
+    except Exception as e:
+        return None, None, f"Error fetching repo contents: {str(e)}", False
+
+def process_file_content(file_info, owner, repo, is_hf=False):
+    """Process individual file content from a repository."""
+    content = ""
+    file_path = file_info['path']
+    
+    try:
+        if is_hf:
+            file_url = f"https://huggingface.co/spaces/{owner}/{repo}/raw/main/{file_path}"
+        else:
+            file_url = f"{GITHUB_API}{owner}/{repo}/contents/{file_path}"
+        
+        response = requests.get(file_url)
+        response.raise_for_status()
+        
+        if is_hf:
+            content_raw = response.content
+            size = len(content_raw)
+            file_extension = file_path.split('.')[-1] if '.' in file_path else ''
+            mime_type, _ = mimetypes.guess_type(file_path)
+            is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json']
+            
+            if is_text:
+                try:
+                    text_content = content_raw.decode('utf-8')
+                    if file_extension == 'json':
+                        try:
+                            json_data = json.loads(text_content)
+                            formatted_json = json.dumps(json_data, indent=2)
+                            content = f"### File: {file_path}\n```json\n{formatted_json}\n```\n\n"
+                        except json.JSONDecodeError:
+                            content = f"### File: {file_path}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n"
+                    else:
+                        content = f"### File: {file_path}\n```{file_extension or 'text'}\n{text_content}\n```\n\n"
+                except UnicodeDecodeError:
+                    content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
+            else:
+                content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
+        else:  # GitHub
+            data = response.json()
+            if 'content' in data:
+                content_raw = base64.b64decode(data['content'])
+                size = data['size']
+                file_extension = file_path.split('.')[-1] if '.' in file_path else ''
+                mime_type, _ = mimetypes.guess_type(file_path)
+                is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json']
+                
+                if is_text:
+                    try:
+                        text_content = content_raw.decode('utf-8')
+                        if file_extension == 'json':
+                            try:
+                                json_data = json.loads(text_content)
+                                formatted_json = json.dumps(json_data, indent=2)
+                                content = f"### File: {file_path}\n```json\n{formatted_json}\n```\n\n"
+                            except json.JSONDecodeError:
+                                content = f"### File: {file_path}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n"
+                        else:
+                            content = f"### File: {file_path}\n```{file_extension or 'text'}\n{text_content}\n```\n\n"
+                    except UnicodeDecodeError:
+                        content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
+                else:
+                    content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
+            else:
+                content = f"### File: {file_path}\n[No content available]\n\n"
+    except Exception as e:
+        content = f"### File: {file_path}\n[Error fetching file content: {str(e)}]\n\n"
+    
+    return content
+
+def process_uploaded_file(file):
+    """Process uploaded file content."""
+    content = ""
+    filename = file.filename
+    file_extension = filename.split('.')[-1] if '.' in filename else ''
+    
+    try:
+        content_raw = file.read()  # Read file content into memory
+        size = len(content_raw)    # Compute size in bytes
+        
+        mime_type, _ = mimetypes.guess_type(filename)
+        is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json']
+        
+        if is_text:
+            try:
+                text_content = content_raw.decode('utf-8')
+                if file_extension == 'json':
+                    try:
+                        json_data = json.loads(text_content)
+                        formatted_json = json.dumps(json_data, indent=2)
+                        content = f"### File: {filename}\n```json\n{formatted_json}\n```\n\n"
+                    except json.JSONDecodeError:
+                        content = f"### File: {filename}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n"
+                else:
+                    content = f"### File: {filename}\n```{file_extension or 'text'}\n{text_content}\n```\n\n"
+            except UnicodeDecodeError:
+                content = f"### File: {filename}\n[Binary file - {size} bytes]\n\n"
+        else:
+            content = f"### File: {filename}\n[Binary file - {size} bytes]\n\n"
+    except Exception as e:
+        content = f"### File: {filename}\n[Error processing file: {str(e)}]\n\n"
+    
+    return content
+
+def create_markdown_document(url=None, files=None):
+    """Create markdown document from repo contents or uploaded files."""
+    if url:
+        owner, repo, contents, is_hf = get_repo_contents(url)
+        
+        if isinstance(contents, str):  # Error case
+            return f"Error: {contents}"
+        
+        markdown_content = f"# {'Space' if is_hf else 'Repository'}: {owner}/{repo}\n\n"
+        markdown_content += "## File Structure\n```\n"
+        markdown_content += generate_file_tree([item['path'] for item in contents])
+        markdown_content += "```\n\n"
+        markdown_content += f"Below are the contents of all files in the {'space' if is_hf else 'repository'}:\n\n"
+        
+        for item in contents:
+            markdown_content += process_file_content(item, owner, repo, is_hf)
+    else:  # Handle uploaded files
+        markdown_content = "# Uploaded Files\n\n"
+        markdown_content += "## File Structure\n```\n"
+        markdown_content += generate_file_tree([file.filename for file in files])
+        markdown_content += "```\n\n"
+        markdown_content += "Below are the contents of all uploaded files:\n\n"
+        for file in files:
+            markdown_content += process_uploaded_file(file)
+    
+    return markdown_content
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+@app.route('/process', methods=['POST'])
+def process():
+    # Ensure consistent response structure
+    response_data = {'markdown': '', 'html': '', 'filename': '', 'error': None}
+    
+    if 'files[]' in request.files:
+        files = request.files.getlist('files[]')
+        if not files:
+            response_data['error'] = 'No files uploaded'
+            return jsonify(response_data), 400
+        
+        markdown_content = create_markdown_document(files=files)
+        response_data['markdown'] = "```markdown\n" + markdown_content + "\n```"
+        response_data['html'] = markdown.markdown(markdown_content)
+        response_data['filename'] = "uploaded_files_summary.md"
+    else:
+        repo_url = request.json.get('repo_url')
+        if not repo_url:
+            response_data['error'] = 'Please provide a repository URL or upload files'
+            return jsonify(response_data), 400
+        
+        markdown_content = create_markdown_document(repo_url)
+        owner, repo, contents, is_hf = get_repo_contents(repo_url)
+        if not owner:
+            response_data['error'] = markdown_content  # Error message from get_repo_contents
+            return jsonify(response_data), 400
+        
+        response_data['markdown'] = markdown_content
+        response_data['html'] = markdown.markdown(markdown_content)
+        response_data['filename'] = f"{owner}_{repo}_summary.md"
+    
+    return jsonify(response_data)
+
+@app.route('/download', methods=['POST'])
+def download():
+    markdown_content = request.json.get('markdown')
+    filename = request.json.get('filename')
+    
+    buffer = io.BytesIO()
+    buffer.write(markdown_content.encode('utf-8'))
+    buffer.seek(0)
+    
+    return send_file(
+        buffer,
+        as_attachment=True,
+        download_name=filename,
+        mimetype='text/markdown'
+    )
+with open("html_template.html", "r") as f:
+    html_template=f.read()
+f.close()
+
+
+if not os.path.exists('templates'):
+    os.makedirs('templates')
+with open('templates/index.html', 'w') as f:
+    f.write(html_template)
+
+if __name__ == '__main__':
+    app.run(host="0.0.0.0", port=7860, debug=True)