Spaces:
Sleeping
Sleeping
| from flask import Flask, render_template, request, jsonify, send_file | |
| from huggingface_hub import HfApi | |
| import requests | |
| import base64 | |
| import markdown | |
| import json | |
| import mimetypes | |
| import os | |
| import io | |
| import uuid | |
| from pathlib import Path | |
| app = Flask(__name__) | |
| GITHUB_API = "https://api.github.com/repos/" | |
| #HF_API = "https://huggingface.co/api/spaces/" | |
| def generate_file_tree(paths): | |
| """Generate a simple file tree from a list of paths.""" | |
| print("generating file tree") | |
| tree = ["📁 Root"] | |
| sorted_paths = sorted(paths) | |
| for path in sorted_paths: | |
| parts = path.split('/') | |
| indent = " " * (len(parts) - 1) | |
| tree.append(f"{indent}📄 {parts[-1]}") | |
| print("generating file tree - Complete") | |
| return "\n".join(tree) + "\n\n" | |
| def get_all_files(owner, repo, path="", is_hf=False): | |
| """Recursively fetch all files from a repository.""" | |
| if is_hf: | |
| # Attempt to fetch file list from Hugging Face Space (publicly accessible files) | |
| api_url = f"https://huggingface.co/spaces/{owner}/{repo}/tree/main/{path}".rstrip('/') | |
| else: | |
| api_url = f"{GITHUB_API}{owner}/{repo}/contents/{path}".rstrip('/') | |
| try: | |
| response = requests.get(api_url) | |
| print(response.content) | |
| response = requests.get(api_url, headers={"Accept": "application/json"}) | |
| print(str(response.json())) | |
| response.raise_for_status() | |
| items = response.json() | |
| # Hugging Face might not return JSON in the same format; adjust if HTML is returned | |
| if isinstance(items, str): # If response isn’t JSON, it’s likely HTML | |
| return None # Fallback to error handling | |
| files = [] | |
| for item in items: | |
| if item['type'] == 'file': | |
| files.append(item) | |
| elif item['type'] == 'dir': | |
| files.extend(get_all_files(owner, repo, item['path'], is_hf)) | |
| print(files) | |
| return files | |
| except Exception as e: | |
| return None | |
| def get_hf_files(repo,name,path=""): | |
| api = HfApi() | |
| f_ist = (api.list_repo_files(repo_id=f'{repo}/{name}', repo_type="space")) | |
| print (f_ist) | |
| file_list = [] | |
| #file_out = [] | |
| if not os.path.exists(name): | |
| os.makedirs(name) | |
| for d_app in f_ist: | |
| if "/" in d_app: | |
| dir_1=d_app.split("/",1)[0] | |
| file_list.extend(get_hf_files(repo,name,dir_1)) | |
| ''' | |
| rem_1=d_app.split("/",1)[1] | |
| if not os.path.exists(f'{name}/{dir_1}'): | |
| os.makedirs(f'{name}/{dir_1}') | |
| if "/" in rem_1: | |
| dir_2=rem_1.split("/",1)[0] | |
| rem_2=rem_1.split("/",1)[1] | |
| if not os.path.exists(f'{name}/{dir_1}/{dir_2}'): | |
| os.makedirs(f'{name}/{dir_1}/{dir_2}') | |
| sf=rem_2.split(".",1)[1] | |
| pf=rem_2.split(".",1)[0] | |
| f_name=f'{dir_1}/{dir2}/{pf}.{sf}' | |
| else: | |
| sf=rem_1.split(".",1)[1] | |
| pf=rem_1.split(".",1)[0] | |
| f_name=f'{dir_1}/{pf}.{sf}' | |
| print(f_name) | |
| ''' | |
| else: | |
| sf=d_app.split(".",1)[1] | |
| pf=d_app.split(".",1)[0] | |
| f_name=f'{pf}.{sf}' | |
| pass | |
| r = requests.get(f'https://huggingface.co/spaces/{repo}/{name}/raw/main/{d_app}') | |
| print(d_app) | |
| #print (r.text) | |
| uid = uuid.uuid4() | |
| file = open(f'{name}/{f_name}','wb') | |
| file.write(r.content) | |
| file.close() | |
| file_list.append({"path":f'{d_app}'}) | |
| print(file_list) | |
| return file_list | |
| def get_repo_contents(url): | |
| """Parse URL and fetch repository contents.""" | |
| try: | |
| if "huggingface.co" in url: | |
| parts = url.rstrip('/').split('/') | |
| owner, repo = parts[-2], parts[-1] | |
| # Fallback approach: manually fetch known files or use a simpler file list | |
| # For now, assume a flat structure and fetch known files directly | |
| # This is a workaround until a proper API token or endpoint is confirmed | |
| known_files = [ | |
| {'path': 'app.py', 'type': 'file'}, | |
| {'path': 'README.md', 'type': 'file'} | |
| # Add more known paths or implement HTML scraping if needed | |
| ] | |
| #files = get_all_files(owner, repo, "", True) or known_files | |
| files = get_hf_files(owner, repo) | |
| return owner, repo, files, True | |
| else: # Assume GitHub URL | |
| parts = url.rstrip('/').split('/') | |
| owner, repo = parts[-2], parts[-1] | |
| files = get_all_files(owner, repo, "", False) | |
| if files is None: | |
| raise Exception("Failed to fetch GitHub repository contents") | |
| return owner, repo, files, False | |
| except Exception as e: | |
| return None, None, f"Error fetching repo contents: {str(e)}", False | |
| def process_file_content(file_info, owner, repo, is_hf=False): | |
| """Process individual file content from a repository.""" | |
| content = "" | |
| file_path = file_info['path'] | |
| try: | |
| if is_hf: | |
| file_url = f"https://huggingface.co/spaces/{owner}/{repo}/raw/main/{file_path}" | |
| else: | |
| file_url = f"{GITHUB_API}{owner}/{repo}/contents/{file_path}" | |
| response = requests.get(file_url) | |
| response.raise_for_status() | |
| if is_hf: | |
| content_raw = response.content | |
| size = len(content_raw) | |
| file_extension = file_path.split('.')[-1] if '.' in file_path else '' | |
| mime_type, _ = mimetypes.guess_type(file_path) | |
| is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json'] | |
| if is_text: | |
| try: | |
| text_content = content_raw.decode('utf-8') | |
| if file_extension == 'json': | |
| try: | |
| json_data = json.loads(text_content) | |
| formatted_json = json.dumps(json_data, indent=2) | |
| content = f"### File: {file_path}\n```json\n{formatted_json}\n```\n\n" | |
| except json.JSONDecodeError: | |
| content = f"### File: {file_path}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n" | |
| else: | |
| content = f"### File: {file_path}\n```{file_extension or 'text'}\n{text_content}\n```\n\n" | |
| except UnicodeDecodeError: | |
| content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n" | |
| else: | |
| content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n" | |
| else: # GitHub | |
| data = response.json() | |
| if 'content' in data: | |
| content_raw = base64.b64decode(data['content']) | |
| size = data['size'] | |
| file_extension = file_path.split('.')[-1] if '.' in file_path else '' | |
| mime_type, _ = mimetypes.guess_type(file_path) | |
| is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json'] | |
| if is_text: | |
| try: | |
| text_content = content_raw.decode('utf-8') | |
| if file_extension == 'json': | |
| try: | |
| json_data = json.loads(text_content) | |
| formatted_json = json.dumps(json_data, indent=2) | |
| content = f"### File: {file_path}\n```json\n{formatted_json}\n```\n\n" | |
| except json.JSONDecodeError: | |
| content = f"### File: {file_path}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n" | |
| else: | |
| content = f"### File: {file_path}\n```{file_extension or 'text'}\n{text_content}\n```\n\n" | |
| except UnicodeDecodeError: | |
| content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n" | |
| else: | |
| content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n" | |
| else: | |
| content = f"### File: {file_path}\n[No content available]\n\n" | |
| except Exception as e: | |
| content = f"### File: {file_path}\n[Error fetching file content: {str(e)}]\n\n" | |
| return content | |
| def process_uploaded_file(file): | |
| """Process uploaded file content.""" | |
| content = "" | |
| filename = file.filename | |
| file_extension = filename.split('.')[-1] if '.' in filename else '' | |
| try: | |
| content_raw = file.read() # Read file content into memory | |
| size = len(content_raw) # Compute size in bytes | |
| mime_type, _ = mimetypes.guess_type(filename) | |
| is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json'] | |
| if is_text: | |
| try: | |
| text_content = content_raw.decode('utf-8') | |
| if file_extension == 'json': | |
| try: | |
| json_data = json.loads(text_content) | |
| formatted_json = json.dumps(json_data, indent=2) | |
| content = f"### File: {filename}\n```json\n{formatted_json}\n```\n\n" | |
| except json.JSONDecodeError: | |
| content = f"### File: {filename}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n" | |
| else: | |
| content = f"### File: {filename}\n```{file_extension or 'text'}\n{text_content}\n```\n\n" | |
| except UnicodeDecodeError: | |
| content = f"### File: {filename}\n[Binary file - {size} bytes]\n\n" | |
| else: | |
| content = f"### File: {filename}\n[Binary file - {size} bytes]\n\n" | |
| except Exception as e: | |
| content = f"### File: {filename}\n[Error processing file: {str(e)}]\n\n" | |
| return content | |
| def create_markdown_document(url=None, files=None): | |
| """Create markdown document from repo contents or uploaded files.""" | |
| if url: | |
| owner, repo, contents, is_hf = get_repo_contents(url) | |
| if isinstance(contents, str): # Error case | |
| return f"Error: {contents}" | |
| markdown_content = f"# {'Space' if is_hf else 'Repository'}: {owner}/{repo}\n\n" | |
| markdown_content += "## File Structure\n```\n" | |
| markdown_content += generate_file_tree([item['path'] for item in contents]) | |
| markdown_content += "```\n\n" | |
| markdown_content += f"Below are the contents of all files in the {'space' if is_hf else 'repository'}:\n\n" | |
| for item in contents: | |
| markdown_content += process_file_content(item, owner, repo, is_hf) | |
| else: # Handle uploaded files | |
| markdown_content = "# Uploaded Files\n\n" | |
| markdown_content += "## File Structure\n```\n" | |
| markdown_content += generate_file_tree([file.filename for file in files]) | |
| markdown_content += "```\n\n" | |
| markdown_content += "Below are the contents of all uploaded files:\n\n" | |
| for file in files: | |
| markdown_content += process_uploaded_file(file) | |
| return markdown_content | |
| def index(): | |
| return render_template('index.html') | |
| def process(): | |
| # Ensure consistent response structure | |
| response_data = {'markdown': '', 'html': '', 'filename': '', 'error': None} | |
| if 'files[]' in request.files: | |
| files = request.files.getlist('files[]') | |
| if not files: | |
| response_data['error'] = 'No files uploaded' | |
| return jsonify(response_data), 400 | |
| markdown_content = create_markdown_document(files=files) | |
| response_data['markdown'] = "```markdown\n" + markdown_content + "\n```" | |
| response_data['html'] = markdown.markdown(markdown_content) | |
| response_data['filename'] = "uploaded_files_summary.md" | |
| else: | |
| repo_url = request.json.get('repo_url') | |
| if not repo_url: | |
| response_data['error'] = 'Please provide a repository URL or upload files' | |
| return jsonify(response_data), 400 | |
| markdown_content = create_markdown_document(repo_url) | |
| owner, repo, contents, is_hf = get_repo_contents(repo_url) | |
| if not owner: | |
| response_data['error'] = markdown_content # Error message from get_repo_contents | |
| return jsonify(response_data), 400 | |
| response_data['markdown'] = markdown_content | |
| response_data['html'] = markdown.markdown(markdown_content) | |
| response_data['filename'] = f"{owner}_{repo}_summary.md" | |
| return jsonify(response_data) | |
| def download(): | |
| markdown_content = request.json.get('markdown') | |
| filename = request.json.get('filename') | |
| buffer = io.BytesIO() | |
| buffer.write(markdown_content.encode('utf-8')) | |
| buffer.seek(0) | |
| return send_file( | |
| buffer, | |
| as_attachment=True, | |
| download_name=filename, | |
| mimetype='text/markdown' | |
| ) | |
| with open("html_template.html", "r") as f: | |
| html_template=f.read() | |
| f.close() | |
| if not os.path.exists('templates'): | |
| os.makedirs('templates') | |
| with open('templates/index.html', 'w') as f: | |
| f.write(html_template) | |
| if __name__ == '__main__': | |
| app.run(host="0.0.0.0", port=7860, debug=True) |