Update js/index.js

js/index.js

@@ -79,15 +79,28 @@ document.getElementById('repoForm').addEventListener('submit', async function (e
 
             if (lastString) {
                 console.log("Fetching HF references for lastString:", lastString);
-                const refs = await getHuggingFaceReferences(owner, repo, repo_type, accessToken);
-                console.log("Available HF Refs:", refs);
-                const matchingRef = refs.find(ref => lastString.startsWith(ref + '/'));
-                if (matchingRef) {
-                    refFromUrl = matchingRef;
-                    pathFromUrl = lastString.slice(matchingRef.length + 1);
-                } else {
-                    refFromUrl = lastString.split('/')[0];
-                    pathFromUrl = lastString.substring(refFromUrl.length + 1);
+                // Note: HF API doesn't always require querying refs if we assume main, 
+                // but good for robustness if user pastes a non-main branch URL.
+                try {
+                    const refs = await getHuggingFaceReferences(owner, repo, repo_type, accessToken);
+                    console.log("Available HF Refs:", refs);
+                    const matchingRef = refs.find(ref => lastString.startsWith(ref + '/'));
+                    if (matchingRef) {
+                        refFromUrl = matchingRef;
+                        pathFromUrl = lastString.slice(matchingRef.length + 1);
+                    } else if (refs.includes(lastString)) {
+                        refFromUrl = lastString;
+                    } else {
+                        // Fallback logic if refs fetch fails or format is different
+                        const parts = lastString.split('/');
+                        refFromUrl = parts[0];
+                        pathFromUrl = parts.slice(1).join('/');
+                    }
+                } catch (e) {
+                    console.warn("Could not fetch HF refs, trying to parse from URL directly.", e);
+                    const parts = lastString.split('/');
+                    refFromUrl = parts[0] || 'main';
+                    pathFromUrl = parts.slice(1).join('/');
                 }
             }
             
@@ -96,7 +109,7 @@ document.getElementById('repoForm').addEventListener('submit', async function (e
             tree = await fetchHuggingFaceTree(owner, repo, repo_type, refFromUrl, pathFromUrl, accessToken);
         }
         
-        console.log("Final tree object before display:", tree);
+        console.log("Final tree object passed to display:", tree);
         displayDirectoryStructure(tree);
         document.getElementById('generateTextButton').style.display = 'flex';
         document.getElementById('downloadZipButton').style.display = 'flex';
@@ -133,27 +146,34 @@ document.getElementById('generateTextButton').addEventListener('click', async fu
         document.getElementById('copyButton').style.display = 'flex';
         document.getElementById('downloadButton').style.display = 'flex';
     } catch (error) {
+        console.error("Error generating text:", error);
         outputText.value = `Error generating text file: ${error.message}\n\n` +
             "Please ensure:\n" +
             "1. You have selected at least one file from the directory structure.\n" +
             "2. Your access token (if provided) is valid and has the necessary permissions.\n" +
-            "3. You have a stable internet connection.";
+            "3. You have a stable internet connection.\n" +
+            "4. Note: Very large files or LFS files might fail to fetch as text.";
     }
 });
 
 // Event listener for downloading zip file
 document.getElementById('downloadZipButton').addEventListener('click', async function () {
     const accessToken = document.getElementById('accessToken').value;
+    const outputText = document.getElementById('outputText');
+    outputText.value = 'Generating zip file...';
 
     try {
         const selectedFiles = getSelectedFiles();
         if (selectedFiles.length === 0) {
             throw new Error('No files selected');
         }
+        // For zip, we might want to handle binary files differently in future, 
+        // but currently fetchFileContents gets text. 
         const fileContents = await fetchFileContents(selectedFiles, accessToken, currentRepoInfo.source);
         await createAndDownloadZip(fileContents);
+        outputText.value = 'Zip file downloaded successfully.';
     } catch (error) {
-        const outputText = document.getElementById('outputText');
+        console.error("Error generating zip:", error);
         outputText.value = `Error generating zip file: ${error.message}\n\n` +
             "Please ensure:\n" +
             "1. You have selected at least one file from the directory structure.\n" +
@@ -188,36 +208,66 @@ document.getElementById('downloadButton').addEventListener('click', function ()
 
 // Parse GitHub or Hugging Face repository URL
 function parseRepoUrl(url) {
-    url = url.replace(/\/$/, '');
+    url = url.trim().replace(/\/$/, '');
+    // GitHub pattern
     const githubPattern = /^https:\/\/github\.com\/([^\/]+)\/([^\/]+)(?:\/tree\/(.+))?$/;
-    const hfPattern = /^https:\/\/huggingface\.co\/(?:(datasets|spaces)\/)?([^\/]+)\/([^\/]+)(?:\/tree\/(.+))?$/;
-
     let match = url.match(githubPattern);
     if (match) {
         return {
             source: 'github',
             owner: match[1],
             repo: match[2],
-            lastString: match[4] || ''
+            lastString: match[3] || ''
         };
     }
 
-    match = url.match(hfPattern);
-    if (match) {
+    // Hugging Face pattern
+    // Matches: https://huggingface.co/username/repo
+    // Matches: https://huggingface.co/datasets/username/repo
+    // Matches: https://huggingface.co/spaces/username/repo
+    // Handles optional /tree/branch/path
+    const hfUrlParts = new URL(url);
+    if (hfUrlParts.hostname === 'huggingface.co') {
+        const pathSegments = hfUrlParts.pathname.split('/').filter(Boolean);
         let repo_type = 'model';
-        if (match[1] === 'datasets') repo_type = 'dataset';
-        if (match[1] === 'spaces') repo_type = 'space';
-        
+        let owner, repo, treeIndex;
+
+        if (pathSegments[0] === 'datasets') {
+            repo_type = 'dataset';
+            owner = pathSegments[1];
+            repo = pathSegments[2];
+            treeIndex = pathSegments.indexOf('tree', 3);
+        } else if (pathSegments[0] === 'spaces') {
+            repo_type = 'space';
+            owner = pathSegments[1];
+            repo = pathSegments[2];
+            treeIndex = pathSegments.indexOf('tree', 3);
+        } else {
+            // Models don't have a prefix
+            owner = pathSegments[0];
+            repo = pathSegments[1];
+            treeIndex = pathSegments.indexOf('tree', 2);
+        }
+
+        if (!owner || !repo) {
+            throw new Error('Invalid Hugging Face URL format.');
+        }
+
+        let lastString = '';
+        if (treeIndex !== -1 && treeIndex + 1 < pathSegments.length) {
+            lastString = pathSegments.slice(treeIndex + 1).join('/');
+        }
+
         return {
             source: 'huggingface',
             repo_type: repo_type,
-            owner: match[2],
-            repo: match[3],
-            lastString: match[4] || ''
+            owner: owner,
+            repo: repo,
+            lastString: lastString
         };
     }
 
-    throw new Error('Invalid GitHub or Hugging Face repository URL.');
+    throw new Error('Invalid URL. Please provide a valid GitHub or Hugging Face repository URL.');
 }
 
 // Fetch GitHub repository references
@@ -251,15 +301,20 @@ async function getHuggingFaceReferences(owner, repo, repo_type, token) {
     if (token) headers['Authorization'] = `Bearer ${token}`;
 
     const response = await fetch(url, { headers });
-    if (!response.ok) handleFetchError(response, 'huggingface');
+    if (!response.ok) {
+        // HF might return 404 for refs on private repos without token, or just empty list.
+        console.warn("Could not fetch HF refs:", response.status);
+        return ['main']; // fallback
+    }
 
     const data = await response.json();
     const branches = data.branches ? data.branches.map(b => b.name) : [];
     const tags = data.tags ? data.tags.map(t => t.name) : [];
-    return [...branches, ...tags];
+    const refs = [...branches, ...tags];
+    return refs.length > 0 ? refs : ['main'];
 }
 
-// Fetch repository SHA
+// Fetch repository SHA (GitHub)
 async function fetchRepoSha(owner, repo, ref, path, token) {
     const url = `https://api.github.com/repos/${owner}/${repo}/contents/${path ? `${path}` : ''}${ref ? `?ref=${ref}` : ''}`;
     const headers = { 'Accept': 'application/vnd.github.object+json' };
@@ -269,7 +324,16 @@ async function fetchRepoSha(owner, repo, ref, path, token) {
     if (!response.ok) handleFetchError(response, 'github');
     
     const data = await response.json();
-    return data.sha;
+    // If path leads to a file, data is object. If dir, data is array.
+    if (Array.isArray(data)) {
+        // It's a directory, get last commit sha for this dir to get tree
+        const commitUrl = `https://api.github.com/repos/${owner}/${repo}/commits?path=${path || ''}&sha=${ref || ''}&per_page=1`;
+        const commitResp = await fetch(commitUrl, { headers });
+        if (!commitResp.ok) handleFetchError(commitResp, 'github');
+        const commitData = await commitResp.json();
+        return commitData[0].commit.tree.sha;
+    }
+    return data.sha; // It's a file
 }
 
 // Fetch GitHub repository tree
@@ -278,17 +342,27 @@ async function fetchGitHubRepoTree(owner, repo, sha, token) {
     const headers = { 'Accept': 'application/vnd.github+json' };
     if (token) headers['Authorization'] = `token ${token}`;
 
+    console.log("Fetching GitHub tree from:", url);
     const response = await fetch(url, { headers });
     if (!response.ok) handleFetchError(response, 'github');
     
     const data = await response.json();
-    return data.tree;
+    
+    // Map GitHub tree to internal format
+    return data.tree.map(item => ({
+        path: item.path,
+        type: item.type, // 'blob' or 'tree'
+        urlType: 'github',
+        url: item.url // git blob api url
+    }));
 }
 
 // Fetch Hugging Face repository tree
 async function fetchHuggingFaceTree(owner, repo, repo_type, ref, path, token) {
     const typePath = repo_type === 'model' ? 'models' : repo_type === 'dataset' ? 'datasets' : 'spaces';
-    const url = `https://huggingface.co/api/${typePath}/${owner}/${repo}/tree/${ref}`;
+    // FIX: Added ?recursive=true to get all files, including those in subfolders
+    const url = `https://huggingface.co/api/${typePath}/${owner}/${repo}/tree/${ref}?recursive=true`;
+    
     console.log("Fetching HF tree from URL:", url);
     
     const headers = {};
@@ -297,17 +371,20 @@ async function fetchHuggingFaceTree(owner, repo, repo_type, ref, path, token) {
     const response = await fetch(url, { headers });
     if (!response.ok) handleFetchError(response, 'huggingface');
     
-    let tree = await response.json();
-    console.log("Raw HF API response:", JSON.parse(JSON.stringify(tree)));
-
+    let rawTree = await response.json();
+    console.log("Raw HF API response (recursive):", rawTree);
 
+    // Filter by path if provided in URL
     if (path) {
-        console.log("Filtering HF tree with path:", path);
-        tree = tree.filter(item => item.path.startsWith(path + '/') || item.path === path);
-        console.log("Filtered HF tree:", JSON.parse(JSON.stringify(tree)));
+        console.log("Filtering HF tree by path:", path);
+        // Keep files that start with the path
+        rawTree = rawTree.filter(item => item.path.startsWith(path + '/') || item.path === path);
     }
     
-    const mappedTree = tree.map(item => {
+    // Map HF tree to internal format
+    // HF recursive tree returns only files. Directory objects are not included.
+    // This is good for utils.js as it builds directores from file paths.
+    const mappedTree = rawTree.map(item => {
         let repoIdForUrl;
         switch (repo_type) {
             case 'dataset':
@@ -319,14 +396,16 @@ async function fetchHuggingFaceTree(owner, repo, repo_type, ref, path, token) {
             default: // model
                 repoIdForUrl = `${owner}/${repo}`;
         }
-        const mappedItem = {
+        
+        return {
             path: item.path,
-            type: (item.type === 'file' || item.type === 'lfs') ? 'blob' : 'tree',
+            // HF types: 'file', 'lfs', 'directory' (shallow only). 
+            // With recursive=true, we mainly get files. Map to 'blob' for utils.js
+            type: (item.type === 'directory') ? 'tree' : 'blob', 
             urlType: 'hf',
-            url: `https://huggingface.co/${repoIdForUrl}/raw/${ref}/${item.path}`
+            // Construct raw URL
+            url: `https://huggingface.co/${repoIdForUrl}/resolve/${ref}/${item.path}`
         };
-        // console.log("Mapping item:", item, "to:", mappedItem);
-        return mappedItem;
     });
 
     console.log("Mapped HF tree:", mappedTree);
@@ -334,35 +413,59 @@ async function fetchHuggingFaceTree(owner, repo, repo_type, ref, path, token) {
 }
 
 // Handle fetch errors
-function handleFetchError(response, source = 'github') {
+function handleFetchError(response, source) {
+    console.error(`Fetch Error (${source}):`, response.status, response.statusText);
     if (response.status === 403 && source === 'github' && response.headers.get('X-RateLimit-Remaining') === '0') {
-        throw new Error('GitHub API rate limit exceeded. Please try again later or provide a valid access token to increase your rate limit.');
+        throw new Error('GitHub API rate limit exceeded. Please try again later or provide a valid access token.');
     }
     if (response.status === 401) {
-        throw new Error(`Authentication error. Please check if your access token is valid and has the required permissions.`);
+        throw new Error(`Authentication error. Please check if your access token is valid for ${source}.`);
     }
     if (response.status === 404) {
-        throw new Error(`Repository, branch, or path not found. Please check that the URL, branch/tag, and path are correct and accessible.`);
+        throw new Error(`Repository, branch, or path not found on ${source}. Check URL.`);
     }
-    throw new Error(`Failed to fetch repository data. Status: ${response.status}. Please check your input and try again.`);
+    throw new Error(`Failed to fetch data from ${source}. Status: ${response.status}.`);
 }
 
 // Fetch contents of selected files
 async function fetchFileContents(files, token, source) {
     const contents = await Promise.all(files.map(async file => {
+        console.log(`Fetching content for: ${file.path} from ${source}`);
         let headers = {};
-        if (token) {
-            headers['Authorization'] = source === 'github' ? `token ${token}` : `Bearer ${token}`;
-        }
+        let fetchUrl = file.url;
+
         if (source === 'github') {
+            if (token) headers['Authorization'] = `token ${token}`;
             headers['Accept'] = 'application/vnd.github.v3.raw';
+        } else if (source === 'huggingface') {
+            if (token) headers['Authorization'] = `Bearer ${token}`;
+            // HF raw URLs don't need specific accept headers for text
         }
         
-        const response = await fetch(file.url, { headers });
-        if (!response.ok) handleFetchError(response, source);
-        
-        const text = await response.text();
-        return { url: file.url, path: file.path, text };
+        try {
+            const response = await fetch(fetchUrl, { headers });
+            if (!response.ok) handleFetchError(response, source);
+            
+            // Check for LFS pointer in HF
+            if (source === 'huggingface') {
+                const contentLength = response.headers.get('content-length');
+                if (contentLength && parseInt(contentLength) > 5000000) { // Arbitrary 5MB limit for text representation
+                    return { url: file.url, path: file.path, text: `[File too large to display directly: ${contentLength} bytes]` };
+                }
+            }
+
+            const text = await response.text();
+            
+            // Basic check if it looks like an LFS pointer
+            if (source === 'huggingface' && text.startsWith('version https://git-lfs.github.com/spec/v1')) {
+                 return { url: file.url, path: file.path, text: `[Git LFS Pointer]\n${text}` };
+            }
+
+            return { url: file.url, path: file.path, text };
+        } catch (error) {
+            console.error(`Error fetching ${file.path}:`, error);
+            return { url: file.url, path: file.path, text: `[Error fetching file: ${error.message}]` };
+        }
     }));
     return contents;
 }
@@ -390,6 +493,7 @@ async function createAndDownloadZip(fileContents) {
     const zip = new JSZip();
 
     fileContents.forEach(file => {
+        // Remove leading slash if present for zip path
         const filePath = file.path.startsWith('/') ? file.path.slice(1) : file.path;
         zip.file(filePath, file.text);
     });
@@ -398,7 +502,7 @@ async function createAndDownloadZip(fileContents) {
     const url = URL.createObjectURL(content);
     const a = document.createElement('a');
     a.href = url;
-    a.download = 'repo_contents.zip';
+    a.download = `${currentRepoInfo.repo}_contents.zip`;
     document.body.appendChild(a);
     a.click();
     document.body.removeChild(a);