Add markdown rendering support for VLM output

css/styles.css

@@ -194,4 +194,100 @@ button:focus, input:focus, select:focus {
     counter-increment: line;
     content: counter(line);
     @apply inline-block w-12 mr-4 text-right text-gray-400 dark:text-gray-600 select-none;
+}
+
+/* Markdown content styling */
+.markdown-content {
+    @apply text-gray-900 dark:text-gray-100;
+}
+
+.markdown-content h1 {
+    @apply text-2xl font-bold mb-4 text-gray-900 dark:text-gray-100;
+}
+
+.markdown-content h2 {
+    @apply text-xl font-bold mb-3 text-gray-900 dark:text-gray-100;
+}
+
+.markdown-content h3 {
+    @apply text-lg font-bold mb-2 text-gray-900 dark:text-gray-100;
+}
+
+.markdown-content h4 {
+    @apply text-base font-bold mb-2 text-gray-900 dark:text-gray-100;
+}
+
+.markdown-content p {
+    @apply mb-4 leading-relaxed;
+}
+
+.markdown-content ul {
+    @apply list-disc list-inside mb-4 pl-4;
+}
+
+.markdown-content ol {
+    @apply list-decimal list-inside mb-4 pl-4;
+}
+
+.markdown-content li {
+    @apply mb-1;
+}
+
+.markdown-content blockquote {
+    @apply border-l-4 border-gray-300 dark:border-gray-600 pl-4 my-4 italic text-gray-700 dark:text-gray-300;
+}
+
+.markdown-content code {
+    @apply bg-gray-100 dark:bg-gray-800 px-1 py-0.5 rounded text-sm font-mono;
+}
+
+.markdown-content pre {
+    @apply bg-gray-100 dark:bg-gray-800 p-4 rounded-lg overflow-x-auto mb-4;
+}
+
+.markdown-content pre code {
+    @apply bg-transparent p-0;
+}
+
+.markdown-content a {
+    @apply text-blue-600 dark:text-blue-400 underline hover:text-blue-800 dark:hover:text-blue-300;
+}
+
+.markdown-content table {
+    @apply w-full mb-4 border-collapse;
+    display: table !important;
+}
+
+.markdown-content th, .markdown-content td {
+    @apply border border-gray-300 dark:border-gray-600 px-3 py-2;
+}
+
+.markdown-content th {
+    @apply bg-gray-100 dark:bg-gray-800 font-bold;
+}
+
+/* Override prose styles for tables */
+.prose table {
+    @apply w-full mb-4 border-collapse;
+    display: table !important;
+}
+
+.prose thead {
+    @apply bg-gray-50 dark:bg-gray-800;
+}
+
+.prose tbody {
+    @apply bg-white dark:bg-gray-900;
+}
+
+.prose th {
+    @apply px-3 py-3.5 text-left text-sm font-semibold text-gray-900 dark:text-gray-100 border border-gray-300 dark:border-gray-600;
+}
+
+.prose td {
+    @apply px-3 py-4 text-sm text-gray-900 dark:text-gray-100 border border-gray-300 dark:border-gray-600;
+}
+
+.markdown-content hr {
+    @apply border-t border-gray-300 dark:border-gray-600 my-6;
 }

index.html

@@ -9,6 +9,7 @@
     <script src="https://unpkg.com/htmx.org@1.9.10"></script>
     <script src="https://unpkg.com/alpinejs@3.13.3/dist/cdn.min.js" defer></script>
     <script src="https://cdn.tailwindcss.com"></script>
+    <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
     
     <!-- Tailwind Config -->
     <script>
@@ -98,8 +99,21 @@
                         <option value="char">Character Diff</option>
                         <option value="word">Word Diff</option>
                         <option value="line">Line Diff</option>
+                        <option value="markdown" x-show="hasMarkdown">Markdown Diff</option>
                     </select>
                     
+                    <button
+                        x-show="hasMarkdown"
+                        @click="renderMarkdown = !renderMarkdown"
+                        :class="renderMarkdown ? 'bg-blue-100 dark:bg-blue-900 text-blue-700 dark:text-blue-300' : 'text-gray-500 hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-200'"
+                        class="p-2 rounded-md transition-colors"
+                        title="Toggle markdown rendering"
+                    >
+                        <svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                            <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z"></path>
+                        </svg>
+                    </button>
+                    
                     <button
                         @click="exportComparison()"
                         class="p-2 text-gray-500 hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-200"
@@ -241,6 +255,14 @@
                                         <span x-text="wordStats.original || '-'"></span> → <span x-text="wordStats.improved || '-'"></span>
                                     </span>
                                 </div>
+                                <div x-show="hasMarkdown" class="mt-2 flex items-center justify-center">
+                                    <span class="inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium bg-purple-100 dark:bg-purple-900 text-purple-800 dark:text-purple-200">
+                                        <svg class="w-3 h-3 mr-1" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                                            <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z"></path>
+                                        </svg>
+                                        Markdown Detected
+                                    </span>
+                                </div>
                             </div>
                         </div>
                     </div>
@@ -287,9 +309,17 @@
                             </div>
                         </div>
                         <div>
-                            <h3 class="text-sm font-medium text-gray-700 dark:text-gray-300 mb-3">Improved OCR</h3>
-                            <div class="prose prose-sm dark:prose-invert max-w-none">
-                                <pre class="whitespace-pre-wrap font-mono text-xs bg-gray-50 dark:bg-gray-800 text-gray-900 dark:text-gray-100 p-4 rounded-lg" x-text="getImprovedText()"></pre>
+                            <h3 class="text-sm font-medium text-gray-700 dark:text-gray-300 mb-3">
+                                Improved OCR
+                                <span x-show="renderMarkdown && hasMarkdown" class="ml-2 text-xs bg-blue-100 dark:bg-blue-900 text-blue-700 dark:text-blue-300 px-2 py-1 rounded">
+                                    Markdown
+                                </span>
+                            </h3>
+                            <div class="max-w-none">
+                                <div x-show="!renderMarkdown">
+                                    <pre class="whitespace-pre-wrap font-mono text-xs bg-gray-50 dark:bg-gray-800 text-gray-900 dark:text-gray-100 p-4 rounded-lg" x-text="getImprovedText()"></pre>
+                                </div>
+                                <div x-show="renderMarkdown" x-html="getImprovedTextRendered()" class="markdown-content"></div>
                             </div>
                         </div>
                     </div>
@@ -300,8 +330,11 @@
                     </div>
                     
                     <!-- Improved Only -->
-                    <div x-show="activeTab === 'improved'" class="prose prose-sm dark:prose-invert max-w-none">
-                        <pre class="whitespace-pre-wrap font-mono text-xs bg-gray-50 dark:bg-gray-800 text-gray-900 dark:text-gray-100 p-4 rounded-lg" x-text="getImprovedText()"></pre>
+                    <div x-show="activeTab === 'improved'" class="max-w-none">
+                        <div x-show="!renderMarkdown">
+                            <pre class="whitespace-pre-wrap font-mono text-xs bg-gray-50 dark:bg-gray-800 text-gray-900 dark:text-gray-100 p-4 rounded-lg" x-text="getImprovedText()"></pre>
+                        </div>
+                        <div x-show="renderMarkdown" x-html="getImprovedTextRendered()" class="markdown-content"></div>
                     </div>
                 </div>
             </div>

js/app.js

@@ -24,6 +24,8 @@ document.addEventListener('alpine:init', () => {
         showAbout: false,
         showFlowView: false,
         showDock: false,
+        renderMarkdown: false,
+        hasMarkdown: false,
         
         // Flow view state
         flowItems: [],
@@ -48,6 +50,9 @@ document.addEventListener('alpine:init', () => {
         // API instance
         api: null,
         
+        // Markdown cache
+        markdownCache: new Map(),
+        
         async init() {
             // Initialize API
             this.api = new DatasetAPI();
@@ -125,6 +130,9 @@ document.addEventListener('alpine:init', () => {
             this.loading = true;
             this.error = null;
             
+            // Clear markdown cache when loading new dataset
+            this.markdownCache.clear();
+            
             try {
                 // Validate dataset
                 await this.api.validateDataset(this.datasetId);
@@ -164,6 +172,10 @@ document.addEventListener('alpine:init', () => {
                 this.currentSample = data.row;
                 this.currentIndex = index;
                 
+                // Check if improved text contains markdown
+                const improvedText = this.getImprovedText();
+                this.hasMarkdown = this.detectMarkdown(improvedText);
+                
                 // Update diff when sample changes
                 this.updateDiff();
                 
@@ -215,6 +227,139 @@ document.addEventListener('alpine:init', () => {
             return this.currentSample[columns.improvedText] || 'No improved text found';
         },
         
+        detectMarkdown(text) {
+            // Check for common markdown patterns
+            const markdownPatterns = [
+                /^#{1,6}\s/m,           // Headers
+                /\*\*[^*]+\*\*/,        // Bold
+                /\*[^*]+\*/,            // Italic
+                /\[[^\]]+\]\([^)]+\)/,  // Links
+                /^[-*+]\s/m,            // Lists
+                /^\d+\.\s/m,            // Numbered lists
+                /^>/m,                  // Blockquotes
+                /```[\s\S]*?```/,       // Code blocks
+                /`[^`]+`/,              // Inline code
+                /\|.*\|.*\|/m,          // Tables (basic detection)
+                /<table>/i,             // HTML tables
+                /<thead>/i              // HTML table headers
+            ];
+            
+            return markdownPatterns.some(pattern => pattern.test(text));
+        },
+        
+        escapeHtml(text) {
+            const div = document.createElement('div');
+            div.textContent = text;
+            return div.innerHTML;
+        },
+        
+        renderMarkdownText(text) {
+            if (!text || !this.renderMarkdown) return text;
+            
+            // Check cache first
+            const cacheKey = `${this.currentIndex}_${text.substring(0, 100)}`;
+            if (this.markdownCache.has(cacheKey)) {
+                return this.markdownCache.get(cacheKey);
+            }
+            
+            try {
+                // Configure marked options for security
+                const renderer = new marked.Renderer();
+                
+                // Override link rendering to open in new tab and sanitize
+                const self = this;
+                renderer.link = function(href, title, text) {
+                    // Basic URL sanitization
+                    const safeHref = href.replace(/javascript:/gi, '').replace(/data:/gi, '');
+                    const safeTitle = (title || '').replace(/"/g, '&quot;');
+                    const safeText = self.escapeHtml(text);
+                    return `<a href="${safeHref}" title="${safeTitle}" target="_blank" rel="noopener noreferrer" class="text-blue-600 dark:text-blue-400 underline">${safeText}</a>`;
+                };
+                
+                // Override image rendering for safety
+                renderer.image = function(href, title, text) {
+                    const safeHref = href.replace(/javascript:/gi, '').replace(/data:/gi, '');
+                    const safeTitle = (title || '').replace(/"/g, '&quot;');
+                    const safeAlt = self.escapeHtml(text);
+                    return `<img src="${safeHref}" alt="${safeAlt}" title="${safeTitle}" class="max-w-full h-auto rounded">`;
+                };
+                
+                // Override HTML rendering to prevent XSS but allow safe table elements
+                renderer.html = function(html) {
+                    // Allow specific safe HTML tags for tables
+                    const allowedTags = ['table', 'thead', 'tbody', 'tr', 'th', 'td'];
+                    const tagPattern = new RegExp(`</?(?:${allowedTags.join('|')})(?:\\s[^>]*)?>`, 'gi');
+                    
+                    // Check if the HTML contains only allowed tags
+                    const strippedHtml = html.replace(tagPattern, '');
+                    const hasDisallowedTags = /<[^>]+>/.test(strippedHtml);
+                    
+                    if (!hasDisallowedTags) {
+                        // Return the HTML if it only contains allowed table tags
+                        return html;
+                    }
+                    
+                    // Strip all HTML by default
+                    return '';
+                };
+                
+                marked.setOptions({
+                    renderer: renderer,
+                    breaks: true,        // Convert \n to <br>
+                    gfm: true,          // GitHub Flavored Markdown
+                    pedantic: false,
+                    smartLists: true,
+                    smartypants: true,
+                    headerIds: false,    // Disable header IDs for security
+                    mangle: false,       // Don't mangle email addresses
+                    sanitize: false,     // We handle sanitization ourselves
+                    tables: true         // Enable table parsing
+                });
+                
+                // Render markdown
+                let html = marked.parse(text);
+                
+                // Add Tailwind classes to common elements
+                html = html.replace(/<h1>/g, '<h1 class="text-2xl font-bold mb-4 text-gray-900 dark:text-gray-100">');
+                html = html.replace(/<h2>/g, '<h2 class="text-xl font-bold mb-3 text-gray-900 dark:text-gray-100">');
+                html = html.replace(/<h3>/g, '<h3 class="text-lg font-bold mb-2 text-gray-900 dark:text-gray-100">');
+                html = html.replace(/<h4>/g, '<h4 class="text-base font-bold mb-2 text-gray-900 dark:text-gray-100">');
+                html = html.replace(/<p>/g, '<p class="mb-4 text-gray-900 dark:text-gray-100">');
+                html = html.replace(/<ul>/g, '<ul class="list-disc list-inside mb-4 text-gray-900 dark:text-gray-100">');
+                html = html.replace(/<ol>/g, '<ol class="list-decimal list-inside mb-4 text-gray-900 dark:text-gray-100">');
+                html = html.replace(/<li>/g, '<li class="mb-1">');
+                html = html.replace(/<blockquote>/g, '<blockquote class="border-l-4 border-gray-300 dark:border-gray-600 pl-4 my-4 italic text-gray-700 dark:text-gray-300">');
+                html = html.replace(/<code>/g, '<code class="bg-gray-100 dark:bg-gray-800 px-1 py-0.5 rounded text-sm font-mono">');
+                html = html.replace(/<pre>/g, '<pre class="bg-gray-100 dark:bg-gray-800 p-4 rounded-lg overflow-x-auto mb-4">');
+                html = html.replace(/<table>/g, '<table class="min-w-full divide-y divide-gray-300 dark:divide-gray-600 mb-4 border border-gray-300 dark:border-gray-600">');
+                html = html.replace(/<thead>/g, '<thead class="bg-gray-50 dark:bg-gray-800">');
+                html = html.replace(/<tbody>/g, '<tbody class="bg-white dark:bg-gray-900 divide-y divide-gray-200 dark:divide-gray-700">');
+                html = html.replace(/<tr>/g, '<tr class="hover:bg-gray-50 dark:hover:bg-gray-800">');
+                html = html.replace(/<th>/g, '<th class="px-3 py-3.5 text-left text-sm font-semibold text-gray-900 dark:text-gray-100 border border-gray-300 dark:border-gray-600">');
+                html = html.replace(/<td>/g, '<td class="px-3 py-4 text-sm text-gray-900 dark:text-gray-100 border border-gray-300 dark:border-gray-600">');
+                
+                const result = `<div class="prose prose-sm dark:prose-invert max-w-none">${html}</div>`;
+                
+                // Cache the result (limit cache size to prevent memory issues)
+                if (this.markdownCache.size > 50) {
+                    // Remove oldest entries
+                    const firstKey = this.markdownCache.keys().next().value;
+                    this.markdownCache.delete(firstKey);
+                }
+                this.markdownCache.set(cacheKey, result);
+                
+                return result;
+            } catch (error) {
+                console.error('Markdown rendering error:', error);
+                return text;
+            }
+        },
+        
+        getImprovedTextRendered() {
+            const text = this.getImprovedText();
+            return this.renderMarkdownText(text);
+        },
+        
         getImageData() {
             if (!this.currentSample) return null;
             const columns = this.api.detectColumns(null, this.currentSample);
@@ -252,6 +397,10 @@ document.addEventListener('alpine:init', () => {
                 case 'line':
                     this.diffHtml = createLineDiff(original, improved);
                     break;
+                case 'markdown':
+                    // Pass the render function bound to this context
+                    this.diffHtml = createMarkdownDiff(original, improved, (text) => this.renderMarkdownText(text));
+                    break;
             }
         },
         

js/diff-utils.js

@@ -216,4 +216,42 @@ function calculateSimilarity(original, improved) {
     const maxLength = Math.max(original.length, improved.length);
     
     return Math.round((lcsLength / maxLength) * 100);
+}
+
+/**
+ * Create markdown-aware diff showing original text vs rendered markdown
+ */
+function createMarkdownDiff(original, improved, renderFunction) {
+    if (!original || !improved) {
+        return '<p class="text-gray-500">No text to compare</p>';
+    }
+    
+    let html = '<div class="grid grid-cols-2 gap-6">';
+    
+    // Original text (plain)
+    html += '<div>';
+    html += '<h4 class="text-sm font-medium text-gray-700 dark:text-gray-300 mb-3">Original (Plain Text)</h4>';
+    html += '<div class="font-mono text-xs bg-gray-50 dark:bg-gray-800 text-gray-900 dark:text-gray-100 p-4 rounded-lg overflow-x-auto">';
+    html += '<pre class="whitespace-pre-wrap">' + escapeHtml(original) + '</pre>';
+    html += '</div>';
+    html += '</div>';
+    
+    // Improved text (rendered markdown)
+    html += '<div>';
+    html += '<h4 class="text-sm font-medium text-gray-700 dark:text-gray-300 mb-3">Improved (Rendered Markdown)</h4>';
+    html += '<div class="bg-gray-50 dark:bg-gray-800 p-4 rounded-lg overflow-x-auto">';
+    
+    // Render the markdown using the provided function
+    if (renderFunction && typeof renderFunction === 'function') {
+        html += renderFunction(improved);
+    } else {
+        // Fallback to plain text if no render function provided
+        html += '<pre class="whitespace-pre-wrap font-mono text-xs">' + escapeHtml(improved) + '</pre>';
+    }
+    
+    html += '</div>';
+    html += '</div>';
+    html += '</div>';
+    
+    return html;
 }