Simplify interface: focus on reconstruction test with embedding count and compression stats

app.py

@@ -1,730 +1,270 @@
 """
-B2NL-IntelligentTokenizer v6.2.1 - Gradio Demo
-
-⚠️ IMPORTANT: Currently in AUTOREGRESSIVE MODE (Teacher Forcing Training)
-- Current: ~500ms inference (accurate but slow)
-- Coming Soon (November 2025): Non-autoregressive training (<50ms, 10x faster)
-
-🚀 Progressive Byte-to-Natural Language Tokenizer with 16:1 Fixed Compression
-📊 Embedding Preprocessing Model for Inter-modal Communication
-🌐 Trained on FLORES-200 dataset supporting 204 languages
-
-Key Features:
-- Fixed 16:1 compression ratio (3 tokens per 48-byte chunk)
-- Autoregressive reconstruction with high accuracy
-- Sliding window processing for long texts
-- Real-time compression statistics
-- Multi-language support with semantic preservation
-
-Architecture:
-- Encoder: 4-layer transformer with progressive splitting
-- Decoder: 6-layer transformer with cross-attention
-- Total Parameters: 230.3M
-- Gumbel-Softmax for differentiable token selection
-
-Purpose:
-This model serves as a preprocessing layer that converts raw text into compressed
-semantic embeddings, enabling efficient inter-modal communication between different
-AI systems. By separating language understanding from task-specific inference,
-it provides a universal representation layer for multi-modal AI applications.
+B2NL-IntelligentTokenizer v6.2.1 - Simple Demo
 """
 
 import gradio as gr
 import torch
-import torch.nn.functional as F
-import numpy as np
 import sys
 import io
-from pathlib import Path
 import time
-from typing import Dict, List, Tuple, Optional
-from difflib import SequenceMatcher
+from pathlib import Path
 
-# Fix Windows Unicode output
+# Fix Windows Unicode
 if sys.platform == 'win32':
     sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
     sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
 
-# Add project paths
-sys.path.insert(0, str(Path(__file__).parent.parent.parent / "intelligent-tokenizer_v6.2.1"))
-sys.path.insert(0, str(Path(__file__).parent.parent.parent / "intelligent-tokenizer_v6.2.1/core"))
-
-try:
-    from core.unified_model import IntelligentTokenizerV62
-    from core.tokenizer import ByteTokenizerV62
-except ImportError:
-    print("Warning: Could not import from core, trying alternative path...")
-    from unified_model import IntelligentTokenizerV62
-    from tokenizer import ByteTokenizerV62
-
-# Global variables
-model = None
-device = None
-tokenizer = None
-
-def load_model(checkpoint_path: str = None):
-    """
-    Load the trained B2NL-IntelligentTokenizer model
-
-    This loads the checkpoint containing the trained weights from
-    100 epochs of training on the FLORES-200 dataset.
-    """
-    global model, device, tokenizer
-
-    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-    print(f"Using device: {device}")
-
-    # Initialize model
-    model = IntelligentTokenizerV62()
-
-    # Load checkpoint if provided
-    if checkpoint_path and Path(checkpoint_path).exists():
-        print(f"Loading checkpoint from {checkpoint_path}")
-        checkpoint = torch.load(checkpoint_path, map_location=device, weights_only=False)
-        if 'model_state_dict' in checkpoint:
-            model.load_state_dict(checkpoint['model_state_dict'])
-            print(f"Loaded checkpoint from epoch {checkpoint.get('epoch', 'N/A')}")
-        else:
-            model.load_state_dict(checkpoint)
-
-    model = model.to(device)
-    model.eval()
-
-    # Initialize tokenizer
-    tokenizer = ByteTokenizerV62()
-
-    # Count parameters
-    total_params = sum(p.numel() for p in model.parameters())
-    print(f"Model loaded successfully! Total parameters: {total_params/1e6:.1f}M")
-
-    return model
-
-def autoregressive_generate(encoder_outputs, max_length=48):
-    """
-    Autoregressive generation from compressed embeddings
-
-    This is the proper way to generate text from the compressed representation,
-    using the decoder in autoregressive mode with teacher forcing disabled.
-    """
-    # Get all encoder hidden states (decoder needs all 4 layers for cross-attention)
-    if 'all_hidden_states' in encoder_outputs:
-        encoder_all_hidden = encoder_outputs['all_hidden_states']
-    else:
-        compressed = encoder_outputs.get('compressed', encoder_outputs.get('hidden_states'))
-        encoder_all_hidden = [compressed] * 4
-
-    batch_size = encoder_all_hidden[0].shape[0]
-    device = encoder_all_hidden[0].device
-
-    # Start with BOS token
-    generated = torch.full((batch_size, 1), tokenizer.BOS, dtype=torch.long, device=device)
-
-    # Generate tokens autoregressively
-    for _ in range(max_length - 1):
-        with torch.no_grad():
-            gen_mask = torch.ones_like(generated, dtype=torch.bool)
-
-            # Run decoder with current sequence
-            decoder_outputs = model.decoder(
-                encoder_all_hidden=encoder_all_hidden,
-                decoder_input_ids=generated,
-                attention_mask=gen_mask,
-                use_cache=False
+# Add paths
+sys.path.insert(0, 'core')
+
+from unified_model import IntelligentTokenizerV62
+from tokenizer import ByteTokenizerV62
+
+class B2NLTokenizer:
+    def __init__(self):
+        self.model = None
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        self.load_model()
+
+    def load_model(self):
+        """Load model from HuggingFace or local"""
+        try:
+            # Try HuggingFace first
+            from huggingface_hub import hf_hub_download
+            checkpoint_path = hf_hub_download(
+                repo_id="ggunio/B2NL-IntelligentTokenizer-v6.2.1",
+                filename="pytorch_model.bin"
             )
-
-            # Get logits for the last position
-            logits = decoder_outputs['logits'][:, -1, :]
-
-            # Sample next token (greedy decoding for best accuracy)
-            next_token = torch.argmax(logits, dim=-1, keepdim=True)
-
-            # Append to generated sequence
-            generated = torch.cat([generated, next_token], dim=1)
-
-            # Stop if EOS is generated
-            if (next_token == tokenizer.EOS).all():
-                break
-
-    return generated
-
-def process_with_sliding_window(text: str,
-                               chunk_size: int = 46,
-                               overlap: int = 8) -> Dict:
-    """
-    Process long text with sliding window approach
-
-    The model processes 48-byte chunks (46 content + 2 special tokens).
-    For longer texts, we use an 8-byte overlap to maintain context.
-
-    Args:
-        text: Input text
-        chunk_size: Size of each chunk (default 46 bytes)
-        overlap: Overlap between chunks (default 8 bytes)
-
-    Returns:
-        Dictionary with chunks and metadata
-    """
-    text_bytes = text.encode('utf-8')
-    total_bytes = len(text_bytes)
-
-    chunks = []
-    positions = []
-
-    # Handle short text
-    if total_bytes <= chunk_size:
-        chunks.append(text)
-        positions.append((0, total_bytes))
-    else:
-        # Sliding window processing
-        pos = 0
-        while pos < total_bytes:
-            end_pos = min(pos + chunk_size, total_bytes)
-
-            # Extract chunk with proper UTF-8 handling
-            chunk_bytes = text_bytes[pos:end_pos]
-
-            # Ensure valid UTF-8 boundary
-            while end_pos > pos and end_pos < total_bytes:
-                try:
-                    chunk_text = text_bytes[pos:end_pos].decode('utf-8')
+            print(f"Loading from HuggingFace")
+        except:
+            # Try local paths
+            checkpoint_paths = [
+                "pytorch_model.bin",
+                "checkpoints/v62/16.0/epoch_100.pt",
+                "D:/intelligent-tokenizer/intelligent-tokenizer_v6.2.1/checkpoints/v62/16.0/epoch_100.pt"
+            ]
+            checkpoint_path = None
+            for path in checkpoint_paths:
+                if Path(path).exists():
+                    checkpoint_path = path
                     break
-                except UnicodeDecodeError:
-                    end_pos -= 1
-
-            chunk_text = text_bytes[pos:end_pos].decode('utf-8', errors='ignore')
-            chunks.append(chunk_text)
-            positions.append((pos, end_pos))
-
-            # Move window with overlap
-            pos += chunk_size - overlap
-
-            # Avoid tiny final chunk
-            if total_bytes - pos < overlap:
-                break
-
-    return {
-        'chunks': chunks,
-        'positions': positions,
-        'total_bytes': total_bytes,
-        'num_chunks': len(chunks)
-    }
-
-def compress_text(text: str,
-                 show_details: bool = True) -> Tuple[str, Dict]:
-    """
-    Compress text using B2NL-IntelligentTokenizer
-
-    The model achieves a fixed 16:1 compression ratio by encoding
-    each 48-byte chunk into exactly 3 semantic tokens.
-
-    Returns:
-        (status_message, statistics_dict)
-    """
-    if not model:
-        return "❌ Model not loaded! Please load the model first.", {}
-
-    if not text:
-        return "⚠️ Please enter text to compress.", {}
-
-    try:
-        # Process with sliding window
-        window_result = process_with_sliding_window(text)
-        chunks = window_result['chunks']
-        total_bytes = window_result['total_bytes']
-
-        # Compress each chunk
-        all_embeddings = []
-        chunk_details = []
-
-        for i, chunk in enumerate(chunks):
-            with torch.no_grad():
-                # Encode chunk
-                encoded = tokenizer.encode(chunk)
-                if isinstance(encoded, dict):
-                    input_ids = encoded['input_ids'].unsqueeze(0).to(device)
-                    attention_mask = encoded['attention_mask'].unsqueeze(0).to(device)
-                else:
-                    input_ids = encoded.unsqueeze(0).to(device)
-                    attention_mask = torch.ones_like(input_ids).to(device)
 
-                # Get encoder output
-                encoder_output = model.encoder(
-                    input_ids=input_ids,
-                    attention_mask=attention_mask
-                )
+            if not checkpoint_path:
+                print("❌ Model not found")
+                return
 
-                # Extract compressed embeddings
-                compressed = encoder_output.get('compressed')
+        # Load model
+        self.model = IntelligentTokenizerV62()
+        checkpoint = torch.load(checkpoint_path, map_location=self.device, weights_only=False)
 
-                # Get actual token count
-                if 'num_tokens' in encoder_output:
-                    num_tokens = round(encoder_output['num_tokens'])
-                elif compressed is not None:
-                    num_tokens = compressed.shape[1]
-                else:
-                    num_tokens = 3  # Default for 16:1 ratio
-
-                if compressed is not None:
-                    all_embeddings.append(compressed)
-                    chunk_details.append({
-                        'chunk_id': i + 1,
-                        'text': chunk[:30] + '...' if len(chunk) > 30 else chunk,
-                        'bytes': len(chunk.encode('utf-8')),
-                        'tokens': num_tokens
-                    })
-
-        # Calculate statistics
-        total_tokens = sum(detail['tokens'] for detail in chunk_details)
-        compression_ratio = total_bytes / max(1, total_tokens)
-
-        stats = {
-            'total_bytes': total_bytes,
-            'total_tokens': total_tokens,
-            'num_chunks': len(chunks),
-            'compression_ratio': f"{compression_ratio:.1f}:1",
-            'avg_tokens_per_chunk': total_tokens / max(1, len(chunks))
-        }
-
-        # Build detailed message
-        if show_details:
-            details = f"✅ **Compression Complete!**\n\n"
-            details += f"📊 **Input Statistics:**\n"
-            details += f"- Total bytes: {total_bytes}\n"
-            details += f"- Number of chunks: {len(chunks)}\n\n"
-            details += f"🗜️ **Compression Results:**\n"
-            details += f"- Total tokens generated: {total_tokens}\n"
-            details += f"- **Compression ratio: {compression_ratio:.1f}:1**\n"
-            details += f"- Average tokens per chunk: {stats['avg_tokens_per_chunk']:.1f}\n\n"
-
-            if len(chunk_details) <= 5:
-                details += "📝 **Chunk Details:**\n"
-                for detail in chunk_details:
-                    details += f"  • Chunk {detail['chunk_id']}: {detail['bytes']} bytes → {detail['tokens']} tokens\n"
-
-            details += f"\n💡 **Note:** Fixed 16:1 compression means each 48-byte chunk "
-            details += f"is compressed to exactly 3 tokens, preserving semantic meaning."
-
-            return details, stats
+        if 'model_state_dict' in checkpoint:
+            self.model.load_state_dict(checkpoint['model_state_dict'])
         else:
-            return f"Compressed: {total_bytes} bytes → {total_tokens} tokens ({compression_ratio:.1f}:1)", stats
-
-    except Exception as e:
-        return f"❌ Error during compression: {str(e)}", {}
-
-def reconstruct_text(text: str,
-                    temperature: float = 0.1,
-                    top_k: int = 10,
-                    streaming: bool = True) -> str:
-    """
-    Reconstruct text from compressed representation using autoregressive generation
-
-    This function compresses the input text and then reconstructs it using
-    the decoder in autoregressive mode. We use low temperature and Top-K
-    sampling for maximum reconstruction accuracy.
+            self.model.load_state_dict(checkpoint)
 
-    Args:
-        text: Original text to compress and reconstruct
-        temperature: Generation temperature (0.1 = very deterministic)
-        top_k: Number of top tokens to sample from (10 = highly constrained)
-        streaming: Whether to simulate streaming output
+        self.model = self.model.to(self.device)
+        self.model.eval()
+        print(f"✅ Model loaded on {self.device}")
 
-    Returns:
-        Detailed reconstruction results with accuracy metrics
-    """
-    if not model:
-        return "❌ Model not loaded! Please load the model first."
+    def process_text(self, text, temperature=0.1):
+        """Process text and return detailed results"""
+        if not self.model or not text:
+            return "Please enter text", "", ""
 
-    if not text:
-        return "⚠️ Please enter text to reconstruct."
+        try:
+            start_time = time.time()
 
-    try:
-        # Process with sliding window
-        window_result = process_with_sliding_window(text)
-        chunks = window_result['chunks']
+            # Compress (get embedding info)
+            compressed = self.model.compress(text)
+            num_tokens = compressed['num_tokens']
+            text_bytes = len(text.encode('utf-8'))
+            compression_ratio = compressed['compression_ratio']
 
-        reconstructed_chunks = []
-
-        for chunk in chunks:
+            # Reconstruct
             with torch.no_grad():
-                # Encode chunk
-                encoded = tokenizer.encode(chunk)
-                if isinstance(encoded, dict):
-                    input_ids = encoded['input_ids'].unsqueeze(0).to(device)
-                    attention_mask = encoded['attention_mask'].unsqueeze(0).to(device)
-                else:
-                    input_ids = encoded.unsqueeze(0).to(device)
-                    attention_mask = torch.ones_like(input_ids).to(device)
+                reconstructed = self.model.generate(text, temperature=temperature)
 
-                # Get encoder outputs
-                encoder_outputs = model.encoder(
-                    input_ids=input_ids,
-                    attention_mask=attention_mask
-                )
+            elapsed_time = (time.time() - start_time) * 1000
 
-                # Generate using autoregressive decoding
-                generated_ids = autoregressive_generate(encoder_outputs, max_length=48)
+            # Calculate accuracy
+            min_len = min(len(text), len(reconstructed))
+            matches = sum(1 for i in range(min_len) if text[i] == reconstructed[i])
+            accuracy = (matches / len(text)) * 100 if text else 0
 
-                # Decode to text
-                reconstructed = tokenizer.decode(generated_ids[0])
+            # Format results
+            stats = f"""📊 **Compression Statistics**
+• Input: {text_bytes} bytes → {num_tokens} tokens
+• Compression: {compression_ratio:.1f}:1 ({(1/compression_ratio)*100:.1f}% of original)
+• Embeddings generated: {num_tokens}
+• Processing time: {elapsed_time:.1f}ms
+• Reconstruction accuracy: {accuracy:.1f}%"""
 
-                # Trim to original chunk length
-                chunk_len = len(chunk.encode('utf-8'))
-                reconstructed = reconstructed[:chunk_len]
+            details = f"""🔤 **Original Text** ({len(text)} chars, {text_bytes} bytes):
+{text}
 
-                reconstructed_chunks.append(reconstructed)
+🔄 **Reconstructed Text** ({len(reconstructed)} chars):
+{reconstructed}
 
-                if streaming:
-                    time.sleep(0.05)  # Simulate streaming
+✅ **Match Rate**: {accuracy:.1f}% ({matches}/{len(text)} characters)"""
 
-        # Combine chunks (with overlap handling)
-        if len(reconstructed_chunks) == 1:
-            full_reconstruction = reconstructed_chunks[0]
-        else:
-            # First chunk in full
-            full_reconstruction = reconstructed_chunks[0]
-            # Subsequent chunks: skip overlap bytes
-            for i in range(1, len(reconstructed_chunks)):
-                chunk_text = reconstructed_chunks[i]
-                # Skip approximately 8 bytes (overlap) - simplified
-                if len(chunk_text) > 3:
-                    full_reconstruction += chunk_text[3:]
-                else:
-                    full_reconstruction += chunk_text
-
-        # Calculate accuracy using SequenceMatcher
-        similarity = SequenceMatcher(None, text, full_reconstruction[:len(text)]).ratio()
-
-        # Build result message
-        result = f"🔄 **Reconstruction Complete!**\n\n"
-        result += f"📝 **Original Text:**\n{text[:200]}{'...' if len(text) > 200 else ''}\n\n"
-        result += f"🎯 **Reconstructed Text:**\n{full_reconstruction[:200]}{'...' if len(full_reconstruction) > 200 else ''}\n\n"
-        result += f"📊 **Reconstruction Statistics:**\n"
-        result += f"- **Accuracy: {similarity:.1%}**\n"
-        result += f"- Original bytes: {len(text.encode('utf-8'))}\n"
-        result += f"- Reconstructed bytes: {len(full_reconstruction.encode('utf-8'))}\n"
-        result += f"- Chunks processed: {len(chunks)}\n\n"
-
-        result += f"⚙️ **Generation Settings:**\n"
-        result += f"- Temperature: {temperature} (Lower = More precise)\n"
-        result += f"- Top-K: {top_k} (Lower = More deterministic)\n"
-        result += f"- Method: Autoregressive decoding\n\n"
-
-        if similarity >= 0.95:
-            result += "✨ **Excellent reconstruction!** Near-perfect accuracy achieved."
-        elif similarity >= 0.85:
-            result += "✅ **Good reconstruction!** High accuracy with minor differences."
-        elif similarity >= 0.70:
-            result += "⚠️ **Moderate reconstruction.** Some semantic meaning preserved."
-        else:
-            result += "❌ **Poor reconstruction.** Consider retraining or adjusting parameters."
-
-        return result
-
-    except Exception as e:
-        return f"❌ Error during reconstruction: {str(e)}"
-
-def compare_performance(text: str) -> str:
-    """
-    Compare B2NL tokenizer with traditional tokenizers
-
-    Shows how our 16:1 fixed compression compares to BPE and SentencePiece
-    in terms of token efficiency and potential cost savings.
-    """
-    if not text:
-        return "⚠️ Please enter text for comparison."
-
-    try:
-        text_bytes = len(text.encode('utf-8'))
-
-        # Traditional tokenizer estimates (empirical averages)
-        # BPE (GPT-2/3): ~4 bytes per token
-        # SentencePiece: ~4.5 bytes per token
-        # WordPiece (BERT): ~3.5 bytes per token
-        bpe_tokens = text_bytes // 4
-        sentencepiece_tokens = text_bytes // 4.5
-        wordpiece_tokens = text_bytes // 3.5
-
-        # Our compression
-        _, stats = compress_text(text, show_details=False)
-        our_tokens = stats.get('total_tokens', 0)
-
-        # Calculate improvements
-        if our_tokens > 0:
-            vs_bpe = bpe_tokens / our_tokens
-            vs_sp = sentencepiece_tokens / our_tokens
-            vs_wp = wordpiece_tokens / our_tokens
-
-            savings_bpe = (1 - our_tokens/bpe_tokens) * 100
-            savings_sp = (1 - our_tokens/sentencepiece_tokens) * 100
-            savings_wp = (1 - our_tokens/wordpiece_tokens) * 100
-        else:
-            vs_bpe = vs_sp = vs_wp = 0
-            savings_bpe = savings_sp = savings_wp = 0
-
-        comparison = "## 📊 Tokenizer Comparison\n\n"
-
-        # Table format
-        comparison += "| Tokenizer | Tokens | Compression | Savings |\n"
-        comparison += "|-----------|--------|-------------|----------|\n"
-        comparison += f"| BPE (GPT-2/3) | {bpe_tokens} | Baseline | - |\n"
-        comparison += f"| SentencePiece | {int(sentencepiece_tokens)} | {bpe_tokens/max(1,sentencepiece_tokens):.1f}x | {int(savings_sp-savings_bpe)}% |\n"
-        comparison += f"| WordPiece (BERT) | {int(wordpiece_tokens)} | {bpe_tokens/max(1,wordpiece_tokens):.1f}x | {int(savings_wp-savings_bpe)}% |\n"
-        comparison += f"| **B2NL v6.2.1** | **{our_tokens}** | **{vs_bpe:.1f}x** | **{int(savings_bpe)}%** |\n\n"
-
-        # Summary
-        comparison += f"### 🚀 Key Achievements:\n"
-        comparison += f"- **{vs_bpe:.1f}x** more efficient than BPE tokenization\n"
-        comparison += f"- **{int(savings_bpe)}%** reduction in token count\n"
-        comparison += f"- Fixed 16:1 compression ratio (predictable costs)\n"
-        comparison += f"- Semantic preservation across 204 languages\n\n"
-
-        # Cost implications
-        comparison += f"### 💰 Cost Implications:\n"
-        comparison += f"For LLM APIs charging per token:\n"
-        comparison += f"- Traditional: ${bpe_tokens * 0.002:.2f} (at $0.002/1K tokens)\n"
-        comparison += f"- B2NL: ${our_tokens * 0.002:.2f}\n"
-        comparison += f"- **Savings: ${(bpe_tokens - our_tokens) * 0.002:.2f} ({int(savings_bpe)}%)**\n\n"
-
-        comparison += "📌 **Note:** B2NL serves as a preprocessing layer, converting text to "
-        comparison += "compressed embeddings before feeding to inference models."
-
-        return comparison
-
-    except Exception as e:
-        return f"❌ Error during comparison: {str(e)}"
-
-# Create Gradio interface
-def create_demo():
-    """Create the interactive Gradio demo interface"""
-
-    with gr.Blocks(title="B2NL-IntelligentTokenizer v6.2.1", theme=gr.themes.Soft()) as demo:
-        gr.Markdown("""
-        # 🚀 B2NL-IntelligentTokenizer v6.2.1
-        ### Progressive Byte-to-Natural Language Tokenizer with 16:1 Fixed Compression
+            return stats, details, reconstructed
 
-        ---
+        except Exception as e:
+            return f"Error: {str(e)}", "", ""
 
-        **🎯 Purpose:** This model serves as an **embedding preprocessing layer** for inter-modal
-        communication, converting raw text into compressed semantic representations that can be
-        efficiently processed by downstream AI models.
+# Initialize
+tokenizer = B2NLTokenizer()
 
-        **🌐 Training:** Trained on the FLORES-200 dataset covering 204 languages with 100 epochs
-        of progressive splitting optimization.
+# Gradio Interface
+with gr.Blocks(title="B2NL v6.2.1", theme=gr.themes.Soft()) as app:
+    gr.Markdown("""
+    # 🚀 B2NL-IntelligentTokenizer v6.2.1
 
-        **⚡ Innovation:** Achieves fixed 16:1 compression ratio (3 tokens per 48-byte chunk) while
-        maintaining semantic integrity through Gumbel-Softmax differentiable token selection.
-        """)
+    **Fixed 16:1 compression** | **204 languages** | **Autoregressive mode** (~500ms)
+    """)
 
+    with gr.Tab("🔄 Reconstruction Test"):
         with gr.Row():
-            with gr.Column(scale=1):
-                gr.Markdown("""
-                ### 📊 Model Specifications
-                - **Architecture:** 4L Encoder + 6L Decoder
-                - **Parameters:** 230.3M
-                - **Compression:** 16:1 fixed ratio
-                - **Chunk Size:** 48 bytes (46 + BOS/EOS)
-                - **Output:** 3 tokens per chunk
-                - **Languages:** 204 (FLORES-200)
-                """)
-            with gr.Column(scale=1):
-                gr.Markdown("""
-                ### 🎯 Key Features
-                - ✅ Fixed compression ratio (predictable)
-                - ✅ Sliding window for long texts
-                - ✅ Autoregressive reconstruction
-                - ✅ Multi-language semantic preservation
-                - ✅ Streaming processing support
-                - ✅ 80%+ reconstruction accuracy
-                """)
-
-        # Load model section
-        with gr.Row():
-            checkpoint_path = gr.Textbox(
-                label="📁 Checkpoint Path",
-                placeholder="Path to epoch_100.pt checkpoint...",
-                value="D:/intelligent-tokenizer/intelligent-tokenizer_v6.2.1/checkpoints/v62/16.0/epoch_100.pt"
-            )
-            load_btn = gr.Button("🔧 Load Model", variant="primary", scale=0)
-            status = gr.Textbox(label="Status", value="⏳ Model not loaded", scale=0)
-
-        # Main tabs
-        with gr.Tabs():
-            with gr.TabItem("🗜️ Compression Analysis"):
-                gr.Markdown("### Analyze text compression with detailed statistics")
-                with gr.Row():
-                    with gr.Column():
-                        input_text = gr.Textbox(
-                            label="Input Text",
-                            placeholder="Enter any text in any of 204 supported languages...",
-                            lines=10
-                        )
-                        compress_btn = gr.Button("🗜️ Compress", variant="primary")
-
-                    with gr.Column():
-                        compression_output = gr.Textbox(
-                            label="Compression Results",
-                            lines=10
-                        )
-                        compression_stats = gr.JSON(label="Detailed Statistics")
-
-            with gr.TabItem("🔄 Reconstruction Test"):
-                gr.Markdown("### Test compression and reconstruction accuracy")
-                with gr.Row():
-                    with gr.Column():
-                        recon_input = gr.Textbox(
-                            label="Text to Reconstruct",
-                            placeholder="Enter text to compress and reconstruct...",
-                            lines=8
-                        )
-                        with gr.Row():
-                            temperature = gr.Slider(
-                                minimum=0.01, maximum=1.0, value=0.1, step=0.01,
-                                label="Temperature (0.1 = Precise)"
-                            )
-                            top_k = gr.Slider(
-                                minimum=1, maximum=50, value=10, step=1,
-                                label="Top-K (10 = Deterministic)"
-                            )
-                        reconstruct_btn = gr.Button("🔄 Reconstruct", variant="primary")
-
-                    with gr.Column():
-                        reconstruction_output = gr.Textbox(
-                            label="Reconstruction Results",
-                            lines=15
-                        )
-
-            with gr.TabItem("📊 Tokenizer Comparison"):
-                gr.Markdown("### Compare with traditional tokenizers (BPE, SentencePiece)")
-                with gr.Row():
-                    with gr.Column():
-                        compare_input = gr.Textbox(
-                            label="Text for Comparison",
-                            placeholder="Enter text to compare tokenization efficiency...",
-                            lines=8
-                        )
-                        compare_btn = gr.Button("📊 Compare", variant="primary")
-
-                    with gr.Column():
-                        comparison_output = gr.Markdown()
-
-            with gr.TabItem("📝 Example Tests"):
-                gr.Markdown("### Pre-configured test examples in various languages")
+            with gr.Column():
+                input_text = gr.Textbox(
+                    label="Input Text",
+                    placeholder="Enter any text in any of 204 languages...",
+                    lines=5
+                )
+
                 gr.Examples(
                     examples=[
-                        ["The quick brown fox jumps over the lazy dog."],
-                        ["안녕하세요. 오늘 날씨가 정말 좋네요!"],
-                        ["今天天气很好，适合出去散步。"],
-                        ["Bonjour le monde! Comment allez-vous aujourd'hui?"],
-                        ["مرحبا بالعالم! كيف حالك اليوم؟"],
-                        ["こんにちは世界！今日はいい天気ですね。"],
-                        ["Привет мир! Как дела сегодня?"],
-                        ["Multi-language: Hello 안녕하세요 你好 こんにちは"]
+                        # Major languages
+                        "Hello, world! How are you today?",
+                        "안녕하세요, 반갑습니다. 오늘 날씨가 좋네요.",
+                        "你好世界！今天天气很好。",
+                        "こんにちは世界！今日はいい天気ですね。",
+                        "Bonjour le monde! Comment allez-vous?",
+                        "Hola mundo! ¿Cómo estás hoy?",
+                        "Привет мир! Как дела?",
+                        "مرحبا بالعالم! كيف حالك اليوم؟",
+                        "Olá mundo! Como você está?",
+                        "Hallo Welt! Wie geht es dir?",
+                        # More diverse languages
+                        "नमस्ते दुनिया! आप कैसे हैं?",  # Hindi
+                        "হ্যালো বিশ্ব! আপনি কেমন আছেন?",  # Bengali
+                        "สวัสดีชาวโลก! คุณเป็นอย่างไรบ้าง?",  # Thai
+                        "Xin chào thế giới! Bạn khỏe không?",  # Vietnamese
+                        "Kamusta mundo! Kumusta ka?",  # Filipino
+                        "Jambo dunia! Habari yako?",  # Swahili
+                        "Γεια σου κόσμε! Πώς είσαι;",  # Greek
+                        "שלום עולם! מה שלומך?",  # Hebrew
+                        "Selam dünya! Nasılsın?",  # Turkish
+                        "Salam dünya! Necəsən?",  # Azerbaijani
                     ],
-                    inputs=[input_text]
+                    inputs=input_text,
+                    label="Example texts (204 languages supported)"
                 )
 
-            with gr.TabItem("📚 Documentation"):
-                gr.Markdown("""
-                ### Technical Details
-
-                **Model Architecture:**
-                - **Encoder:** 4-layer transformer with progressive splitting mechanism
-                - **Decoder:** 6-layer transformer with multi-level cross-attention
-                - **Token Selection:** Gumbel-Softmax with temperature annealing
-                - **Attention:** Multi-Query Attention (MQA) with 8x KV cache reduction
-
-                **Training Details:**
-                - **Dataset:** FLORES-200 (204 languages)
-                - **Epochs:** 100
-                - **Batch Size:** 128
-                - **Learning Rate:** 3e-5 with cosine annealing
-                - **Loss:** Weighted combination of reconstruction, compression, and boundary losses
-
-                **Compression Mechanism:**
-                - Input text is split into 48-byte chunks (46 content + 2 special tokens)
-                - Each chunk is compressed to exactly 3 semantic tokens
-                - Achieves fixed 16:1 compression ratio
-                - Uses sliding window with 8-byte overlap for long texts
-
-                **Use Cases:**
-                1. **LLM Cost Reduction:** Reduce token counts by ~75%
-                2. **Cross-modal Communication:** Universal embedding layer
-                3. **Multilingual Processing:** Unified representation for 204 languages
-                4. **Bandwidth Optimization:** Compress text for transmission
-
-                **Limitations:**
-                - Mixed language text may have lower reconstruction accuracy
-                - Optimized for semantic preservation, not exact character matching
-                - Requires GPU for optimal performance
-
-                **Citation:**
-                ```
-                @model{b2nl2024,
-                  title={B2NL-IntelligentTokenizer: Progressive Byte-to-Natural Language Tokenization},
-                  author={ggunio},
-                  year={2024},
-                  version={6.2.1},
-                  url={https://huggingface.co/ggunio/B2NL-IntelligentTokenizer}
-                }
-                ```
-                """)
-
-        # Event handlers
-        def load_model_handler(path):
-            try:
-                if not path:
-                    return "⚠️ Please provide a checkpoint path"
-                load_model(path)
-                return "✅ Model loaded successfully! Ready for inference."
-            except Exception as e:
-                return f"❌ Error loading model: {str(e)}"
-
-        load_btn.click(
-            load_model_handler,
-            inputs=[checkpoint_path],
-            outputs=[status]
-        )
+                temperature = gr.Slider(
+                    minimum=0.1,
+                    maximum=1.0,
+                    value=0.1,
+                    step=0.1,
+                    label="Temperature (0.1 = Most accurate)"
+                )
 
-        compress_btn.click(
-            compress_text,
-            inputs=[input_text],
-            outputs=[compression_output, compression_stats]
-        )
+                process_btn = gr.Button("🔄 Compress & Reconstruct", variant="primary", size="lg")
 
-        reconstruct_btn.click(
-            reconstruct_text,
-            inputs=[recon_input, temperature, top_k],
-            outputs=[reconstruction_output]
-        )
+            with gr.Column():
+                stats_output = gr.Markdown(label="Statistics")
+                details_output = gr.Markdown(label="Details")
 
-        compare_btn.click(
-            compare_performance,
-            inputs=[compare_input],
-            outputs=[comparison_output]
+    with gr.Tab("📊 Batch Test"):
+        gr.Markdown("""
+        Test multiple texts at once to compare compression rates across languages.
+        """)
+
+        batch_input = gr.Textbox(
+            label="Enter multiple texts (one per line)",
+            placeholder="Enter texts in different languages...\nOne text per line",
+            lines=10,
+            value="""Hello, world!
+안녕하세요, 반갑습니다.
+你好世界！
+こんにちは世界！
+Bonjour le monde!"""
         )
 
-        # Auto-load model on startup
-        demo.load(
-            lambda: "⏳ Ready to load model. Click 'Load Model' to begin.",
-            outputs=[status]
+        batch_btn = gr.Button("🔄 Process Batch", variant="primary")
+        batch_output = gr.Dataframe(
+            headers=["Text", "Language", "Bytes", "Tokens", "Compression", "Accuracy"],
+            label="Batch Results"
         )
 
-    return demo
+    # Connect functions
+    process_btn.click(
+        fn=lambda text, temp: tokenizer.process_text(text, temp),
+        inputs=[input_text, temperature],
+        outputs=[stats_output, details_output]
+    )
+
+    def process_batch(texts):
+        if not texts:
+            return []
+
+        results = []
+        for text in texts.strip().split('\n'):
+            if not text.strip():
+                continue
+
+            stats, details, reconstructed = tokenizer.process_text(text.strip(), 0.1)
+
+            # Parse stats for table
+            if "Error" not in stats:
+                # Detect language (simple heuristic)
+                if any(ord(c) >= 0x3040 and ord(c) <= 0x309F for c in text):
+                    lang = "Japanese"
+                elif any(ord(c) >= 0xAC00 and ord(c) <= 0xD7AF for c in text):
+                    lang = "Korean"
+                elif any(ord(c) >= 0x4E00 and ord(c) <= 0x9FFF for c in text):
+                    lang = "Chinese"
+                elif any(ord(c) >= 0x0600 and ord(c) <= 0x06FF for c in text):
+                    lang = "Arabic"
+                elif any(ord(c) >= 0x0400 and ord(c) <= 0x04FF for c in text):
+                    lang = "Russian"
+                else:
+                    lang = "English/Latin"
+
+                text_bytes = len(text.encode('utf-8'))
+                compressed = tokenizer.model.compress(text)
+                num_tokens = compressed['num_tokens']
+                compression_ratio = compressed['compression_ratio']
+
+                # Calculate accuracy
+                min_len = min(len(text), len(reconstructed))
+                matches = sum(1 for i in range(min_len) if text[i] == reconstructed[i])
+                accuracy = (matches / len(text)) * 100 if text else 0
+
+                results.append([
+                    text[:50] + "..." if len(text) > 50 else text,
+                    lang,
+                    text_bytes,
+                    num_tokens,
+                    f"{compression_ratio:.1f}:1",
+                    f"{accuracy:.1f}%"
+                ])
+
+        return results
+
+    batch_btn.click(
+        fn=process_batch,
+        inputs=batch_input,
+        outputs=batch_output
+    )
+
+    gr.Markdown("""
+    ---
+    **Note**: This model uses autoregressive generation (teacher forcing training).
+    Non-autoregressive training planned for November 2025 will provide 10x speedup.
+
+    Model: [ggunio/B2NL-IntelligentTokenizer-v6.2.1](https://huggingface.co/ggunio/B2NL-IntelligentTokenizer-v6.2.1)
+    """)
 
 if __name__ == "__main__":
-    # Create and launch demo
-    demo = create_demo()
-
-    print("="*60)
-    print("B2NL-IntelligentTokenizer v6.2.1 - Gradio Demo")
-    print("="*60)
-    print("Launching interactive demo...")
-    print("Share link will be generated for public access")
-    print("="*60)
-
-    demo.launch(
-        server_name="0.0.0.0",
-        server_port=7860,
-        share=True,  # Create public link
-        debug=False  # Set to True for debugging
-    )
+    app.launch()