Update app.py

app.py

@@ -1,10 +1,414 @@
-# Alternative approach using collapsible sections instead of tabs
-# Replace the create_entity_table_gradio_tabs function and the output display section
+import gradio as gr
+import torch
+from gliner import GLiNER
+import pandas as pd
+import warnings
+import random
+import re
+import time
+warnings.filterwarnings('ignore')
 
-def create_entity_results_accordion(entities, entity_colors):
-    """Create collapsible accordion-style results instead of tabs"""
+# Common NER entity types
+STANDARD_ENTITIES = [
+    'DATE', 'EVENT', 'FAC', 'GPE', 'LANG', 'LOC',
+    'MISC', 'NORP', 'ORG', 'PER', 'PRODUCT', 'Work of Art'
+]
+
+# Colour schemes
+STANDARD_COLORS = {
+    'DATE': '#FF6B6B',      # Red
+    'EVENT': '#4ECDC4',     # Teal
+    'FAC': '#45B7D1',       # Blue
+    'GPE': '#F9CA24',       # Yellow
+    'LANG': '#6C5CE7',      # Purple
+    'LOC': '#A0E7E5',       # Light Cyan
+    'MISC': '#FD79A8',      # Pink
+    'NORP': '#8E8E93',      # Grey
+    'ORG': '#55A3FF',       # Light Blue
+    'PER': '#00B894',       # Green
+    'PRODUCT': '#E17055',   # Orange-Red
+    'WORK OF ART': '#DDA0DD'  # Plum
+}
+
+# Additional colours for custom entities
+CUSTOM_COLOR_PALETTE = [
+    '#FF9F43', '#10AC84', '#EE5A24', '#0FBC89', '#5F27CD',
+    '#FF3838', '#2F3640', '#3742FA', '#2ED573', '#FFA502',
+    '#FF6348', '#1E90FF', '#FF1493', '#32CD32', '#FFD700',
+    '#FF4500', '#DA70D6', '#00CED1', '#FF69B4', '#7B68EE'
+]
+
+class HybridNERManager:
+    def __init__(self):
+        self.gliner_model = None
+        self.spacy_model = None
+        self.flair_models = {}
+        self.all_entity_colors = {}
+        self.model_names = [
+            'entities_flair_ner-large',
+            'entities_spacy_en_core_web_trf', 
+            'entities_flair_ner-ontonotes-large',
+            'entities_gliner_knowledgator/modern-gliner-bi-large-v1.0'
+        ]
+
+    def load_model(self, model_name):
+        """Load the specified model"""
+        try:
+            if 'spacy' in model_name:
+                return self.load_spacy_model()
+            elif 'flair' in model_name:
+                return self.load_flair_model(model_name)
+            elif 'gliner' in model_name:
+                return self.load_gliner_model()
+        except Exception as e:
+            print(f"Error loading {model_name}: {str(e)}")
+            return None
+
+    def load_spacy_model(self):
+        """Load spaCy model for common NER"""
+        if self.spacy_model is None:
+            try:
+                import spacy
+                try:
+                    # Try transformer model first, fallback to small model
+                    self.spacy_model = spacy.load("en_core_web_trf")
+                    print("✓ spaCy transformer model loaded successfully")
+                except OSError:
+                    try:
+                        self.spacy_model = spacy.load("en_core_web_sm")
+                        print("✓ spaCy common model loaded successfully") 
+                    except OSError:
+                        print("spaCy model not found. Using GLiNER for all entity types.")
+                        return None
+            except Exception as e:
+                print(f"Error loading spaCy model: {str(e)}")
+                return None
+        return self.spacy_model
+
+    def load_flair_model(self, model_name):
+        """Load Flair models"""
+        if model_name not in self.flair_models:
+            try:
+                from flair.models import SequenceTagger
+                if 'ontonotes' in model_name:
+                    model = SequenceTagger.load("flair/ner-english-ontonotes-large")
+                    print("✓ Flair OntoNotes model loaded successfully")
+                else:
+                    model = SequenceTagger.load("flair/ner-english-large") 
+                    print("✓ Flair large model loaded successfully")
+                self.flair_models[model_name] = model
+            except Exception as e:
+                print(f"Error loading {model_name}: {str(e)}")
+                # Fallback to GLiNER
+                return self.load_gliner_model()
+        return self.flair_models[model_name]
+
+    def load_gliner_model(self):
+        """Load GLiNER model for custom entities"""
+        if self.gliner_model is None:
+            try:
+                # Try the modern GLiNER model first, fallback to stable model
+                self.gliner_model = GLiNER.from_pretrained("knowledgator/gliner-bi-large-v1.0")
+                print("✓ GLiNER knowledgator model loaded successfully")
+            except Exception as e:
+                print(f"Primary GLiNER model failed: {str(e)}")
+                try:
+                    # Fallback to stable model
+                    self.gliner_model = GLiNER.from_pretrained("urchade/gliner_medium-v2.1")
+                    print("✓ GLiNER fallback model loaded successfully")
+                except Exception as e2:
+                    print(f"Error loading GLiNER model: {str(e2)}")
+                    return None
+        return self.gliner_model
+
+    def assign_colours(self, standard_entities, custom_entities):
+        """Assign colours to all entity types"""
+        self.all_entity_colors = {}
+
+        # Assign common colours
+        for entity in standard_entities:
+            # Handle the special case of "Work of Art"
+            colour_key = "WORK OF ART" if entity == "Work of Art" else entity.upper()
+            self.all_entity_colors[entity.upper()] = STANDARD_COLORS.get(colour_key, '#CCCCCC')
+
+        # Assign custom colours
+        for i, entity in enumerate(custom_entities):
+            if i < len(CUSTOM_COLOR_PALETTE):
+                self.all_entity_colors[entity.upper()] = CUSTOM_COLOR_PALETTE[i]
+            else:
+                # Generate random colour if we run out
+                self.all_entity_colors[entity.upper()] = f"#{random.randint(0, 0xFFFFFF):06x}"
+
+        return self.all_entity_colors
+
+    def extract_entities_by_model(self, text, entity_types, model_name, threshold=0.3):
+        """Extract entities using the specified model"""
+        if 'spacy' in model_name:
+            return self.extract_spacy_entities(text, entity_types)
+        elif 'flair' in model_name:
+            return self.extract_flair_entities(text, entity_types, model_name)
+        elif 'gliner' in model_name:
+            return self.extract_gliner_entities(text, entity_types, threshold, is_custom=False)
+        else:
+            return []
+
+    def extract_spacy_entities(self, text, entity_types):
+        """Extract entities using spaCy"""
+        model = self.load_spacy_model()
+        if model is None:
+            return []
+
+        try:
+            doc = model(text)
+            entities = []
+            for ent in doc.ents:
+                if ent.label_ in entity_types:
+                    entities.append({
+                        'text': ent.text,
+                        'label': ent.label_,
+                        'start': ent.start_char,
+                        'end': ent.end_char,
+                        'confidence': 1.0,  # spaCy doesn't provide confidence scores
+                        'source': 'spaCy'
+                    })
+            return entities
+        except Exception as e:
+            print(f"Error with spaCy extraction: {str(e)}")
+            return []
+
+    def extract_flair_entities(self, text, entity_types, model_name):
+        """Extract entities using Flair"""
+        model = self.load_flair_model(model_name)
+        if model is None:
+            return []
+
+        try:
+            from flair.data import Sentence
+            sentence = Sentence(text)
+            model.predict(sentence)
+            entities = []
+            for entity in sentence.get_spans('ner'):
+                # Map Flair labels to our common set
+                label = entity.tag
+                if label == 'PERSON':
+                    label = 'PER'
+                elif label == 'ORGANIZATION':
+                    label = 'ORG'
+                elif label == 'LOCATION':
+                    label = 'LOC'
+                elif label == 'MISCELLANEOUS':
+                    label = 'MISC'
+
+                if label in entity_types:
+                    entities.append({
+                        'text': entity.text,
+                        'label': label,
+                        'start': entity.start_position,
+                        'end': entity.end_position,
+                        'confidence': entity.score,
+                        'source': f'Flair-{model_name.split("-")[-1]}'
+                    })
+            return entities
+        except Exception as e:
+            print(f"Error with Flair extraction: {str(e)}")
+            return []
+
+    def extract_gliner_entities(self, text, entity_types, threshold=0.3, is_custom=True):
+        """Extract entities using GLiNER"""
+        model = self.load_gliner_model()
+        if model is None:
+            return []
+
+        try:
+            entities = model.predict_entities(text, entity_types, threshold=threshold)
+            result = []
+            for entity in entities:
+                result.append({
+                    'text': entity['text'],
+                    'label': entity['label'].upper(),
+                    'start': entity['start'],
+                    'end': entity['end'],
+                    'confidence': entity.get('score', 0.0),
+                    'source': 'GLiNER-Custom' if is_custom else 'GLiNER-Common'
+                })
+            return result
+        except Exception as e:
+            print(f"Error with GLiNER extraction: {str(e)}")
+            return []
+
+def find_overlapping_entities(entities):
+    """Find and share overlapping entities - specifically entities found by BOTH common NER models AND custom entities"""
     if not entities:
-        return "<p>No entities found.</p>"
+        return []
+
+    # Sort entities by start position
+    sorted_entities = sorted(entities, key=lambda x: x['start'])
+    shared_entities = []
+
+    i = 0
+    while i < len(sorted_entities):
+        current_entity = sorted_entities[i]
+        overlapping_entities = [current_entity]
+
+        # Find all entities that overlap with current entity
+        j = i + 1
+        while j < len(sorted_entities):
+            next_entity = sorted_entities[j]
+
+            # Check if entities overlap (same text span or overlapping positions)
+            if (current_entity['start'] <= next_entity['start'] < current_entity['end'] or
+                next_entity['start'] <= current_entity['start'] < current_entity['end'] or
+                current_entity['text'].lower() == next_entity['text'].lower()):
+                overlapping_entities.append(next_entity)
+                sorted_entities.pop(j)
+            else:
+                j += 1
+
+        # Create shared entity only if we have BOTH common and custom entities
+        if len(overlapping_entities) == 1:
+            shared_entities.append(overlapping_entities[0])
+        else:
+            # Check if this is a true "shared" entity (common + custom)
+            has_common = False
+            has_custom = False
+            
+            for entity in overlapping_entities:
+                source = entity.get('source', '')
+                if source in ['spaCy', 'GLiNER-Common'] or source.startswith('Flair-'):
+                    has_common = True
+                elif source == 'GLiNER-Custom':
+                    has_custom = True
+            
+            if has_common and has_custom:
+                # This is a true shared entity (common + custom)
+                shared_entity = share_entities(overlapping_entities)
+                shared_entities.append(shared_entity)
+            else:
+                # These are just overlapping entities from the same source type, keep separate
+                shared_entities.extend(overlapping_entities)
+
+        i += 1
+
+    return shared_entities
+
+def share_entities(entity_list):
+    """Share multiple overlapping entities into one"""
+    if len(entity_list) == 1:
+        return entity_list[0]
+
+    # Use the entity with the longest text span as the base
+    base_entity = max(entity_list, key=lambda x: len(x['text']))
+
+    # Collect all labels and sources
+    labels = [entity['label'] for entity in entity_list]
+    sources = [entity['source'] for entity in entity_list]
+    confidences = [entity['confidence'] for entity in entity_list]
+
+    return {
+        'text': base_entity['text'],
+        'start': base_entity['start'],
+        'end': base_entity['end'],
+        'labels': labels,
+        'sources': sources,
+        'confidences': confidences,
+        'is_shared': True,
+        'entity_count': len(entity_list)
+    }
+
+def create_highlighted_html(text, entities, entity_colors):
+    """Create HTML with highlighted entities"""
+    if not entities:
+        return f"<div style='padding: 15px; border: 1px solid #ddd; border-radius: 5px; background-color: #fafafa;'><p>{text}</p></div>"
+
+    # Find and share overlapping entities
+    shared_entities = find_overlapping_entities(entities)
+
+    # Sort by start position
+    sorted_entities = sorted(shared_entities, key=lambda x: x['start'])
+
+    # Create HTML with highlighting
+    html_parts = []
+    last_end = 0
+
+    for entity in sorted_entities:
+        # Add text before entity
+        html_parts.append(text[last_end:entity['start']])
+
+        if entity.get('is_shared', False):
+            # Handle shared entity with multiple colours
+            html_parts.append(create_shared_entity_html(entity, entity_colors))
+        else:
+            # Handle single entity
+            html_parts.append(create_single_entity_html(entity, entity_colors))
+
+        last_end = entity['end']
+
+    # Add remaining text
+    html_parts.append(text[last_end:])
+
+    highlighted_text = ''.join(html_parts)
+
+    return f"""
+    <div style='padding: 15px; border: 2px solid #ddd; border-radius: 8px; background-color: #fafafa; margin: 10px 0;'>
+        <h4 style='margin: 0 0 15px 0; color: #333;'>📝 Text with Highlighted Entities</h4>
+        <div style='line-height: 1.8; font-size: 16px; background-color: white; padding: 15px; border-radius: 5px;'>{highlighted_text}</div>
+    </div>
+    """
+
+def create_single_entity_html(entity, entity_colors):
+    """Create HTML for a single entity"""
+    label = entity['label']
+    colour = entity_colors.get(label.upper(), '#CCCCCC')
+    confidence = entity.get('confidence', 0.0)
+    source = entity.get('source', 'Unknown')
+
+    return (f'<span style="background-color: {colour}; padding: 2px 4px; '
+            f'border-radius: 3px; margin: 0 1px; '
+            f'border: 1px solid {colour}; color: white; font-weight: bold;" '
+            f'title="{label} ({source}) - confidence: {confidence:.2f}">'
+            f'{entity["text"]}</span>')
+
+def create_shared_entity_html(entity, entity_colors):
+    """Create HTML for a shared entity with multiple colours"""
+    labels = entity['labels']
+    sources = entity['sources']
+    confidences = entity['confidences']
+
+    # Get colours for each label
+    colours = []
+    for label in labels:
+        colour = entity_colors.get(label.upper(), '#CCCCCC')
+        colours.append(colour)
+
+    # Create gradient background
+    if len(colours) == 2:
+        gradient = f"linear-gradient(to right, {colours[0]} 50%, {colours[1]} 50%)"
+    else:
+        # For more colours, create equal segments
+        segment_size = 100 / len(colours)
+        gradient_parts = []
+        for i, colour in enumerate(colours):
+            start = i * segment_size
+            end = (i + 1) * segment_size
+            gradient_parts.append(f"{colour} {start}%, {colour} {end}%")
+        gradient = f"linear-gradient(to right, {', '.join(gradient_parts)})"
+
+    # Create tooltip
+    tooltip_parts = []
+    for i, label in enumerate(labels):
+        tooltip_parts.append(f"{label} ({sources[i]}) - {confidences[i]:.2f}")
+    tooltip = " | ".join(tooltip_parts)
+
+    return (f'<span style="background: {gradient}; padding: 2px 4px; '
+            f'border-radius: 3px; margin: 0 1px; '
+            f'border: 2px solid #333; color: white; font-weight: bold;" '
+            f'title="SHARED: {tooltip}">'
+            f'{entity["text"]} 🤝</span>')
+
+def create_entity_table_gradio_tabs(entities, entity_colors):
+    """Create Gradio tabs for entity results"""
+    if not entities:
+        return "No entities found."
 
     # Share overlapping entities
     shared_entities = find_overlapping_entities(entities)
@@ -22,133 +426,485 @@ def create_entity_results_accordion(entities, entity_colors):
         entity_groups[key].append(entity)
 
     if not entity_groups:
-        return "<p>No entities found.</p>"
-
-    # Create accordion HTML
-    accordion_html = """
-    <style>
-        .accordion-item {
-            border: 1px solid #ddd;
-            border-radius: 8px;
-            margin-bottom: 10px;
-            overflow: hidden;
-        }
-        .accordion-header {
-            background-color: #f8f9fa;
-            padding: 15px;
-            cursor: pointer;
-            display: flex;
-            justify-content: space-between;
-            align-items: center;
-            transition: background-color 0.3s;
-        }
-        .accordion-header:hover {
-            background-color: #e9ecef;
-        }
-        .accordion-header.active {
-            background-color: #4ECDC4;
-            color: white;
-        }
-        .accordion-content {
-            padding: 0 15px;
-            max-height: 0;
-            overflow: hidden;
-            transition: max-height 0.3s ease-out, padding 0.3s ease-out;
-        }
-        .accordion-content.show {
-            padding: 15px;
-            max-height: 2000px;
-        }
-        .entity-badge {
-            background-color: #007bff;
-            color: white;
-            padding: 4px 12px;
-            border-radius: 15px;
-            font-size: 14px;
-            font-weight: bold;
-        }
-        .confidence-high { color: #28a745; }
-        .confidence-medium { color: #ffc107; }
-        .confidence-low { color: #dc3545; }
-    </style>
-    
-    <div class="accordion-container">
-    """
-    
-    # Add shared entities section if any
-    if 'SHARED_ENTITIES' in entity_groups:
-        shared_entities_list = entity_groups['SHARED_ENTITIES']
-        accordion_html += f"""
-        <div class="accordion-item">
-            <div class="accordion-header" onclick="toggleAccordion(this)">
-                <div>
-                    <span style="font-size: 20px; margin-right: 10px;">🤝</span>
-                    <strong>Shared Entities</strong>
-                    <span class="entity-badge" style="margin-left: 10px;">{len(shared_entities_list)} found</span>
-                </div>
-                <span>▼</span>
-            </div>
-            <div class="accordion-content">
-                {create_entity_table_html(shared_entities_list, 'SHARED_ENTITIES', '#666666', is_shared=True)}
-            </div>
-        </div>
-        """
+        return "No entities found."
+
+    # Create content for each tab
+    tab_contents = {}
     
-    # Add other entity types
     for entity_type, entities_of_type in entity_groups.items():
         if entity_type == 'SHARED_ENTITIES':
-            continue
+            colour = '#666666'
+            header = f"🤝 Shared Entities ({len(entities_of_type)} found)"
             
-        colour = entity_colors.get(entity_type.upper(), '#f0f0f0')
-        is_standard = entity_type in STANDARD_ENTITIES
-        icon = "🎯" if is_standard else "✨"
-        
-        accordion_html += f"""
-        <div class="accordion-item">
-            <div class="accordion-header" onclick="toggleAccordion(this)">
-                <div>
-                    <span style="font-size: 20px; margin-right: 10px;">{icon}</span>
-                    <strong>{entity_type}</strong>
-                    <span class="entity-badge" style="margin-left: 10px; background-color: {colour};">{len(entities_of_type)} found</span>
-                </div>
-                <span>▼</span>
-            </div>
-            <div class="accordion-content">
-                {create_entity_table_html(entities_of_type, entity_type, colour)}
-            </div>
-        </div>
-        """
+            # Create table for shared entities
+            table_html = f"""
+            <div style="margin: 15px 0;">
+                <h4 style="color: {colour}; margin-bottom: 15px;">{header}</h4>
+                <table style="width: 100%; border-collapse: collapse; border: 1px solid #ddd;">
+                    <thead>
+                        <tr style="background-color: {colour}; color: white;">
+                            <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Entity Text</th>
+                            <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">All Labels</th>
+                            <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Sources</th>
+                            <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Count</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+            """
+            
+            for entity in entities_of_type:
+                labels_text = " | ".join(entity['labels'])
+                sources_text = " | ".join(entity['sources'])
+                
+                table_html += f"""
+                        <tr style="background-color: #fff;">
+                            <td style="padding: 10px; border: 1px solid #ddd; font-weight: bold;">{entity['text']}</td>
+                            <td style="padding: 10px; border: 1px solid #ddd;">{labels_text}</td>
+                            <td style="padding: 10px; border: 1px solid #ddd;">{sources_text}</td>
+                            <td style="padding: 10px; border: 1px solid #ddd; text-align: center;">
+                                <span style='background-color: #28a745; color: white; padding: 2px 6px; border-radius: 10px; font-size: 11px;'>
+                                    {entity['entity_count']}
+                                </span>
+                            </td>
+                        </tr>
+                """
+            
+            table_html += "</tbody></table></div>"
+            tab_contents[f"🤝 SHARED ({len(entities_of_type)})"] = table_html
+            
+        else:
+            colour = entity_colors.get(entity_type.upper(), '#f0f0f0')
+            # Determine if it's common or custom
+            is_standard = entity_type in STANDARD_ENTITIES
+            icon = "🎯" if is_standard else "✨"
+            source_text = "Common NER" if is_standard else "Custom GLiNER"
+            header = f"{icon} {source_text} - {entity_type} ({len(entities_of_type)} found)"
+            
+            # Create table for this entity type
+            table_html = f"""
+            <div style="margin: 15px 0;">
+                <h4 style="color: {colour}; margin-bottom: 15px;">{header}</h4>
+                <table style="width: 100%; border-collapse: collapse; border: 1px solid #ddd;">
+                    <thead>
+                        <tr style="background-color: {colour}; color: white;">
+                            <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Entity Text</th>
+                            <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Confidence</th>
+                            <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Type</th>
+                            <th style="padding: 12px; text-align: left; border: 1px solid #ddd;">Source</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+            """
+            
+            # Sort by confidence score
+            entities_of_type.sort(key=lambda x: x.get('confidence', 0), reverse=True)
+            
+            for entity in entities_of_type:
+                confidence = entity.get('confidence', 0.0)
+                confidence_colour = "#28a745" if confidence > 0.7 else "#ffc107" if confidence > 0.4 else "#dc3545"
+                source = entity.get('source', 'Unknown')
+                source_badge = f"<span style='background-color: #007bff; color: white; padding: 2px 6px; border-radius: 10px; font-size: 11px;'>{source}</span>"
+                
+                table_html += f"""
+                        <tr style="background-color: #fff;">
+                            <td style="padding: 10px; border: 1px solid #ddd; font-weight: bold;">{entity['text']}</td>
+                            <td style="padding: 10px; border: 1px solid #ddd;">
+                                <span style="color: {confidence_colour}; font-weight: bold;">
+                                    {confidence:.3f}
+                                </span>
+                            </td>
+                            <td style="padding: 10px; border: 1px solid #ddd;">{entity['label']}</td>
+                            <td style="padding: 10px; border: 1px solid #ddd;">{source_badge}</td>
+                        </tr>
+                """
+            
+            table_html += "</tbody></table></div>"
+            tab_label = f"{icon} {entity_type} ({len(entities_of_type)})"
+            tab_contents[tab_label] = table_html
     
-    accordion_html += """
-    </div>
+    return tab_contents
+
+def create_legend_html(entity_colors, standard_entities, custom_entities):
+    """Create a legend showing entity colours"""
+    if not entity_colors:
+        return ""
+
+    html = "<div style='margin: 15px 0; padding: 15px; background-color: #f8f9fa; border-radius: 8px;'>"
+    html += "<h4 style='margin: 0 0 15px 0;'>🎨 Entity Type Legend</h4>"
+
+    if standard_entities:
+        html += "<div style='margin-bottom: 15px;'>"
+        html += "<h5 style='margin: 0 0 8px 0;'>🎯 Common Entities:</h5>"
+        html += "<div style='display: flex; flex-wrap: wrap; gap: 8px;'>"
+        for entity_type in standard_entities:
+            colour = entity_colors.get(entity_type.upper(), '#ccc')
+            html += f"<span style='background-color: {colour}; padding: 4px 8px; border-radius: 15px; color: white; font-weight: bold; font-size: 12px;'>{entity_type}</span>"
+        html += "</div></div>"
+
+    if custom_entities:
+        html += "<div>"
+        html += "<h5 style='margin: 0 0 8px 0;'>✨ Custom Entities:</h5>"
+        html += "<div style='display: flex; flex-wrap: wrap; gap: 8px;'>"
+        for entity_type in custom_entities:
+            colour = entity_colors.get(entity_type.upper(), '#ccc')
+            html += f"<span style='background-color: {colour}; padding: 4px 8px; border-radius: 15px; color: white; font-weight: bold; font-size: 12px;'>{entity_type}</span>"
+        html += "</div></div>"
+
+    html += "</div>"
+    return html
+
+# Initialize the NER manager
+ner_manager = HybridNERManager()
+
+def process_text(text, standard_entities, custom_entities_str, confidence_threshold, selected_model, progress=gr.Progress()):
+    """Main processing function for Gradio interface with progress tracking"""
+    if not text.strip():
+        return "❌ Please enter some text to analyse", "", {}
+
+    progress(0.1, desc="Initialising...")
     
-    <script>
-    function toggleAccordion(header) {
-        const content = header.nextElementSibling;
-        const arrow = header.querySelector('span:last-child');
-        
-        // Toggle active class
-        header.classList.toggle('active');
-        
-        // Toggle content visibility
-        content.classList.toggle('show');
-        
-        // Toggle arrow
-        arrow.textContent = content.classList.contains('show') ? '▲' : '▼';
-    }
+    # Parse custom entities
+    custom_entities = []
+    if custom_entities_str.strip():
+        custom_entities = [entity.strip() for entity in custom_entities_str.split(',') if entity.strip()]
+
+    # Parse common entities
+    selected_standard = [entity for entity in standard_entities if entity]
+
+    if not selected_standard and not custom_entities:
+        return "❌ Please select at least one common entity type OR enter custom entity types", "", {}
+
+    progress(0.2, desc="Loading models...")
+    
+    all_entities = []
+
+    # Extract common entities using selected model
+    if selected_standard and selected_model:
+        progress(0.4, desc="Extracting common entities...")
+        standard_entities_results = ner_manager.extract_entities_by_model(text, selected_standard, selected_model, confidence_threshold)
+        all_entities.extend(standard_entities_results)
+
+    # Extract custom entities using GLiNER
+    if custom_entities:
+        progress(0.6, desc="Extracting custom entities...")
+        custom_entity_results = ner_manager.extract_gliner_entities(text, custom_entities, confidence_threshold, is_custom=True)
+        all_entities.extend(custom_entity_results)
+
+    if not all_entities:
+        return "❌ No entities found. Try lowering the confidence threshold or using different entity types.", "", {}
+
+    progress(0.8, desc="Processing results...")
+    
+    # Assign colours
+    entity_colors = ner_manager.assign_colours(selected_standard, custom_entities)
+
+    # Create outputs
+    legend_html = create_legend_html(entity_colors, selected_standard, custom_entities)
+    highlighted_html = create_highlighted_html(text, all_entities, entity_colors)
+    tab_contents = create_entity_table_gradio_tabs(all_entities, entity_colors)
+
+    progress(0.9, desc="Creating summary...")
     
-    // Auto-expand first accordion
-    document.addEventListener('DOMContentLoaded', function() {
-        const firstAccordion = document.querySelector('.accordion-header');
-        if (firstAccordion) {
-            toggleAccordion(firstAccordion);
-        }
-    });
-    </script>
+    # Create summary with shared entities terminology
+    # Note: Shared entities are those found by BOTH common NER models AND custom GLiNER
+    total_entities = len(all_entities)
+    shared_entities = find_overlapping_entities(all_entities)
+    final_count = len(shared_entities)
+    shared_count = sum(1 for e in shared_entities if e.get('is_shared', False))
+
+    summary = f"""
+    ## 📊 Analysis Summary
+    - **Total entities found:** {total_entities}
+    - **Final entities displayed:** {final_count}
+    - **Shared entities:** {shared_count}
+    - **Average confidence:** {sum(e.get('confidence', 0) for e in all_entities) / total_entities:.3f}
     """
+
+    progress(1.0, desc="Complete!")
     
-    return accordion_html
+    return summary, legend_html + highlighted_html, tab_contents
+
+# Create Gradio interface
+def create_interface():
+    with gr.Blocks(title="Hybrid NER + GLiNER Tool", theme=gr.themes.Soft()) as demo:
+        gr.Markdown("""
+        # 🎯 Hybrid NER + Custom GLiNER Entity Recognition Tool
+        
+        Combine common NER categories with your own custom entity types! This tool uses both traditional NER models and GLiNER for comprehensive entity extraction.
+        
+        ## 🤝 NEW: Overlapping entities are automatically shared with split-colour highlighting!
+        
+        ### How to use:
+        1. **📝 Enter your text** in the text area below
+        2. **🎯 Select a model** from the dropdown for common entities
+        3. **☑️ Select common entities** you want to find (PER, ORG, LOC, etc.)
+        4. **✨ Add custom entities** (comma-separated) like "relationships, occupations, skills"
+        5. **⚙️ Adjust confidence threshold** 
+        6. **🔍 Click "Analyse Text"** to see results with tabbed output
+        """)
+        
+        with gr.Row():
+            with gr.Column(scale=2):
+                text_input = gr.Textbox(
+                    label="📝 Text to Analyse",
+                    placeholder="Enter your text here...",
+                    lines=6,
+                    max_lines=10
+                )
+            
+            with gr.Column(scale=1):
+                confidence_threshold = gr.Slider(
+                    minimum=0.1,
+                    maximum=0.9,
+                    value=0.3,
+                    step=0.1,
+                    label="🎚️ Confidence Threshold"
+                )
+        
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("### 🎯 Common Entity Types")
+                
+                # Model selector
+                model_dropdown = gr.Dropdown(
+                    choices=ner_manager.model_names,
+                    value=ner_manager.model_names[0],
+                    label="Select Model for Common Entities",
+                    info="Choose which model to use for common NER"
+                )
+                
+                # Common entities with select all functionality
+                standard_entities = gr.CheckboxGroup(
+                    choices=STANDARD_ENTITIES,
+                    value=['PER', 'ORG', 'LOC', 'MISC'],  # Default selection
+                    label="Select Common Entities"
+                )
+                
+                # Select/Deselect All button
+                with gr.Row():
+                    select_all_btn = gr.Button("🔘 Deselect All", size="sm")
+                    
+                # Function for select/deselect all
+                def toggle_all_entities(current_selection):
+                    if len(current_selection) > 0:
+                        # If any are selected, deselect all
+                        return [], "☑️ Select All"
+                    else:
+                        # If none selected, select all
+                        return STANDARD_ENTITIES, "🔘 Deselect All"
+                
+                select_all_btn.click(
+                    fn=toggle_all_entities,
+                    inputs=[standard_entities],
+                    outputs=[standard_entities, select_all_btn]
+                )
+            
+            with gr.Column():
+                gr.Markdown("### ✨ Custom Entity Types")
+                custom_entities = gr.Textbox(
+                    label="Custom Entities (comma-separated)",
+                    placeholder="e.g. relationships, occupations, skills, emotions",
+                    lines=3
+                )
+                gr.Markdown("""
+                **Examples:**
+                - relationships, occupations, skills
+                - emotions, actions, objects  
+                - medical conditions, treatments
+                - financial terms, business roles
+                """)
+        
+        analyse_btn = gr.Button("🔍 Analyse Text", variant="primary", size="lg")
+        
+        # Output sections
+        with gr.Row():
+            summary_output = gr.Markdown(label="Summary")
+        
+        with gr.Row():
+            highlighted_output = gr.HTML(label="Highlighted Text")
+        
+        # Create dynamic tabs for results
+        results_tabs = gr.State({})
+        
+        def update_tabs(tab_contents):
+            """Update the results tabs based on the analysis"""
+            if not tab_contents or not isinstance(tab_contents, dict):
+                return {gr.HTML("No results to display"): gr.update(visible=True)}
+            
+            # Create tabs dynamically
+            tab_components = {}
+            for tab_name, content in tab_contents.items():
+                tab_components[tab_name] = gr.HTML(content)
+            
+            return tab_components
+        
+        # Results section with tabs
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("### 📋 Detailed Results")
+                
+                # We'll update this section dynamically
+                results_container = gr.HTML(label="Results")
+        
+        # Function to process and display results
+        def process_and_display(text, standard_entities, custom_entities, confidence_threshold, selected_model):
+            # Get results from main processing function
+            summary, highlighted, tab_contents = process_text(
+                text, standard_entities, custom_entities, confidence_threshold, selected_model
+            )
+            
+            # Create tabs HTML manually since Gradio dynamic tabs are complex
+            if isinstance(tab_contents, dict) and tab_contents:
+                # Generate unique IDs to avoid conflicts
+                import time
+                timestamp = str(int(time.time() * 1000))
+                
+                tabs_html = f"""
+                <div style="margin: 20px 0;" id="tab-container-{timestamp}">
+                    <div style="border-bottom: 2px solid #ddd; margin-bottom: 20px;">
+                """
+                
+                # Create tab buttons
+                tab_names = list(tab_contents.keys())
+                for i, tab_name in enumerate(tab_names):
+                    active_style = "background-color: #f8f9fa; border-bottom: 3px solid #4ECDC4;" if i == 0 else "background-color: #fff;"
+                    default_bg = '#f8f9fa' if i == 0 else '#fff'
+                    tabs_html += f"""
+                        <button onclick="showResultTab{timestamp}('{i}')" id="result-tab-{timestamp}-{i}"
+                                style="padding: 12px 24px; margin-right: 5px; border: 1px solid #ddd;
+                                       border-bottom: none; cursor: pointer; font-weight: bold; {active_style}
+                                       transition: all 0.3s ease;"
+                                onmouseover="this.style.backgroundColor='#e9ecef'" 
+                                onmouseout="this.style.backgroundColor='{default_bg}'">
+                            {tab_name}
+                        </button>
+                    """
+                
+                tabs_html += "</div>"
+                
+                # Create tab content
+                for i, (tab_name, content) in enumerate(tab_contents.items()):
+                    display_style = "display: block;" if i == 0 else "display: none;"
+                    tabs_html += f"""
+                        <div id="result-content-{timestamp}-{i}" style="{display_style}">
+                            {content}
+                        </div>
+                    """
+                
+                # Add JavaScript for tab switching with unique function name
+                tabs_html += f"""
+                    <script>
+                    function showResultTab{timestamp}(tabIndex) {{
+                        console.log('Tab clicked:', tabIndex);
+                        
+                        // Hide all content for this specific tab container
+                        var contents = document.querySelectorAll('[id^="result-content-{timestamp}-"]');
+                        contents.forEach(function(content) {{
+                            content.style.display = 'none';
+                        }});
+                        
+                        // Reset all tab styles for this specific tab container
+                        var tabs = document.querySelectorAll('[id^="result-tab-{timestamp}-"]');
+                        tabs.forEach(function(tab) {{
+                            tab.style.backgroundColor = '#fff';
+                            tab.style.borderBottom = 'none';
+                        }});
+                        
+                        // Show selected content
+                        var targetContent = document.getElementById('result-content-{timestamp}-' + tabIndex);
+                        if (targetContent) {{
+                            targetContent.style.display = 'block';
+                        }}
+                        
+                        // Highlight selected tab
+                        var activeTab = document.getElementById('result-tab-{timestamp}-' + tabIndex);
+                        if (activeTab) {{
+                            activeTab.style.backgroundColor = '#f8f9fa';
+                            activeTab.style.borderBottom = '3px solid #4ECDC4';
+                        }}
+                    }}
+                    
+                    // Ensure tabs are clickable after DOM load
+                    document.addEventListener('DOMContentLoaded', function() {{
+                        var tabs = document.querySelectorAll('[id^="result-tab-{timestamp}-"]');
+                        tabs.forEach(function(tab, index) {{
+                            tab.addEventListener('click', function(e) {{
+                                e.preventDefault();
+                                showResultTab{timestamp}(index.toString());
+                            }});
+                        }});
+                    }});
+                    
+                    // Also try immediate setup in case DOM is already loaded
+                    setTimeout(function() {{
+                        var tabs = document.querySelectorAll('[id^="result-tab-{timestamp}-"]');
+                        tabs.forEach(function(tab, index) {{
+                            tab.onclick = function(e) {{
+                                e.preventDefault();
+                                showResultTab{timestamp}(index.toString());
+                                return false;
+                            }};
+                        }});
+                    }}, 100);
+                    </script>
+                </div>
+                """
+                
+                results_display = tabs_html
+            else:
+                results_display = str(tab_contents) if tab_contents else "No results to display"
+            
+            return summary, highlighted, results_display
+        
+        # Connect the button to the processing function
+        analyse_btn.click(
+            fn=process_and_display,
+            inputs=[
+                text_input,
+                standard_entities,
+                custom_entities,
+                confidence_threshold,
+                model_dropdown
+            ],
+            outputs=[summary_output, highlighted_output, results_container]
+        )
+        
+        # Add examples
+        gr.Examples(
+            examples=[
+                [
+                    "John Smith works at Google in New York. He graduated from Stanford University in 2015 and specialises in artificial intelligence research. His wife Sarah is a doctor at Mount Sinai Hospital.",
+                    ["PER", "ORG", "LOC", "DATE"],
+                    "relationships, occupations, educational background",
+                    0.3,
+                    "entities_spacy_en_core_web_trf"
+                ],
+                [
+                    "The meeting between CEO Jane Doe and the board of directors at Microsoft headquarters in Seattle discussed the Q4 financial results and the new AI strategy for 2024.",
+                    ["PER", "ORG", "LOC", "DATE"],
+                    "corporate roles, business events, financial terms",
+                    0.4,
+                    "entities_flair_ner-ontonotes-large"
+                ],
+                [
+                    "Dr. Emily Watson published a research paper on machine learning algorithms at MIT. She collaborates with her colleague Prof. David Chen on natural language processing projects.",
+                    ["PER", "ORG", "Work of Art"],
+                    "academic titles, research topics, collaborations",
+                    0.3,
+                    "entities_gliner_knowledgator/modern-gliner-bi-large-v1.0"
+                ]
+            ],
+            inputs=[
+                text_input,
+                standard_entities,
+                custom_entities,
+                confidence_threshold,
+                model_dropdown
+            ]
+        )
+
+    return demo
 
-# Then in your process_text function, replace the tab creation with:
-# tab_contents = create_entity_results_accordion(all_entities, entity_colors)
-# And return it as a single HTML output instead of multiple tab outputs
+if __name__ == "__main__":
+    demo = create_interface()
+    demo.launch()