# ai_hunter_enhanced.py # Combined AI Hunter configuration GUI and detection logic import tkinter as tk from tkinter import ttk import ttkbootstrap as tb import json import os import re import unicodedata from difflib import SequenceMatcher from collections import Counter class AIHunterConfigGUI: """GUI for configuring AI Hunter detection parameters""" def __init__(self, parent, config_dict, callback=None): """ Initialize with reference to main config dictionary Args: parent: Parent window config_dict: Reference to main translator config dictionary callback: Function to call after saving """ self.parent = parent self.config = config_dict # Reference to main config self.callback = callback self.window = None # Default AI Hunter settings structure self.default_ai_hunter = { 'enabled': True, 'ai_hunter_max_workers': 1, 'retry_attempts': 6, 'disable_temperature_change': False, 'sample_size': 3000, 'thresholds': { 'exact': 90, 'text': 35, 'semantic': 85, 'structural': 85, 'character': 90, 'pattern': 80 }, 'weights': { 'exact': 1.5, 'text': 1.2, 'semantic': 1.0, 'structural': 1.0, 'character': 0.8, 'pattern': 0.8 }, 'detection_mode': 'weighted_average', 'multi_method_requirements': { 'methods_required': 3, 'min_methods': ['semantic', 'structural'] }, 'preprocessing': { 'remove_html_spacing': True, 'normalize_unicode': True, 'ignore_case': True, 'remove_extra_whitespace': True }, 'edge_filters': { 'min_text_length': 500, 'max_length_ratio': 1.3, 'min_length_ratio': 0.7 }, 'language_detection': { 'enabled': False, 'target_language': 'english', 'threshold_characters': 500, 'languages': { 'english': ['en'], 'japanese': ['ja', 'jp'], 'korean': ['ko', 'kr'], 'chinese': ['zh', 'zh-cn', 'zh-tw'], 'spanish': ['es'], 'french': ['fr'], 'german': ['de'], 'russian': ['ru'], 'arabic': ['ar'], 'hindi': ['hi'], 'portuguese': ['pt'], 'italian': ['it'], 'dutch': ['nl'], 'thai': ['th'], 'vietnamese': ['vi'], 'turkish': ['tr'], 'polish': ['pl'], 'swedish': ['sv'], 'danish': ['da'], 'norwegian': ['no'], 'finnish': ['fi'] } } } # Initialize AI Hunter config in main config if not present if 'ai_hunter_config' not in self.config: self.config['ai_hunter_config'] = self.default_ai_hunter.copy() else: # Merge with defaults to ensure all keys exist self.config['ai_hunter_config'] = self._merge_configs( self.default_ai_hunter, self.config['ai_hunter_config'] ) def _merge_configs(self, default, existing): """Recursively merge existing config with defaults""" result = default.copy() for key, value in existing.items(): if key in result and isinstance(result[key], dict) and isinstance(value, dict): result[key] = self._merge_configs(result[key], value) else: result[key] = value return result def get_ai_config(self): """Get AI Hunter configuration from main config""" return self.config.get('ai_hunter_config', self.default_ai_hunter) def show_ai_hunter_config(self): """Display the AI Hunter configuration window with scrollbar using WindowManager""" if self.window and self.window.winfo_exists(): self.window.lift() return # Import WindowManager if not already available if not hasattr(self, 'wm'): from translator_gui import WindowManager import sys import os base_dir = getattr(sys, '_MEIPASS', os.path.dirname(os.path.abspath(__file__))) self.wm = WindowManager(base_dir) # Create scrollable dialog using WindowManager dialog, scrollable_frame, canvas = self.wm.setup_scrollable( self.parent, "AI Hunter Configuration", width=820, height=None, # Will use default height max_width_ratio=0.9, max_height_ratio=0.85 ) self.window = dialog # Create notebook inside scrollable frame notebook = ttk.Notebook(scrollable_frame) notebook.pack(fill='both', expand=True, padx=10, pady=10) # Tab 1: Detection Thresholds self.create_thresholds_tab(notebook) # Tab 2: Detection Mode self.create_mode_tab(notebook) # Tab 3: Preprocessing self.create_preprocessing_tab(notebook) # Tab 4: Advanced Settings self.create_advanced_tab(notebook) # Buttons at the bottom (inside scrollable frame) button_frame = tk.Frame(scrollable_frame) button_frame.pack(fill='x', padx=10, pady=(10, 20)) tb.Button(button_frame, text="Save", command=self.apply_ai_hunter_settings, bootstyle="success").pack(side='right', padx=5) tb.Button(button_frame, text="Cancel", command=self.window.destroy, bootstyle="secondary").pack(side='right') tb.Button(button_frame, text="Reset to Defaults", command=self.reset_defaults, bootstyle="warning").pack(side='left') # Auto-resize and show self.wm.auto_resize_dialog(dialog, canvas, max_width_ratio=0.9, max_height_ratio=1.1) # Handle window close dialog.protocol("WM_DELETE_WINDOW", lambda: [dialog._cleanup_scrolling(), dialog.destroy()]) def create_thresholds_tab(self, notebook): """Create the thresholds configuration tab""" frame = ttk.Frame(notebook) notebook.add(frame, text="Detection Thresholds") # Title tk.Label(frame, text="Detection Method Thresholds", font=('TkDefaultFont', 12, 'bold')).pack(pady=10) tk.Label(frame, text="Higher values = fewer false positives (more strict)\n" "Lower values = more false positives (more sensitive)", font=('TkDefaultFont', 10), fg='gray').pack(pady=(0, 20)) # Threshold controls self.threshold_vars = {} threshold_frame = tk.Frame(frame) threshold_frame.pack(fill='both', expand=True, padx=20) descriptions = { 'exact': 'Exact Text Match - Direct character-by-character comparison', 'text': 'Smart Text Similarity - Intelligent text comparison with sampling', 'semantic': 'Semantic Analysis - Character names, dialogue patterns, numbers', 'structural': 'Structural Patterns - Paragraph structure, dialogue distribution', 'character': 'Character Overlap - Common character names between chapters', 'pattern': 'Pattern Analysis - Narrative flow and structure patterns' } ai_config = self.get_ai_config() for method, desc in descriptions.items(): method_frame = tk.Frame(threshold_frame) method_frame.pack(fill='x', pady=10) # Method name and description label_frame = tk.Frame(method_frame) label_frame.pack(fill='x') tk.Label(label_frame, text=f"{method.title()}:", font=('TkDefaultFont', 10, 'bold')).pack(side='left') tk.Label(label_frame, text=f" {desc}", font=('TkDefaultFont', 9), fg='gray').pack(side='left', padx=(10, 0)) # Slider and value slider_frame = tk.Frame(method_frame) slider_frame.pack(fill='x', pady=(5, 0)) self.threshold_vars[method] = tk.IntVar(value=ai_config['thresholds'][method]) slider = tb.Scale(slider_frame, from_=10, to=100, variable=self.threshold_vars[method], bootstyle="info", length=400) slider.pack(side='left', padx=(20, 10)) value_label = tk.Label(slider_frame, text="", width=4) value_label.pack(side='left') # Update label when slider changes def update_label(val, label=value_label, var=self.threshold_vars[method]): label.config(text=f"{int(var.get())}%") self.threshold_vars[method].trace('w', lambda *args, f=update_label: f(None)) update_label(None) # Weight configuration tk.Label(frame, text="Method Weights (for weighted average mode)", font=('TkDefaultFont', 11, 'bold')).pack(pady=(30, 10)) self.weight_vars = {} weight_frame = tk.Frame(frame) weight_frame.pack(fill='x', padx=20) for method in descriptions.keys(): w_frame = tk.Frame(weight_frame) w_frame.pack(fill='x', pady=5) tk.Label(w_frame, text=f"{method.title()} weight:", width=20, anchor='w').pack(side='left') self.weight_vars[method] = tk.DoubleVar(value=ai_config['weights'][method]) tb.Spinbox(w_frame, from_=0.1, to=2.0, increment=0.1, textvariable=self.weight_vars[method], width=10).pack(side='left', padx=10) def create_mode_tab(self, notebook): """Create the detection mode configuration tab""" frame = ttk.Frame(notebook) notebook.add(frame, text="Detection Mode") tk.Label(frame, text="Detection Mode Configuration", font=('TkDefaultFont', 12, 'bold')).pack(pady=10) # Detection mode selection mode_frame = tk.LabelFrame(frame, text="Detection Mode", padx=20, pady=20) mode_frame.pack(fill='x', padx=20, pady=10) ai_config = self.get_ai_config() self.mode_var = tk.StringVar(value=ai_config['detection_mode']) modes = [ ('single_method', 'Single Method', 'Flag as duplicate if ANY method exceeds its threshold\n(Most sensitive, most false positives)'), ('multi_method', 'Multi-Method Agreement', 'Require multiple methods to agree before flagging\n(Balanced approach)'), ('weighted_average', 'Weighted Average', 'Calculate weighted average of all methods\n(Most nuanced, least false positives)') ] for value, text, desc in modes: rb_frame = tk.Frame(mode_frame) rb_frame.pack(fill='x', pady=10) tb.Radiobutton(rb_frame, text=text, variable=self.mode_var, value=value, bootstyle="primary").pack(anchor='w') tk.Label(rb_frame, text=desc, font=('TkDefaultFont', 9), fg='gray').pack(anchor='w', padx=(25, 0)) # Multi-method configuration multi_frame = tk.LabelFrame(frame, text="Multi-Method Settings", padx=20, pady=20) multi_frame.pack(fill='x', padx=20, pady=10) tk.Label(multi_frame, text="Number of methods required to agree:", font=('TkDefaultFont', 10)).pack(anchor='w') self.methods_required_var = tk.IntVar( value=ai_config['multi_method_requirements']['methods_required']) tb.Spinbox(multi_frame, from_=1, to=6, textvariable=self.methods_required_var, width=10).pack(anchor='w', pady=5) tk.Label(multi_frame, text="Required methods (at least one must be included):", font=('TkDefaultFont', 10)).pack(anchor='w', pady=(10, 5)) self.required_method_vars = {} for method in ['exact', 'text', 'semantic', 'structural', 'character', 'pattern']: var = tk.BooleanVar( value=method in ai_config['multi_method_requirements']['min_methods']) self.required_method_vars[method] = var tb.Checkbutton(multi_frame, text=method.title(), variable=var, bootstyle="round-toggle").pack(anchor='w', padx=20) def create_preprocessing_tab(self, notebook): """Create the preprocessing configuration tab""" frame = ttk.Frame(notebook) notebook.add(frame, text="Preprocessing") tk.Label(frame, text="Text Preprocessing Options", font=('TkDefaultFont', 12, 'bold')).pack(pady=10) tk.Label(frame, text="Configure how text is processed before comparison", font=('TkDefaultFont', 10), fg='gray').pack(pady=(0, 20)) # Preprocessing options prep_frame = tk.Frame(frame) prep_frame.pack(fill='both', expand=True, padx=20) self.prep_vars = {} ai_config = self.get_ai_config() options = [ ('remove_html_spacing', 'Remove HTML with spacing', 'Replace HTML tags with spaces instead of removing completely'), ('normalize_unicode', 'Normalize Unicode', 'Normalize unicode characters (recommended)'), ('ignore_case', 'Case-insensitive comparison', 'Ignore character case when comparing'), ('remove_extra_whitespace', 'Remove extra whitespace', 'Collapse multiple spaces/newlines into single spaces') ] for key, text, desc in options: var = tk.BooleanVar(value=ai_config['preprocessing'][key]) self.prep_vars[key] = var opt_frame = tk.Frame(prep_frame) opt_frame.pack(fill='x', pady=10) tb.Checkbutton(opt_frame, text=text, variable=var, bootstyle="round-toggle").pack(anchor='w') tk.Label(opt_frame, text=desc, font=('TkDefaultFont', 9), fg='gray').pack(anchor='w', padx=(25, 0)) def create_advanced_tab(self, notebook): """Create the advanced settings tab""" frame = ttk.Frame(notebook) notebook.add(frame, text="Advanced") tk.Label(frame, text="Advanced Settings", font=('TkDefaultFont', 12, 'bold')).pack(pady=10) # General settings general_frame = tk.LabelFrame(frame, text="General", padx=20, pady=20) general_frame.pack(fill='x', padx=20, pady=10) ai_config = self.get_ai_config() # Add separator for better organization ttk.Separator(general_frame, orient='horizontal').pack(fill='x', pady=(0, 10)) # Sample size ss_frame = tk.Frame(general_frame) ss_frame.pack(fill='x', pady=5) tk.Label(ss_frame, text="Sample size:", width=20, anchor='w').pack(side='left') self.sample_size_var = tk.IntVar(value=ai_config['sample_size']) tb.Spinbox(ss_frame, from_=1000, to=10000, increment=500, textvariable=self.sample_size_var, width=10).pack(side='left', padx=10) tk.Label(ss_frame, text="characters", font=('TkDefaultFont', 9)).pack(side='left') # AI Hunter Behavior Settings tk.Label(general_frame, text="AI Hunter Behavior", font=('TkDefaultFont', 10, 'bold')).pack(anchor='w', pady=(0, 5)) # Retry Attempts retry_frame = tk.Frame(general_frame) retry_frame.pack(fill='x', pady=5) tk.Label(retry_frame, text="Retry attempts:", width=20, anchor='w').pack(side='left') self.retry_attempts_var = tk.IntVar(value=ai_config.get('retry_attempts', 3)) tb.Spinbox(retry_frame, from_=1, to=10, textvariable=self.retry_attempts_var, width=10).pack(side='left', padx=10) tk.Label(retry_frame, text="attempts", font=('TkDefaultFont', 9)).pack(side='left') # Temperature Change Toggle temp_frame = tk.Frame(general_frame) temp_frame.pack(fill='x', pady=10) self.disable_temp_change_var = tk.BooleanVar(value=ai_config.get('disable_temperature_change', False)) tb.Checkbutton(temp_frame, text="Disable temperature change behavior", variable=self.disable_temp_change_var, bootstyle="round-toggle").pack(anchor='w') tk.Label(temp_frame, text="Prevents AI Hunter from modifying temperature settings during retries", font=('TkDefaultFont', 9), fg='gray').pack(anchor='w', padx=(25, 0)) # Edge filters edge_frame = tk.LabelFrame(frame, text="Edge Case Filters", padx=20, pady=20) edge_frame.pack(fill='x', padx=20, pady=10) # Min text length min_frame = tk.Frame(edge_frame) min_frame.pack(fill='x', pady=5) tk.Label(min_frame, text="Minimum text length:", width=20, anchor='w').pack(side='left') self.min_length_var = tk.IntVar(value=ai_config['edge_filters']['min_text_length']) tb.Spinbox(min_frame, from_=100, to=2000, increment=100, textvariable=self.min_length_var, width=10).pack(side='left', padx=10) tk.Label(min_frame, text="characters", font=('TkDefaultFont', 9)).pack(side='left') # Length ratios ratio_frame = tk.Frame(edge_frame) ratio_frame.pack(fill='x', pady=10) tk.Label(ratio_frame, text="Length ratio limits:").pack(anchor='w') r_frame = tk.Frame(ratio_frame) r_frame.pack(fill='x', pady=5) tk.Label(r_frame, text="Min ratio:", width=10, anchor='w').pack(side='left', padx=(20, 5)) self.min_ratio_var = tk.DoubleVar(value=ai_config['edge_filters']['min_length_ratio']) tb.Spinbox(r_frame, from_=0.5, to=0.9, increment=0.1, textvariable=self.min_ratio_var, width=8).pack(side='left') tk.Label(r_frame, text="Max ratio:", width=10, anchor='w').pack(side='left', padx=(20, 5)) self.max_ratio_var = tk.DoubleVar(value=ai_config['edge_filters']['max_length_ratio']) tb.Spinbox(r_frame, from_=1.1, to=2.0, increment=0.1, textvariable=self.max_ratio_var, width=8).pack(side='left') tk.Label(edge_frame, text="Chapters with vastly different lengths won't be compared", font=('TkDefaultFont', 9), fg='gray').pack(anchor='w', padx=20) # Language Detection lang_frame = tk.LabelFrame(frame, text="Non-Target Language Detection", padx=20, pady=20) lang_frame.pack(fill='x', padx=20, pady=10) # Enable toggle enable_frame = tk.Frame(lang_frame) enable_frame.pack(fill='x', pady=5) self.lang_enabled_var = tk.BooleanVar(value=ai_config['language_detection']['enabled']) tb.Checkbutton(enable_frame, text="Enable non-target language detection", variable=self.lang_enabled_var, bootstyle="round-toggle").pack(anchor='w') tk.Label(enable_frame, text="Trigger retranslation when too much non-target language is detected", font=('TkDefaultFont', 9), fg='gray').pack(anchor='w', padx=(25, 0)) # Target language selection target_frame = tk.Frame(lang_frame) target_frame.pack(fill='x', pady=10) tk.Label(target_frame, text="Target language:", width=20, anchor='w').pack(side='left') self.target_lang_var = tk.StringVar(value=ai_config['language_detection']['target_language']) lang_options = list(ai_config['language_detection']['languages'].keys()) target_combo = ttk.Combobox(target_frame, textvariable=self.target_lang_var, values=lang_options, state='readonly', width=15) target_combo.pack(side='left', padx=10) tk.Label(target_frame, text="Language that should be in the translation", font=('TkDefaultFont', 9), fg='gray').pack(side='left', padx=(10, 0)) # Threshold setting thresh_frame = tk.Frame(lang_frame) thresh_frame.pack(fill='x', pady=5) tk.Label(thresh_frame, text="Character threshold:", width=20, anchor='w').pack(side='left') self.lang_threshold_var = tk.IntVar(value=ai_config['language_detection']['threshold_characters']) tb.Spinbox(thresh_frame, from_=100, to=2000, increment=50, textvariable=self.lang_threshold_var, width=10).pack(side='left', padx=10) tk.Label(thresh_frame, text="non-target language characters to trigger retranslation", font=('TkDefaultFont', 9), fg='gray').pack(side='left') def apply_ai_hunter_settings(self): """Apply AI Hunter settings to the main config""" ai_config = self.get_ai_config() # Update from GUI variables for method, var in self.threshold_vars.items(): ai_config['thresholds'][method] = var.get() for method, var in self.weight_vars.items(): ai_config['weights'][method] = var.get() ai_config['detection_mode'] = self.mode_var.get() ai_config['multi_method_requirements']['methods_required'] = self.methods_required_var.get() min_methods = [method for method, var in self.required_method_vars.items() if var.get()] ai_config['multi_method_requirements']['min_methods'] = min_methods for key, var in self.prep_vars.items(): ai_config['preprocessing'][key] = var.get() ai_config['sample_size'] = self.sample_size_var.get() ai_config['edge_filters']['min_text_length'] = self.min_length_var.get() ai_config['edge_filters']['min_length_ratio'] = self.min_ratio_var.get() ai_config['edge_filters']['max_length_ratio'] = self.max_ratio_var.get() # Language detection settings ai_config['language_detection']['enabled'] = self.lang_enabled_var.get() ai_config['language_detection']['target_language'] = self.target_lang_var.get() ai_config['language_detection']['threshold_characters'] = self.lang_threshold_var.get() # Update retry attempts and temperature change settings ai_config['retry_attempts'] = self.retry_attempts_var.get() ai_config['disable_temperature_change'] = self.disable_temp_change_var.get() # Update main config self.config['ai_hunter_config'] = ai_config # Call callback if provided (this should trigger main save_configuration) if self.callback: self.callback() self.window.destroy() def reset_defaults(self): """Reset all values to defaults""" import tkinter.messagebox as messagebox result = messagebox.askyesno("Reset to Defaults", "Are you sure you want to reset all settings to defaults?") if result: self.config['ai_hunter_config'] = self.default_ai_hunter.copy() self.window.destroy() self.show_ai_hunter_config() # Reopen with default values class ImprovedAIHunterDetection: """Improved AI Hunter detection methods for TranslateKRtoEN""" def __init__(self, main_config): """ Initialize with reference to main config Args: main_config: Reference to main translator config dictionary """ self.main_config = main_config # Default AI Hunter settings self.default_ai_hunter = { 'enabled': True, 'lookback_chapters': 5, 'retry_attempts': 3, 'disable_temperature_change': False, 'sample_size': 3000, 'thresholds': { 'exact': 90, 'text': 85, 'semantic': 85, 'structural': 85, 'character': 80, 'pattern': 80 }, 'weights': { 'exact': 1.5, 'text': 1.2, 'semantic': 1.0, 'structural': 1.0, 'character': 0.8, 'pattern': 0.8 }, 'detection_mode': 'multi_method', 'multi_method_requirements': { 'methods_required': 2, 'min_methods': ['semantic', 'structural'] }, 'preprocessing': { 'remove_html_spacing': True, 'normalize_unicode': True, 'ignore_case': True, 'remove_extra_whitespace': True }, 'edge_filters': { 'min_text_length': 500, 'max_length_ratio': 1.3, 'min_length_ratio': 0.7 }, 'language_detection': { 'enabled': False, 'target_language': 'english', 'threshold_characters': 500, 'languages': { 'english': ['en'], 'japanese': ['ja', 'jp'], 'korean': ['ko', 'kr'], 'chinese': ['zh', 'zh-cn', 'zh-tw'], 'spanish': ['es'], 'french': ['fr'], 'german': ['de'], 'russian': ['ru'], 'arabic': ['ar'], 'hindi': ['hi'], 'portuguese': ['pt'], 'italian': ['it'], 'dutch': ['nl'], 'thai': ['th'], 'vietnamese': ['vi'], 'turkish': ['tr'], 'polish': ['pl'], 'swedish': ['sv'], 'danish': ['da'], 'norwegian': ['no'], 'finnish': ['fi'] } } } def get_ai_config(self): """Get AI Hunter configuration from main config""" return self.main_config.get('ai_hunter_config', self.default_ai_hunter) def detect_duplicate_ai_hunter_enhanced(self, result, idx, prog, out, current_chapter_num=None): """Enhanced AI Hunter duplicate detection with configurable parameters""" try: print(f"\n ========== AI HUNTER DEBUG START ==========") print(f" 📍 Current chapter index: {idx}") if current_chapter_num: print(f" 📖 Current chapter number: {current_chapter_num}") # Get configuration config = self.get_ai_config() if not config.get('enabled', True): print(f" ⚠️ AI Hunter is disabled") print(f" ========== AI HUNTER DEBUG END ==========\n") return False, 0 # Preprocess text result_clean = self._preprocess_text(result, config['preprocessing']) print(f" 📄 Text length after preprocessing: {len(result_clean)} chars") # Check for non-target language detection if config['language_detection']['enabled']: non_target_detected, non_target_count = self._check_non_target_language( result_clean, config['language_detection'] ) if non_target_detected: print(f"\n 🌐 NON-TARGET LANGUAGE DETECTED!") print(f" Non-target characters found: {non_target_count}") print(f" Threshold: {config['language_detection']['threshold_characters']}") print(f" Target language: {config['language_detection']['target_language']}") print(f" ========== AI HUNTER DEBUG END ==========\n") return True, 100 # High confidence for language detection # Check edge cases if len(result_clean) < config['edge_filters']['min_text_length']: print(f" ⚠️ Text too short ({len(result_clean)} < {config['edge_filters']['min_text_length']})") print(f" ========== AI HUNTER DEBUG END ==========\n") return False, 0 # Extract features print(f" 🔬 Extracting text features...") result_features = self._extract_text_features(result_clean) # Get lookback from main config, then fall back to env var if not found lookback = self.main_config.get('duplicate_lookback_chapters', int(os.getenv('DUPLICATE_LOOKBACK_CHAPTERS', '5'))) # Log configuration print(f"\n 🔧 Configuration:") print(f" Detection mode: {config['detection_mode']}") print(f" Lookback chapters: {lookback}") print(f" Sample size: {config['sample_size']}") # FIX: Get all completed chapters sorted by actual chapter number completed_chapters = [] for chapter_key, chapter_info in prog["chapters"].items(): if chapter_info.get("status") == "completed" and chapter_info.get("output_file"): # Handle both numeric and hash-based chapter keys try: # Get actual_num from progress (this is the real chapter number) chapter_num = chapter_info.get("actual_num") if chapter_num is None: # Try chapter_num as fallback chapter_num = chapter_info.get("chapter_num") if chapter_num is None: # Skip chapters without valid numbers print(f" ⚠️ No chapter number found for key {chapter_key}, skipping") continue completed_chapters.append({ 'key': chapter_key, 'num': chapter_num, 'file': chapter_info.get("output_file"), 'ai_features': chapter_info.get("ai_features") }) except Exception as e: print(f" ⚠️ Error processing chapter {chapter_key}: {e}") continue # Sort by actual chapter number completed_chapters.sort(key=lambda x: x['num']) # If no current chapter number provided, try to infer it if current_chapter_num is None: # The current chapter should be passed in, but if not, we need to find it # Since we're using content hash keys, we can't use idx directly print(f" ⚠️ No current chapter number provided") print(f" 📊 Current index: {idx}") # The current chapter number should have been passed from the wrapper # If it wasn't, we have a problem print(f" ❌ ERROR: Current chapter number not provided to AI Hunter!") print(f" ❌ This indicates the wrapper function is not passing the chapter number correctly") # Emergency: just use a high number so we don't compare against anything current_chapter_num = 999999 print(f" ⚠️ Using index-based chapter number: {current_chapter_num}") print(f"\n 📚 Found {len(completed_chapters)} completed chapters in progress") if completed_chapters: chapter_nums = [ch['num'] for ch in completed_chapters] print(f" 📊 Chapter numbers in progress: {sorted(chapter_nums)[:10]}{'...' if len(chapter_nums) > 10 else ''}") print(f" 🎯 Current chapter number: {current_chapter_num}") print(f" 🔍 Will check against last {lookback} chapters before chapter {current_chapter_num}") # Check previous chapters all_similarities = [] highest_similarity = 0.0 detected_method = None detected_chapter = None # FIX: Look at chapters by actual number, not index chapters_checked = 0 for completed_chapter in reversed(completed_chapters): # Only check chapters that come before the current one if completed_chapter['num'] >= current_chapter_num: continue # Only check up to lookback number of chapters if chapters_checked >= lookback: break chapters_checked += 1 print(f"\n 📝 Checking against chapter {completed_chapter['num']}...") # Get previous chapter features prev_features = completed_chapter.get('ai_features') prev_clean = None # Try to get cached features first if prev_features: print(f" ✅ Using cached features") else: # Read and extract features prev_path = os.path.join(out, completed_chapter['file']) if os.path.exists(prev_path): try: with open(prev_path, 'r', encoding='utf-8') as f: prev_content = f.read() prev_clean = self._preprocess_text(prev_content, config['preprocessing']) # Check length ratio len_ratio = len(result_clean) / max(1, len(prev_clean)) if (len_ratio < config['edge_filters']['min_length_ratio'] or len_ratio > config['edge_filters']['max_length_ratio']): print(f" ⚠️ Length ratio out of bounds: {len_ratio:.2f}") continue prev_features = self._extract_text_features(prev_clean) print(f" 📄 Extracted features from file") except Exception as e: print(f" ❌ Failed to read file: {e}") continue else: print(f" ❌ File not found: {prev_path}") continue # Calculate similarities print(f" 🔍 Calculating similarities...") similarities = self._calculate_all_similarities( result_clean, result_features, prev_clean, prev_features, config ) # Store for reporting all_similarities.append({ 'chapter': completed_chapter['num'], 'similarities': similarities }) # Log similarity scores for method, score in similarities.items(): if score > 0: print(f" {method}: {int(score*100)}%") # Check if duplicate based on configured mode is_duplicate, confidence, methods_triggered = self._evaluate_duplicate( similarities, config ) if is_duplicate: print(f"\n 🚨 DUPLICATE DETECTED!") print(f" Detection mode: {config['detection_mode']}") print(f" Confidence: {int(confidence*100)}%") print(f" Triggered methods: {', '.join(methods_triggered)}") print(f" Match with: Chapter {completed_chapter['num']}") print(f" ========== AI HUNTER DEBUG END ==========\n") return True, int(confidence * 100) # Track highest for reporting for method, sim in similarities.items(): if sim > highest_similarity: highest_similarity = sim detected_method = method detected_chapter = completed_chapter['num'] # No duplicate found print(f"\n ✅ No duplicate found") if detected_method: print(f" Highest similarity: {int(highest_similarity*100)}% via {detected_method}") print(f" Closest match: Chapter {detected_chapter}") # Show top 3 closest matches if all_similarities: print(f"\n 📊 Top 3 closest matches:") sorted_chapters = sorted(all_similarities, key=lambda x: self._get_chapter_score(x['similarities'], config), reverse=True)[:3] for i, chapter_data in enumerate(sorted_chapters, 1): score = self._get_chapter_score(chapter_data['similarities'], config) print(f" {i}. Chapter {chapter_data['chapter']}: {int(score*100)}%") print(f" ========== AI HUNTER DEBUG END ==========\n") return False, 0 except Exception as e: print(f" ❌ AI Hunter detection failed with error: {e}") import traceback print(f" {traceback.format_exc()}") print(f" ========== AI HUNTER DEBUG END ==========\n") return False, 0 def _preprocess_text(self, text, prep_config): """Preprocess text according to configuration""" # Remove HTML if prep_config.get('remove_html_spacing', True): text = re.sub(r'<[^>]+>', ' ', text) else: text = re.sub(r'<[^>]+>', '', text) # Normalize unicode if prep_config.get('normalize_unicode', True): text = unicodedata.normalize('NFKD', text) # Remove extra whitespace if prep_config.get('remove_extra_whitespace', True): text = re.sub(r'\s+', ' ', text) text = re.sub(r'\n\s*\n', '\n\n', text) text = text.strip() # Convert to lowercase if case-insensitive if prep_config.get('ignore_case', True): text = text.lower() return text def _calculate_all_similarities(self, result_clean, result_features, prev_clean, prev_features, config): """Calculate all similarity metrics""" similarities = {} # Method 1: Exact content match if prev_clean is not None: sample_size = min(config['sample_size'], len(result_clean), len(prev_clean)) exact_sim = self._calculate_exact_similarity( result_clean[:sample_size], prev_clean[:sample_size] ) similarities['exact'] = exact_sim # Method 2: Smart text similarity text_sim = self._calculate_smart_similarity( result_clean, prev_clean, config['sample_size'] ) similarities['text'] = text_sim else: similarities['exact'] = 0.0 similarities['text'] = 0.0 # Method 3: Semantic fingerprint semantic_sim = self._calculate_semantic_similarity( result_features.get('semantic', {}), prev_features.get('semantic', {}) ) similarities['semantic'] = semantic_sim # Method 4: Structural signature structural_sim = self._calculate_structural_similarity( result_features.get('structural', {}), prev_features.get('structural', {}) ) similarities['structural'] = structural_sim # Method 5: Character analysis char_sim = self._calculate_character_similarity( result_features.get('characters', []), prev_features.get('characters', []) ) similarities['character'] = char_sim # Method 6: Pattern analysis pattern_sim = self._calculate_pattern_similarity( result_features.get('patterns', {}), prev_features.get('patterns', {}) ) similarities['pattern'] = pattern_sim return similarities def _evaluate_duplicate(self, similarities, config): """Evaluate if similarities indicate a duplicate based on detection mode""" mode = config['detection_mode'] thresholds = {k: v/100.0 for k, v in config['thresholds'].items()} if mode == 'single_method': # Any method exceeding threshold for method, sim in similarities.items(): if sim >= thresholds.get(method, 0.85): return True, sim, [method] return False, 0, [] elif mode == 'multi_method': # Multiple methods must agree triggered_methods = [] for method, sim in similarities.items(): if sim >= thresholds.get(method, 0.85): triggered_methods.append(method) # Check if enough methods triggered required = config.get('multi_method_requirements', {}).get('methods_required', 2) min_methods = config.get('multi_method_requirements', {}).get('min_methods', []) if len(triggered_methods) >= required: # Check if at least one required method is included if not min_methods or any(m in triggered_methods for m in min_methods): # Calculate average confidence of triggered methods confidence = sum(similarities[m] for m in triggered_methods) / len(triggered_methods) return True, confidence, triggered_methods return False, 0, [] elif mode == 'weighted_average': # Calculate weighted average weights = config.get('weights', {}) total_weight = sum(weights.get(m, 1.0) for m in similarities) weighted_sum = sum(similarities[m] * weights.get(m, 1.0) for m in similarities) weighted_avg = weighted_sum / total_weight if total_weight > 0 else 0 # Check if weighted average exceeds average threshold avg_threshold = sum(thresholds.values()) / len(thresholds) if thresholds else 0.85 if weighted_avg >= avg_threshold: # Find which methods contributed most triggered = [m for m, sim in similarities.items() if sim >= thresholds.get(m, 0.85)] return True, weighted_avg, triggered return False, 0, [] return False, 0, [] def _get_chapter_score(self, similarities, config): """Calculate overall score for a chapter comparison""" if config['detection_mode'] == 'weighted_average': weights = config.get('weights', {}) total_weight = sum(weights.get(m, 1.0) for m in similarities) return sum(similarities.get(m, 0) * weights.get(m, 1.0) for m in similarities) / total_weight if total_weight > 0 else 0 else: return max(similarities.values()) if similarities else 0 def _extract_text_features(self, text): """Extract multiple features from text for AI Hunter analysis""" features = { 'semantic': {}, 'structural': {}, 'characters': [], 'patterns': {} } # Semantic fingerprint lines = text.split('\n') # Character extraction (names that appear 3+ times) words = re.findall(r'\b[A-Z][a-z]+\b', text) word_freq = Counter(words) features['characters'] = [name for name, count in word_freq.items() if count >= 3 and name not in { 'The', 'A', 'An', 'In', 'On', 'At', 'To', 'From', 'With', 'By', 'For', 'Of', 'As', 'But', 'And', 'Or', 'He', 'She', 'It', 'They', 'We', 'You', 'What', 'When', 'Where', 'Who', 'Why', 'How', 'That', 'This', 'These' }] # Dialogue patterns dialogue_patterns = re.findall(r'"([^"]+)"', text) features['semantic']['dialogue_count'] = len(dialogue_patterns) features['semantic']['dialogue_lengths'] = [len(d) for d in dialogue_patterns[:10]] # Speaker patterns speaker_patterns = re.findall(r'(\w+)\s+(?:said|asked|replied|shouted|whispered)', text.lower()) features['semantic']['speakers'] = list(set(speaker_patterns[:20])) # Number extraction numbers = re.findall(r'\b\d+\b', text) features['patterns']['numbers'] = numbers[:20] # Structural signature para_lengths = [] dialogue_count = 0 for para in text.split('\n\n'): if para.strip(): para_lengths.append(len(para)) if '"' in para: dialogue_count += 1 features['structural']['para_count'] = len(para_lengths) features['structural']['avg_para_length'] = sum(para_lengths) / max(1, len(para_lengths)) features['structural']['dialogue_ratio'] = dialogue_count / max(1, len(para_lengths)) # Create structural pattern string pattern = [] for para in text.split('\n\n')[:20]: # First 20 paragraphs if para.strip(): if '"' in para: pattern.append('D') # Dialogue elif len(para) > 300: pattern.append('L') # Long elif len(para) < 100: pattern.append('S') # Short else: pattern.append('M') # Medium features['structural']['pattern'] = ''.join(pattern) # Action density action_verbs = len(re.findall(r'\b\w+ed\b', text)) features['semantic']['action_density'] = action_verbs / max(1, len(text.split())) # Text length features['semantic']['text_length'] = len(text) return features def _calculate_exact_similarity(self, text1, text2): """Calculate exact text similarity""" return SequenceMatcher(None, text1, text2).ratio() def _calculate_smart_similarity(self, text1, text2, sample_size): """Smart similarity with configurable sample size""" if len(text1) > sample_size * 3 and len(text2) > sample_size * 3: # Use multiple samples samples1 = [ text1[:sample_size], text1[len(text1)//2 - sample_size//2:len(text1)//2 + sample_size//2], text1[-sample_size:] ] samples2 = [ text2[:sample_size], text2[len(text2)//2 - sample_size//2:len(text2)//2 + sample_size//2], text2[-sample_size:] ] similarities = [SequenceMatcher(None, s1, s2).ratio() for s1, s2 in zip(samples1, samples2)] return sum(similarities) / len(similarities) else: # Use full text up to sample size return SequenceMatcher(None, text1[:sample_size], text2[:sample_size]).ratio() def _calculate_semantic_similarity(self, sem1, sem2): """Calculate semantic fingerprint similarity""" score = 0.0 weights = 0.0 # Compare dialogue counts if 'dialogue_count' in sem1 and 'dialogue_count' in sem2: weights += 0.3 if sem1['dialogue_count'] > 0 or sem2['dialogue_count'] > 0: ratio = min(sem1['dialogue_count'], sem2['dialogue_count']) / \ max(1, max(sem1['dialogue_count'], sem2['dialogue_count'])) score += ratio * 0.3 # Compare speakers if 'speakers' in sem1 and 'speakers' in sem2: weights += 0.4 if sem1['speakers'] and sem2['speakers']: overlap = len(set(sem1['speakers']) & set(sem2['speakers'])) total = len(set(sem1['speakers']) | set(sem2['speakers'])) score += (overlap / max(1, total)) * 0.4 elif not sem1['speakers'] and not sem2['speakers']: score += 0.4 # Both have no speakers # Compare dialogue lengths pattern if 'dialogue_lengths' in sem1 and 'dialogue_lengths' in sem2: weights += 0.2 if sem1['dialogue_lengths'] and sem2['dialogue_lengths']: len1 = sem1['dialogue_lengths'][:10] len2 = sem2['dialogue_lengths'][:10] if len1 and len2: avg1 = sum(len1) / len(len1) avg2 = sum(len2) / len(len2) ratio = min(avg1, avg2) / max(1, max(avg1, avg2)) score += ratio * 0.2 elif not sem1['dialogue_lengths'] and not sem2['dialogue_lengths']: score += 0.2 # Both have no dialogue # Action density if 'action_density' in sem1 and 'action_density' in sem2: weights += 0.1 act_sim = 1 - abs(sem1['action_density'] - sem2['action_density']) score += act_sim * 0.1 return score / max(0.1, weights) def _calculate_structural_similarity(self, struct1, struct2): """Calculate structural signature similarity""" score = 0.0 # Compare paragraph patterns if 'pattern' in struct1 and 'pattern' in struct2: pattern_sim = SequenceMatcher(None, struct1['pattern'], struct2['pattern']).ratio() score += pattern_sim * 0.5 # Compare paragraph statistics if all(k in struct1 for k in ['para_count', 'avg_para_length', 'dialogue_ratio']) and \ all(k in struct2 for k in ['para_count', 'avg_para_length', 'dialogue_ratio']): # Paragraph count ratio para_ratio = min(struct1['para_count'], struct2['para_count']) / \ max(1, max(struct1['para_count'], struct2['para_count'])) score += para_ratio * 0.2 # Average length ratio avg_ratio = min(struct1['avg_para_length'], struct2['avg_para_length']) / \ max(1, max(struct1['avg_para_length'], struct2['avg_para_length'])) score += avg_ratio * 0.15 # Dialogue ratio similarity dialogue_diff = abs(struct1['dialogue_ratio'] - struct2['dialogue_ratio']) score += (1 - min(1, dialogue_diff)) * 0.15 return score def _calculate_character_similarity(self, chars1, chars2): """Calculate character overlap similarity""" if not chars1 or not chars2: return 0.0 # Convert to sets set1 = set(chars1) set2 = set(chars2) # If no overlap at all, return 0 intersection = set1 & set2 if not intersection: return 0.0 # Calculate Jaccard index (intersection over union) union = set1 | set2 jaccard = len(intersection) / len(union) # Also consider the proportion of matching characters relative to each set # This prevents small overlaps from scoring too high overlap1 = len(intersection) / len(set1) overlap2 = len(intersection) / len(set2) # Take the minimum overlap to be more conservative min_overlap = min(overlap1, overlap2) # Combine jaccard and overlap scores # Jaccard penalizes when sets are very different sizes # Min overlap ensures both texts share a significant portion of characters score = (jaccard + min_overlap) / 2 return score def _calculate_pattern_similarity(self, pat1, pat2): """Calculate pattern similarity (numbers, etc.)""" score = 0.0 # Number overlap if 'numbers' in pat1 and 'numbers' in pat2: nums1 = set(pat1['numbers']) nums2 = set(pat2['numbers']) if nums1 or nums2: overlap = len(nums1 & nums2) total = len(nums1 | nums2) score = overlap / max(1, total) else: score = 1.0 # Both have no numbers return score def _check_non_target_language(self, text, lang_config): """Check if text contains too much non-target language""" target_language = lang_config['target_language'].lower() threshold = lang_config['threshold_characters'] # Character ranges for different languages language_ranges = { 'english': [ # Latin script + basic symbols (0x0000, 0x007F), # Basic Latin (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B (0x2000, 0x206F), # General Punctuation (0x20A0, 0x20CF), # Currency Symbols (0xFF00, 0xFFEF), # Halfwidth and Fullwidth Forms ], 'japanese': [ (0x3040, 0x309F), # Hiragana (0x30A0, 0x30FF), # Katakana (0x4E00, 0x9FAF), # CJK Unified Ideographs (0x3400, 0x4DBF), # CJK Extension A (0xFF66, 0xFF9F), # Halfwidth Katakana ], 'korean': [ (0xAC00, 0xD7AF), # Hangul Syllables (0x1100, 0x11FF), # Hangul Jamo (0x3130, 0x318F), # Hangul Compatibility Jamo (0xA960, 0xA97F), # Hangul Jamo Extended-A (0xD7B0, 0xD7FF), # Hangul Jamo Extended-B ], 'chinese': [ (0x4E00, 0x9FAF), # CJK Unified Ideographs (0x3400, 0x4DBF), # CJK Extension A (0x20000, 0x2A6DF), # CJK Extension B (0x2A700, 0x2B73F), # CJK Extension C (0x2B740, 0x2B81F), # CJK Extension D (0x3000, 0x303F), # CJK Symbols and Punctuation ], 'arabic': [ (0x0600, 0x06FF), # Arabic (0x0750, 0x077F), # Arabic Supplement (0x08A0, 0x08FF), # Arabic Extended-A (0xFB50, 0xFDFF), # Arabic Presentation Forms-A (0xFE70, 0xFEFF), # Arabic Presentation Forms-B ], 'russian': [ (0x0400, 0x04FF), # Cyrillic (0x0500, 0x052F), # Cyrillic Supplement (0x2DE0, 0x2DFF), # Cyrillic Extended-A (0xA640, 0xA69F), # Cyrillic Extended-B ], 'thai': [ (0x0E00, 0x0E7F), # Thai ], 'hindi': [ (0x0900, 0x097F), # Devanagari (0xA8E0, 0xA8FF), # Devanagari Extended ], 'spanish': [ # Same as English (Latin script) (0x0000, 0x007F), # Basic Latin (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B ], 'french': [ # Same as English (Latin script) (0x0000, 0x007F), # Basic Latin (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B ], 'german': [ # Same as English (Latin script) (0x0000, 0x007F), # Basic Latin (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B ], 'portuguese': [ # Same as English (Latin script) (0x0000, 0x007F), # Basic Latin (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B ], 'italian': [ # Same as English (Latin script) (0x0000, 0x007F), # Basic Latin (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B ], 'dutch': [ # Same as English (Latin script) (0x0000, 0x007F), # Basic Latin (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B ], 'vietnamese': [ (0x0000, 0x007F), # Basic Latin (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B (0x1EA0, 0x1EFF), # Latin Extended Additional (Vietnamese) ], 'turkish': [ (0x0000, 0x007F), # Basic Latin (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B ], 'polish': [ (0x0000, 0x007F), # Basic Latin (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B ], 'swedish': [ # Same as English (Latin script) (0x0000, 0x007F), # Basic Latin (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B ], 'danish': [ # Same as English (Latin script) (0x0000, 0x007F), # Basic Latin (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B ], 'norwegian': [ # Same as English (Latin script) (0x0000, 0x007F), # Basic Latin (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B ], 'finnish': [ # Same as English (Latin script) (0x0000, 0x007F), # Basic Latin (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B ], } # Get target language ranges target_ranges = language_ranges.get(target_language, language_ranges['english']) # Count characters that are NOT in target language ranges non_target_count = 0 total_letters = 0 for char in text: # Skip whitespace, punctuation, and numbers for counting if char.isspace() or char.isdigit(): continue # Count as letter character total_letters += 1 # Check if character is in any target language range char_code = ord(char) is_target_char = any(start <= char_code <= end for start, end in target_ranges) if not is_target_char: non_target_count += 1 # Debug logging if non_target_count > 0: print(f" 🌐 Language detection: {non_target_count}/{total_letters} non-target chars ({target_language})") # Return True if non-target character count exceeds threshold return non_target_count >= threshold, non_target_count