\n" + chapter_body_content + print(f" → MERGED: Section ({len(section_text)} chars) + Chapter") + else: + print(f" → NOT MERGED: Section too large ({len(section_text)} chars)") + # Remove from processed files so it gets processed separately + processed_files.discard(section_file) + + except Exception as e: + print(f"[WARNING] Failed to merge {file_path}: {e}") + + # === ENHANCED EXTRACTION POINT === + # Initialize variables that will be set by extraction + content_html = None + content_text = None + chapter_title = None + enhanced_extraction_used = False + + # Determine whether to use enhanced extractor based on toggle and provider + use_enhanced = enhanced_extractor and extraction_mode == "enhanced" + force_bs_traditional = False + try: + force_bs = os.getenv('FORCE_BS_FOR_TRADITIONAL', '0') == '1' + model_env = os.getenv('MODEL', '') + if force_bs and is_traditional_translation_api(model_env): + use_enhanced = False + force_bs_traditional = True + except Exception: + pass + + # Use enhanced extractor if available and allowed + if use_enhanced: + print(f"🚀 Using enhanced extraction for: {os.path.basename(file_path)}") + # Get clean text from html2text + clean_content, _, chapter_title = enhanced_extractor.extract_chapter_content( + html_content, enhanced_filtering + ) + enhanced_extraction_used = True + print(f"✅ Enhanced extraction complete: {len(clean_content)} chars") + + # For enhanced mode, store the markdown/plain text + # This will be sent to the translation API as-is + content_html = clean_content # This is MARKDOWN/PLAIN TEXT from html2text + content_text = clean_content # Same clean text for analysis + + # BeautifulSoup method (only for non-enhanced modes) + if not enhanced_extraction_used: + if extraction_mode == "enhanced" and not force_bs_traditional: + # Enhanced mode failed - skip this file + print(f"❌ Skipping {file_path} - enhanced extraction required but not available") + return None + # Parse the (possibly merged) content + protected_html = self.protect_angle_brackets_with_korean(html_content) + + # Use lxml parser which handles both HTML and XHTML well + soup = BeautifulSoup(protected_html, self.parser) + + # Get effective mode for filtering + effective_filtering = enhanced_filtering if extraction_mode == "enhanced" else extraction_mode + + # In full mode, keep the entire HTML structure + if effective_filtering == "full": + content_html = html_content # Keep EVERYTHING + content_text = soup.get_text(strip=True) + else: + # Smart and comprehensive modes extract body content + if soup.body: + content_html = str(soup.body) + content_text = soup.body.get_text(strip=True) + else: + content_html = html_content + content_text = soup.get_text(strip=True) + + # Extract title (with ignore settings support) + chapter_title = None + + # Check ignore settings for batch translation + batch_translate_active = os.getenv('BATCH_TRANSLATE_HEADERS', '0') == '1' + ignore_title_tag = os.getenv('IGNORE_TITLE', '0') == '1' and batch_translate_active + ignore_header_tags = os.getenv('IGNORE_HEADER', '0') == '1' and batch_translate_active + + # Extract from title tag if not ignored + if not ignore_title_tag and soup.title and soup.title.string: + chapter_title = soup.title.string.strip() + + # Extract from header tags if not ignored and no title found + if not chapter_title and not ignore_header_tags: + for header_tag in ['h1', 'h2', 'h3']: + header = soup.find(header_tag) + if header: + chapter_title = header.get_text(strip=True) + break + + # Fallback to filename if nothing found + if not chapter_title: + chapter_title = os.path.splitext(os.path.basename(file_path))[0] + + # Get the effective extraction mode for processing logic + effective_mode = enhanced_filtering if extraction_mode == "enhanced" else extraction_mode + + # Skip truly empty files in smart mode + # BUT: Never skip anything when merging is disabled (to ensure section files are processed) + if effective_mode == "smart" and not disable_merging and len(content_text.strip()) < 10: + print(f"[SKIP] Nearly empty file: {file_path} ({len(content_text)} chars)") + return None + + # Get actual chapter number based on original position + actual_chapter_num = files_to_process.index(file_path) + 1 + + # Mode-specific logic + if effective_mode == "comprehensive" or effective_mode == "full": + # For comprehensive/full mode, use sequential numbering + chapter_num = actual_chapter_num + + if not chapter_title: + chapter_title = os.path.splitext(os.path.basename(file_path))[0] + + detection_method = f"{extraction_mode}_sequential" if extraction_mode == "enhanced" else f"{effective_mode}_sequential" + + elif effective_mode == "smart": + # For smart mode, when merging is disabled, use sequential numbering + if disable_merging: + chapter_num = actual_chapter_num + + if not chapter_title: + chapter_title = os.path.splitext(os.path.basename(file_path))[0] + + detection_method = f"{extraction_mode}_sequential_no_merge" if extraction_mode == "enhanced" else "sequential_no_merge" + else: + # When merging is enabled, try to extract chapter info + protected_html = self.protect_angle_brackets_with_korean(html_content) + soup = BeautifulSoup(protected_html, self.parser) + + # Count headers (thread-safe) + h1_tags = soup.find_all('h1') + h2_tags = soup.find_all('h2') + if h1_tags: + with h1_count_lock: + h1_count += 1 + if h2_tags: + with h2_count_lock: + h2_count += 1 + + # Try to extract chapter number and title + chapter_num, extracted_title, detection_method = self._extract_chapter_info( + soup, file_path, content_text, html_content + ) + + # Use extracted title if we don't have one + if extracted_title and not chapter_title: + chapter_title = extracted_title + + # For hash-based filenames, chapter_num might be None + if chapter_num is None: + chapter_num = actual_chapter_num # Use actual chapter count + detection_method = f"{extraction_mode}_sequential_fallback" if extraction_mode == "enhanced" else "sequential_fallback" + print(f"[DEBUG] No chapter number found in {file_path}, assigning: {chapter_num}") + + # Filter content_html for ignore settings (before processing) + batch_translate_active = os.getenv('BATCH_TRANSLATE_HEADERS', '0') == '1' + ignore_title_tag = os.getenv('IGNORE_TITLE', '0') == '1' and batch_translate_active + ignore_header_tags = os.getenv('IGNORE_HEADER', '0') == '1' and batch_translate_active + + if (ignore_title_tag or ignore_header_tags) and content_html and not enhanced_extraction_used: + # Parse the content HTML to remove ignored tags + content_soup = BeautifulSoup(content_html, self.parser) + + # Remove title tags if ignored + if ignore_title_tag: + for title_tag in content_soup.find_all('title'): + title_tag.decompose() + + # Remove header tags if ignored + if ignore_header_tags: + for header_tag in content_soup.find_all(['h1', 'h2', 'h3']): + header_tag.decompose() + + # Update content_html with filtered version + content_html = str(content_soup) + + # Process images and metadata (same for all modes) + protected_html = self.protect_angle_brackets_with_korean(html_content) + soup = BeautifulSoup(protected_html, self.parser) + images = soup.find_all('img') + has_images = len(images) > 0 + is_image_only_chapter = has_images and len(content_text.strip()) < 500 + + if is_image_only_chapter: + print(f"[DEBUG] Image-only chapter detected: {file_path} ({len(images)} images, {len(content_text)} chars)") + + content_hash = ContentProcessor.get_content_hash(content_html) + + # Collect file size groups for smart mode (thread-safe) + if effective_mode == "smart": + file_size = len(content_text) + with file_size_groups_lock: + if file_size not in file_size_groups: + file_size_groups[file_size] = [] + file_size_groups[file_size].append(file_path) + + # Collect sample texts (thread-safe) + with sample_texts_lock: + if len(sample_texts) < 5: + sample_texts.append(content_text[:1000]) + + # Ensure chapter_num is always an integer + if isinstance(chapter_num, float): + chapter_num = int(chapter_num) + + # Create chapter info + chapter_info = { + "num": chapter_num, # Now guaranteed to have a value + "title": chapter_title or f"Chapter {chapter_num}", + "body": content_html, + "filename": file_path, + "original_filename": os.path.basename(file_path), + "original_basename": os.path.splitext(os.path.basename(file_path))[0], + "content_hash": content_hash, + "detection_method": detection_method if detection_method else "pending", + "file_size": len(content_text), + "has_images": has_images, + "image_count": len(images), + "is_empty": len(content_text.strip()) == 0, + "is_image_only": is_image_only_chapter, + "extraction_mode": extraction_mode, + "file_index": file_index # Store original file index for sorting + } + + # Add enhanced extraction info if used + if enhanced_extraction_used: + chapter_info["enhanced_extraction"] = True + chapter_info["enhanced_filtering"] = enhanced_filtering + chapter_info["preserve_structure"] = preserve_structure + + # Add merge info if applicable + if not disable_merging and file_path in merge_candidates: + chapter_info["was_merged"] = True + chapter_info["merged_with"] = merge_candidates[file_path] + + if effective_mode == "smart": + chapter_info["language_sample"] = content_text[:500] + # Debug for section files + if 'section' in chapter_info['original_basename'].lower(): + print(f"[DEBUG] Added section file to candidates: {chapter_info['original_basename']} (size: {chapter_info['file_size']})") + + return chapter_info + + except Exception as e: + print(f"[ERROR] Failed to process {file_path}: {e}") + import traceback + traceback.print_exc() + return None + + # Process files in parallel or sequentially based on file count + print(f"🚀 Processing {len(files_to_process)} HTML files...") + + # Initial progress + if self.progress_callback: + self.progress_callback(f"Processing {len(files_to_process)} chapters...") + + candidate_chapters = [] # For smart mode + chapters_direct = [] # For other modes + + # Decide whether to use parallel processing + use_parallel = len(files_to_process) > 10 + + if use_parallel: + print("📦 Using parallel processing for better performance...") + + # Process files in parallel + with ThreadPoolExecutor(max_workers=min(8, os.cpu_count() or 4)) as executor: + # Submit all files for processing + future_to_file = { + executor.submit(process_single_html_file, file_path, idx): (file_path, idx) + for idx, file_path in enumerate(files_to_process) + } + + # Collect results as they complete + for future in as_completed(future_to_file): + if is_stop_requested(): + print("❌ Chapter processing stopped by user") + executor.shutdown(wait=False) + return [], 'unknown' + + try: + chapter_info = future.result() + if chapter_info: + effective_mode = enhanced_filtering if extraction_mode == "enhanced" else extraction_mode + + # For smart mode when merging is enabled, collect candidates + # Otherwise, add directly to chapters + if effective_mode == "smart" and not disable_merging: + candidate_chapters.append(chapter_info) + else: + chapters_direct.append(chapter_info) + except Exception as e: + file_path, idx = future_to_file[future] + print(f"[ERROR] Thread error processing {file_path}: {e}") + else: + print("📦 Using sequential processing (small file count)...") + + # Process files sequentially for small EPUBs + for idx, file_path in enumerate(files_to_process): + if is_stop_requested(): + print("❌ Chapter processing stopped by user") + return [], 'unknown' + + chapter_info = process_single_html_file(file_path, idx) + if chapter_info: + effective_mode = enhanced_filtering if extraction_mode == "enhanced" else extraction_mode + + # For smart mode when merging is enabled, collect candidates + # Otherwise, add directly to chapters + if effective_mode == "smart" and not disable_merging: + candidate_chapters.append(chapter_info) + else: + chapters_direct.append(chapter_info) + + # Final progress update + if self.progress_callback: + self.progress_callback(f"Chapter processing complete: {len(candidate_chapters) + len(chapters_direct)} chapters") + + # Sort direct chapters by file index to maintain order + chapters_direct.sort(key=lambda x: x["file_index"]) + + # Post-process smart mode candidates (only when merging is enabled) + effective_mode = enhanced_filtering if extraction_mode == "enhanced" else extraction_mode + if effective_mode == "smart" and candidate_chapters and not disable_merging: + # Check stop before post-processing + if is_stop_requested(): + print("❌ Chapter post-processing stopped by user") + return chapters, 'unknown' + + print(f"\n[SMART MODE] Processing {len(candidate_chapters)} candidate files...") + + # Sort candidates by file index to maintain order + candidate_chapters.sort(key=lambda x: x["file_index"]) + + # Debug: Show what files we have + section_files = [c for c in candidate_chapters if 'section' in c['original_basename'].lower()] + chapter_files = [c for c in candidate_chapters if 'chapter' in c['original_basename'].lower() and 'section' not in c['original_basename'].lower()] + other_files = [c for c in candidate_chapters if c not in section_files and c not in chapter_files] + + print(f" 📊 File breakdown:") + print(f" • Section files: {len(section_files)}") + print(f" • Chapter files: {len(chapter_files)}") + print(f" • Other files: {len(other_files)}") + + # Original smart mode logic when merging is enabled + # First, separate files with detected chapter numbers from those without + numbered_chapters = [] + unnumbered_chapters = [] + + for idx, chapter in enumerate(candidate_chapters): + # Yield periodically during categorization + if idx % 10 == 0 and idx > 0: + time.sleep(0.001) + + if chapter["num"] is not None: + numbered_chapters.append(chapter) + else: + unnumbered_chapters.append(chapter) + + print(f" • Files with chapter numbers: {len(numbered_chapters)}") + print(f" • Files without chapter numbers: {len(unnumbered_chapters)}") + + # Check if we have hash-based filenames (no numbered chapters found) + if not numbered_chapters and unnumbered_chapters: + print(" ⚠️ No chapter numbers found - likely hash-based filenames") + print(" → Using file order as chapter sequence") + + # Sort by file index to maintain order + unnumbered_chapters.sort(key=lambda x: x["file_index"]) + + # Assign sequential numbers + for i, chapter in enumerate(unnumbered_chapters, 1): + chapter["num"] = i + chapter["detection_method"] = f"{extraction_mode}_hash_filename_sequential" if extraction_mode == "enhanced" else "hash_filename_sequential" + if not chapter["title"] or chapter["title"] == chapter["original_basename"]: + chapter["title"] = f"Chapter {i}" + + chapters = unnumbered_chapters + else: + # We have some numbered chapters + chapters = numbered_chapters + + # For unnumbered files, check if they might be duplicates or appendices + if unnumbered_chapters: + print(f" → Analyzing {len(unnumbered_chapters)} unnumbered files...") + + # Get the max chapter number + max_num = max(c["num"] for c in numbered_chapters) + + # Check each unnumbered file + for chapter in unnumbered_chapters: + # Check stop in post-processing loop + if is_stop_requested(): + print("❌ Chapter post-processing stopped by user") + return chapters, 'unknown' + + # Check if it's very small (might be a separator or note) + if chapter["file_size"] < 200: + print(f" [SKIP] Very small file: {chapter['filename']} ({chapter['file_size']} chars)") + continue + + # Check if it has similar size to existing chapters (might be duplicate) + size = chapter["file_size"] + similar_chapters = [c for c in numbered_chapters + if abs(c["file_size"] - size) < 50] + + if similar_chapters: + # Might be a duplicate, skip it + print(f" [SKIP] Possible duplicate: {chapter['filename']} (similar size to {len(similar_chapters)} chapters)") + continue + + # Otherwise, add as appendix + max_num += 1 + chapter["num"] = max_num + chapter["detection_method"] = f"{extraction_mode}_appendix_sequential" if extraction_mode == "enhanced" else "appendix_sequential" + if not chapter["title"] or chapter["title"] == chapter["original_basename"]: + chapter["title"] = f"Appendix {max_num}" + chapters.append(chapter) + print(f" [ADD] Added as chapter {max_num}: {chapter['filename']}") + else: + # For other modes or smart mode with merging disabled + chapters = chapters_direct + + # Sort chapters by number + chapters.sort(key=lambda x: x["num"]) + + # Ensure chapter numbers are integers + # When merging is disabled, all chapters should have integer numbers anyway + for chapter in chapters: + if isinstance(chapter["num"], float): + chapter["num"] = int(chapter["num"]) + + # Final validation + if chapters: + print(f"\n✅ Final chapter count: {len(chapters)}") + print(f" • Chapter range: {chapters[0]['num']} - {chapters[-1]['num']}") + + # Enhanced mode summary + if extraction_mode == "enhanced": + enhanced_count = sum(1 for c in chapters if c.get('enhanced_extraction', False)) + print(f" 🚀 Enhanced extraction used: {enhanced_count}/{len(chapters)} chapters") + + # Check for gaps + chapter_nums = [c["num"] for c in chapters] + expected_nums = list(range(min(chapter_nums), max(chapter_nums) + 1)) + missing = set(expected_nums) - set(chapter_nums) + if missing: + print(f" ⚠️ Missing chapter numbers: {sorted(missing)}") + + # Language detection + combined_sample = ' '.join(sample_texts) if effective_mode == "smart" else '' + detected_language = self._detect_content_language(combined_sample) if combined_sample else 'unknown' + + if chapters: + self._print_extraction_summary(chapters, detected_language, extraction_mode, + h1_count if effective_mode == "smart" else 0, + h2_count if effective_mode == "smart" else 0, + file_size_groups if effective_mode == "smart" else {}) + + return chapters, detected_language + + def _extract_chapter_info(self, soup, file_path, content_text, html_content): + """Extract chapter number and title from various sources with parallel pattern matching""" + chapter_num = None + chapter_title = None + detection_method = None + + # SPECIAL HANDLING: When we have Section/Chapter pairs, differentiate them + filename = os.path.basename(file_path) + + # Handle different naming patterns for Section/Chapter files + if ('section' in filename.lower() or '_section' in filename.lower()) and 'chapter' not in filename.lower(): + # For Section files, add 0.1 to the base number + # Try different patterns + match = re.search(r'No(\d+)', filename) + if not match: + match = re.search(r'^(\d+)[_\-]', filename) + if not match: + match = re.search(r'^(\d+)', filename) + + if match: + base_num = int(match.group(1)) + chapter_num = base_num + 0.1 # Section gets .1 + detection_method = "filename_section_special" + + elif ('chapter' in filename.lower() or '_chapter' in filename.lower()) and 'section' not in filename.lower(): + # For Chapter files, use the base number + # Try different patterns + match = re.search(r'No(\d+)', filename) + if not match: + match = re.search(r'^(\d+)[_\-]', filename) + if not match: + match = re.search(r'^(\d+)', filename) + + if match: + chapter_num = int(match.group(1)) + detection_method = "filename_chapter_special" + + # If not handled by special logic, continue with normal extraction + if not chapter_num: + # Try filename first - use parallel pattern matching for better performance + chapter_patterns = [(pattern, flags, method) for pattern, flags, method in self.pattern_manager.CHAPTER_PATTERNS + if method.endswith('_number')] + + if len(chapter_patterns) > 3: # Only parallelize if we have enough patterns + # Parallel pattern matching for filename + with ThreadPoolExecutor(max_workers=min(4, len(chapter_patterns))) as executor: + def try_pattern(pattern_info): + pattern, flags, method = pattern_info + match = re.search(pattern, file_path, flags) + if match: + try: + num_str = match.group(1) + if num_str.isdigit(): + return int(num_str), f"filename_{method}" + elif method == 'chinese_chapter_cn': + converted = self._convert_chinese_number(num_str) + if converted: + return converted, f"filename_{method}" + except (ValueError, IndexError): + pass + return None, None + + # Submit all patterns + futures = [executor.submit(try_pattern, pattern_info) for pattern_info in chapter_patterns] + + # Check results as they complete + for future in as_completed(futures): + try: + num, method = future.result() + if num: + chapter_num = num + detection_method = method + # Cancel remaining futures + for f in futures: + f.cancel() + break + except Exception: + continue + else: + # Sequential processing for small pattern sets + for pattern, flags, method in chapter_patterns: + match = re.search(pattern, file_path, flags) + if match: + try: + num_str = match.group(1) + if num_str.isdigit(): + chapter_num = int(num_str) + detection_method = f"filename_{method}" + break + elif method == 'chinese_chapter_cn': + converted = self._convert_chinese_number(num_str) + if converted: + chapter_num = converted + detection_method = f"filename_{method}" + break + except (ValueError, IndexError): + continue + + # Try content if not found in filename + if not chapter_num: + # Check ignore settings for batch translation + batch_translate_active = os.getenv('BATCH_TRANSLATE_HEADERS', '0') == '1' + ignore_title_tag = os.getenv('IGNORE_TITLE', '0') == '1' and batch_translate_active + ignore_header_tags = os.getenv('IGNORE_HEADER', '0') == '1' and batch_translate_active + + # Prepare all text sources to check in parallel + text_sources = [] + + # Add title tag if not ignored + if not ignore_title_tag and soup.title and soup.title.string: + title_text = soup.title.string.strip() + text_sources.append(("title", title_text, True)) # True means this can be chapter_title + + # Add headers if not ignored + if not ignore_header_tags: + for header_tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']: + headers = soup.find_all(header_tag) + for header in headers[:3]: # Limit to first 3 of each type + header_text = header.get_text(strip=True) + if header_text: + text_sources.append((f"header_{header_tag}", header_text, True)) + + # Add first paragraphs + first_elements = soup.find_all(['p', 'div'])[:5] + for elem in first_elements: + elem_text = elem.get_text(strip=True) + if elem_text: + text_sources.append(("content", elem_text, False)) # False means don't use as chapter_title + + # Process text sources in parallel if we have many + if len(text_sources) > 5: + with ThreadPoolExecutor(max_workers=min(6, len(text_sources))) as executor: + def extract_from_source(source_info): + source_type, text, can_be_title = source_info + num, method = self._extract_from_text(text, source_type) + return num, method, text if (num and can_be_title) else None + + # Submit all text sources + future_to_source = {executor.submit(extract_from_source, source): source + for source in text_sources} + + # Process results as they complete + for future in as_completed(future_to_source): + try: + num, method, title = future.result() + if num: + chapter_num = num + detection_method = method + if title and not chapter_title: + chapter_title = title + # Cancel remaining futures + for f in future_to_source: + f.cancel() + break + except Exception: + continue + else: + # Sequential processing for small text sets + for source_type, text, can_be_title in text_sources: + num, method = self._extract_from_text(text, source_type) + if num: + chapter_num = num + detection_method = method + if can_be_title and not chapter_title: + chapter_title = text + break + + # Final fallback to filename patterns + if not chapter_num: + filename_base = os.path.basename(file_path) + # Parallel pattern matching for filename extraction + if len(self.pattern_manager.FILENAME_EXTRACT_PATTERNS) > 3: + with ThreadPoolExecutor(max_workers=min(4, len(self.pattern_manager.FILENAME_EXTRACT_PATTERNS))) as executor: + def try_filename_pattern(pattern): + match = re.search(pattern, filename_base, re.IGNORECASE) + if match: + try: + return int(match.group(1)) + except (ValueError, IndexError): + pass + return None + + futures = [executor.submit(try_filename_pattern, pattern) + for pattern in self.pattern_manager.FILENAME_EXTRACT_PATTERNS] + + for future in as_completed(futures): + try: + num = future.result() + if num: + chapter_num = num + detection_method = "filename_number" + for f in futures: + f.cancel() + break + except Exception: + continue + else: + # Sequential for small pattern sets + for pattern in self.pattern_manager.FILENAME_EXTRACT_PATTERNS: + match = re.search(pattern, filename_base, re.IGNORECASE) + if match: + chapter_num = int(match.group(1)) + detection_method = "filename_number" + break + + # Extract title if not already found (with ignore settings support) + if not chapter_title: + # Check ignore settings for batch translation + batch_translate_active = os.getenv('BATCH_TRANSLATE_HEADERS', '0') == '1' + ignore_title_tag = os.getenv('IGNORE_TITLE', '0') == '1' and batch_translate_active + ignore_header_tags = os.getenv('IGNORE_HEADER', '0') == '1' and batch_translate_active + + # Try title tag if not ignored + if not ignore_title_tag and soup.title and soup.title.string: + chapter_title = soup.title.string.strip() + + # Try header tags if not ignored and no title found + if not chapter_title and not ignore_header_tags: + for header_tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']: + header = soup.find(header_tag) + if header: + chapter_title = header.get_text(strip=True) + break + + # Final fallback + if not chapter_title: + chapter_title = f"Chapter {chapter_num}" if chapter_num else None + + chapter_title = re.sub(r'\s+', ' ', chapter_title).strip() if chapter_title else None + + return chapter_num, chapter_title, detection_method + + + def _extract_from_text(self, text, source_type): + """Extract chapter number from text using patterns with parallel matching for large pattern sets""" + # Get patterns that don't end with '_number' + text_patterns = [(pattern, flags, method) for pattern, flags, method in self.pattern_manager.CHAPTER_PATTERNS + if not method.endswith('_number')] + + # Only use parallel processing if we have many patterns + if len(text_patterns) > 5: + with ThreadPoolExecutor(max_workers=min(4, len(text_patterns))) as executor: + def try_text_pattern(pattern_info): + pattern, flags, method = pattern_info + match = re.search(pattern, text, flags) + if match: + try: + num_str = match.group(1) + if num_str.isdigit(): + return int(num_str), f"{source_type}_{method}" + elif method == 'chinese_chapter_cn': + converted = self._convert_chinese_number(num_str) + if converted: + return converted, f"{source_type}_{method}" + except (ValueError, IndexError): + pass + return None, None + + # Submit all patterns + futures = [executor.submit(try_text_pattern, pattern_info) for pattern_info in text_patterns] + + # Check results as they complete + for future in as_completed(futures): + try: + num, method = future.result() + if num: + # Cancel remaining futures + for f in futures: + f.cancel() + return num, method + except Exception: + continue + else: + # Sequential processing for small pattern sets + for pattern, flags, method in text_patterns: + match = re.search(pattern, text, flags) + if match: + try: + num_str = match.group(1) + if num_str.isdigit(): + return int(num_str), f"{source_type}_{method}" + elif method == 'chinese_chapter_cn': + converted = self._convert_chinese_number(num_str) + if converted: + return converted, f"{source_type}_{method}" + except (ValueError, IndexError): + continue + + return None, None + + def _convert_chinese_number(self, cn_num): + """Convert Chinese number to integer""" + if cn_num in self.pattern_manager.CHINESE_NUMS: + return self.pattern_manager.CHINESE_NUMS[cn_num] + + if '十' in cn_num: + parts = cn_num.split('十') + if len(parts) == 2: + tens = self.pattern_manager.CHINESE_NUMS.get(parts[0], 1) if parts[0] else 1 + ones = self.pattern_manager.CHINESE_NUMS.get(parts[1], 0) if parts[1] else 0 + return tens * 10 + ones + + return None + + def _detect_content_language(self, text_sample): + """Detect the primary language of content with parallel processing for large texts""" + + # For very short texts, use sequential processing + if len(text_sample) < 1000: + scripts = { + 'korean': 0, + 'japanese_hiragana': 0, + 'japanese_katakana': 0, + 'chinese': 0, + 'latin': 0 + } + + for char in text_sample: + code = ord(char) + if 0xAC00 <= code <= 0xD7AF: + scripts['korean'] += 1 + elif 0x3040 <= code <= 0x309F: + scripts['japanese_hiragana'] += 1 + elif 0x30A0 <= code <= 0x30FF: + scripts['japanese_katakana'] += 1 + elif 0x4E00 <= code <= 0x9FFF: + scripts['chinese'] += 1 + elif 0x0020 <= code <= 0x007F: + scripts['latin'] += 1 + else: + # For longer texts, use parallel processing + # Split text into chunks for parallel processing + chunk_size = max(500, len(text_sample) // (os.cpu_count() or 4)) + chunks = [text_sample[i:i + chunk_size] for i in range(0, len(text_sample), chunk_size)] + + # Thread-safe accumulator + scripts_lock = threading.Lock() + scripts = { + 'korean': 0, + 'japanese_hiragana': 0, + 'japanese_katakana': 0, + 'chinese': 0, + 'latin': 0 + } + + def process_chunk(text_chunk): + """Process a chunk of text and return script counts""" + local_scripts = { + 'korean': 0, + 'japanese_hiragana': 0, + 'japanese_katakana': 0, + 'chinese': 0, + 'latin': 0 + } + + for char in text_chunk: + code = ord(char) + if 0xAC00 <= code <= 0xD7AF: + local_scripts['korean'] += 1 + elif 0x3040 <= code <= 0x309F: + local_scripts['japanese_hiragana'] += 1 + elif 0x30A0 <= code <= 0x30FF: + local_scripts['japanese_katakana'] += 1 + elif 0x4E00 <= code <= 0x9FFF: + local_scripts['chinese'] += 1 + elif 0x0020 <= code <= 0x007F: + local_scripts['latin'] += 1 + + return local_scripts + + # Process chunks in parallel + with ThreadPoolExecutor(max_workers=min(os.cpu_count() or 4, len(chunks))) as executor: + # Submit all chunks + futures = [executor.submit(process_chunk, chunk) for chunk in chunks] + + # Collect results + for future in as_completed(futures): + try: + chunk_scripts = future.result() + # Thread-safe accumulation + with scripts_lock: + for script, count in chunk_scripts.items(): + scripts[script] += count + except Exception as e: + print(f"[WARNING] Error processing chunk in language detection: {e}") + + # Language determination logic (same as original) + total_cjk = scripts['korean'] + scripts['japanese_hiragana'] + scripts['japanese_katakana'] + scripts['chinese'] + + if scripts['korean'] > total_cjk * 0.3: + return 'korean' + elif scripts['japanese_hiragana'] + scripts['japanese_katakana'] > total_cjk * 0.2: + return 'japanese' + elif scripts['chinese'] > total_cjk * 0.3: + return 'chinese' + elif scripts['latin'] > len(text_sample) * 0.7: + return 'english' + else: + return 'unknown' + + def _print_extraction_summary(self, chapters, detected_language, extraction_mode, h1_count, h2_count, file_size_groups): + """Print extraction summary""" + print(f"\n📊 Chapter Extraction Summary ({extraction_mode.capitalize()} Mode):") + print(f" • Total chapters extracted: {len(chapters)}") + + # Format chapter range handling both int and float + first_num = chapters[0]['num'] + last_num = chapters[-1]['num'] + + print(f" • Chapter range: {first_num} to {last_num}") + print(f" • Detected language: {detected_language}") + + if extraction_mode == "smart": + print(f" • Primary header type: {'
' if h2_count > h1_count else ''}")
+
+ image_only_count = sum(1 for c in chapters if c.get('is_image_only', False))
+ text_only_count = sum(1 for c in chapters if not c.get('has_images', False) and c.get('file_size', 0) >= 500)
+ mixed_count = sum(1 for c in chapters if c.get('has_images', False) and c.get('file_size', 0) >= 500)
+ empty_count = sum(1 for c in chapters if c.get('file_size', 0) < 50)
+
+ print(f" • Text-only chapters: {text_only_count}")
+ print(f" • Image-only chapters: {image_only_count}")
+ print(f" • Mixed content chapters: {mixed_count}")
+ print(f" • Empty/minimal content: {empty_count}")
+
+ # Check for merged chapters
+ merged_count = sum(1 for c in chapters if c.get('was_merged', False))
+ if merged_count > 0:
+ print(f" • Merged chapters: {merged_count}")
+
+ # Check for missing chapters (only for integer sequences)
+ expected_chapters = set(range(chapters[0]['num'], chapters[-1]['num'] + 1))
+ actual_chapters = set(c['num'] for c in chapters)
+ missing = expected_chapters - actual_chapters
+ if missing:
+ print(f" ⚠️ Missing chapter numbers: {sorted(missing)}")
+
+ if extraction_mode == "smart":
+ method_stats = Counter(c['detection_method'] for c in chapters)
+ print(f" 📈 Detection methods used:")
+ for method, count in method_stats.most_common():
+ print(f" • {method}: {count} chapters")
+
+ large_groups = [size for size, files in file_size_groups.items() if len(files) > 1]
+ if large_groups:
+ print(f" ⚠️ Found {len(large_groups)} file size groups with potential duplicates")
+ else:
+ print(f" • Empty/placeholder: {empty_count}")
+
+ if extraction_mode == "full":
+ print(f" 🔍 Full extraction preserved all HTML structure and tags")
+
+ def _extract_epub_metadata(self, zf):
+ """Extract comprehensive metadata from EPUB file including all custom fields"""
+ meta = {}
+ # Use lxml for XML if available
+ xml_parser = 'lxml-xml' if self.parser == 'lxml' else 'xml'
+ try:
+ for name in zf.namelist():
+ if name.lower().endswith('.opf'):
+ opf_content = zf.read(name)
+ soup = BeautifulSoup(opf_content, xml_parser)
+
+ # Extract ALL Dublin Core elements (expanded list)
+ dc_elements = ['title', 'creator', 'subject', 'description',
+ 'publisher', 'contributor', 'date', 'type',
+ 'format', 'identifier', 'source', 'language',
+ 'relation', 'coverage', 'rights']
+
+ for element in dc_elements:
+ tag = soup.find(element)
+ if tag and tag.get_text(strip=True):
+ meta[element] = tag.get_text(strip=True)
+
+ # Extract ALL meta tags (not just series)
+ meta_tags = soup.find_all('meta')
+ for meta_tag in meta_tags:
+ # Try different attribute names for the metadata name
+ name = meta_tag.get('name') or meta_tag.get('property', '')
+ content = meta_tag.get('content', '')
+
+ if name and content:
+ # Store original name for debugging
+ original_name = name
+
+ # Clean up common prefixes
+ if name.startswith('calibre:'):
+ name = name[8:] # Remove 'calibre:' prefix
+ elif name.startswith('dc:'):
+ name = name[3:] # Remove 'dc:' prefix
+ elif name.startswith('opf:'):
+ name = name[4:] # Remove 'opf:' prefix
+
+ # Normalize the field name - replace hyphens with underscores
+ name = name.replace('-', '_')
+
+ # Don't overwrite if already exists (prefer direct tags over meta tags)
+ if name not in meta:
+ meta[name] = content
+
+ # Debug output for custom fields
+ if original_name != name:
+ print(f" • Found custom field: {original_name} → {name}")
+
+ # Special handling for series information (maintain compatibility)
+ if 'series' not in meta:
+ series_tags = soup.find_all('meta', attrs={'name': lambda x: x and 'series' in x.lower()})
+ for series_tag in series_tags:
+ series_name = series_tag.get('content', '')
+ if series_name:
+ meta['series'] = series_name
+ break
+
+ # Extract refines metadata (used by some EPUB creators)
+ refines_metas = soup.find_all('meta', attrs={'refines': True})
+ for refine in refines_metas:
+ property_name = refine.get('property', '')
+ content = refine.get_text(strip=True) or refine.get('content', '')
+
+ if property_name and content:
+ # Clean property name
+ if ':' in property_name:
+ property_name = property_name.split(':')[-1]
+ property_name = property_name.replace('-', '_')
+
+ if property_name not in meta:
+ meta[property_name] = content
+
+ # Log extraction summary
+ print(f"📋 Extracted {len(meta)} metadata fields")
+
+ # Show standard vs custom fields
+ standard_keys = {'title', 'creator', 'language', 'subject', 'description',
+ 'publisher', 'date', 'identifier', 'source', 'rights',
+ 'contributor', 'type', 'format', 'relation', 'coverage'}
+ custom_keys = set(meta.keys()) - standard_keys
+
+ if custom_keys:
+ print(f"📋 Standard fields: {len(standard_keys & set(meta.keys()))}")
+ print(f"📋 Custom fields found: {sorted(custom_keys)}")
+
+ # Show sample values for custom fields (truncated)
+ for key in sorted(custom_keys)[:5]: # Show first 5 custom fields
+ value = str(meta[key])
+ if len(value) > 50:
+ value = value[:47] + "..."
+ print(f" • {key}: {value}")
+
+ if len(custom_keys) > 5:
+ print(f" • ... and {len(custom_keys) - 5} more custom fields")
+
+ break
+
+ except Exception as e:
+ print(f"[WARNING] Failed to extract metadata: {e}")
+ import traceback
+ traceback.print_exc()
+
+ return meta
+
+ def _categorize_resource(self, file_path, file_name):
+ """Categorize a file and return (resource_type, target_dir, safe_filename)"""
+ file_path_lower = file_path.lower()
+ file_name_lower = file_name.lower()
+
+ if file_path_lower.endswith('.css'):
+ return 'css', 'css', sanitize_resource_filename(file_name)
+ elif file_path_lower.endswith(('.ttf', '.otf', '.woff', '.woff2', '.eot')):
+ return 'fonts', 'fonts', sanitize_resource_filename(file_name)
+ elif file_path_lower.endswith(('.jpg', '.jpeg', '.png', '.gif', '.svg', '.bmp', '.webp')):
+ return 'images', 'images', sanitize_resource_filename(file_name)
+ elif (file_path_lower.endswith(('.opf', '.ncx')) or
+ file_name_lower == 'container.xml' or
+ 'container.xml' in file_path_lower):
+ if 'container.xml' in file_path_lower:
+ safe_filename = 'container.xml'
+ else:
+ safe_filename = file_name
+ return 'epub_structure', None, safe_filename
+ elif file_path_lower.endswith(('.js', '.xml', '.txt')):
+ return 'other', None, sanitize_resource_filename(file_name)
+
+ return None
+
+ def _cleanup_old_resources(self, output_dir):
+ """Clean up old resource directories and EPUB structure files"""
+ print("🧹 Cleaning up any existing resource directories...")
+
+ cleanup_success = True
+
+ for resource_type in ['css', 'fonts', 'images']:
+ resource_dir = os.path.join(output_dir, resource_type)
+ if os.path.exists(resource_dir):
+ try:
+ shutil.rmtree(resource_dir)
+ print(f" 🗑️ Removed old {resource_type} directory")
+ except PermissionError as e:
+ print(f" ⚠️ Cannot remove {resource_type} directory (permission denied) - will merge with existing files")
+ cleanup_success = False
+ except Exception as e:
+ print(f" ⚠️ Error removing {resource_type} directory: {e} - will merge with existing files")
+ cleanup_success = False
+
+ epub_structure_files = ['container.xml', 'content.opf', 'toc.ncx']
+ for epub_file in epub_structure_files:
+ input_path = os.path.join(output_dir, epub_file)
+ if os.path.exists(input_path):
+ try:
+ os.remove(input_path)
+ print(f" 🗑️ Removed old {epub_file}")
+ except PermissionError:
+ print(f" ⚠️ Cannot remove {epub_file} (permission denied) - will use existing file")
+ except Exception as e:
+ print(f" ⚠️ Error removing {epub_file}: {e}")
+
+ try:
+ for file in os.listdir(output_dir):
+ if file.lower().endswith(('.opf', '.ncx')):
+ file_path = os.path.join(output_dir, file)
+ try:
+ os.remove(file_path)
+ print(f" 🗑️ Removed old EPUB file: {file}")
+ except PermissionError:
+ print(f" ⚠️ Cannot remove {file} (permission denied)")
+ except Exception as e:
+ print(f" ⚠️ Error removing {file}: {e}")
+ except Exception as e:
+ print(f"⚠️ Error scanning for EPUB files: {e}")
+
+ if not cleanup_success:
+ print("⚠️ Some cleanup operations failed due to file permissions")
+ print(" The program will continue and merge with existing files")
+
+ return cleanup_success
+
+ def _count_existing_resources(self, output_dir, extracted_resources):
+ """Count existing resources when skipping extraction"""
+ for resource_type in ['css', 'fonts', 'images', 'epub_structure']:
+ if resource_type == 'epub_structure':
+ epub_files = []
+ for file in ['container.xml', 'content.opf', 'toc.ncx']:
+ if os.path.exists(os.path.join(output_dir, file)):
+ epub_files.append(file)
+ try:
+ for file in os.listdir(output_dir):
+ if file.lower().endswith(('.opf', '.ncx')) and file not in epub_files:
+ epub_files.append(file)
+ except:
+ pass
+ extracted_resources[resource_type] = epub_files
+ else:
+ resource_dir = os.path.join(output_dir, resource_type)
+ if os.path.exists(resource_dir):
+ try:
+ files = [f for f in os.listdir(resource_dir) if os.path.isfile(os.path.join(resource_dir, f))]
+ extracted_resources[resource_type] = files
+ except:
+ extracted_resources[resource_type] = []
+
+ total_existing = sum(len(files) for files in extracted_resources.values())
+ print(f"✅ Found {total_existing} existing resource files")
+ return extracted_resources
+
+ def _validate_critical_files(self, output_dir, extracted_resources):
+ """Validate that critical EPUB files were extracted"""
+ total_extracted = sum(len(files) for files in extracted_resources.values())
+ print(f"✅ Extracted {total_extracted} resource files:")
+
+ for resource_type, files in extracted_resources.items():
+ if files:
+ if resource_type == 'epub_structure':
+ print(f" • EPUB Structure: {len(files)} files")
+ for file in files:
+ print(f" - {file}")
+ else:
+ print(f" • {resource_type.title()}: {len(files)} files")
+
+ critical_files = ['container.xml']
+ missing_critical = [f for f in critical_files if not os.path.exists(os.path.join(output_dir, f))]
+
+ if missing_critical:
+ print(f"⚠️ WARNING: Missing critical EPUB files: {missing_critical}")
+ print(" This may prevent proper EPUB reconstruction!")
+ else:
+ print("✅ All critical EPUB structure files extracted successfully")
+
+ opf_files = [f for f in extracted_resources['epub_structure'] if f.lower().endswith('.opf')]
+ if not opf_files:
+ print("⚠️ WARNING: No OPF file found! This will prevent EPUB reconstruction.")
+ else:
+ print(f"✅ Found OPF file(s): {opf_files}")
+
+ def _create_extraction_report(self, output_dir, metadata, chapters, extracted_resources):
+ """Create comprehensive extraction report with HTML file tracking"""
+ report_path = os.path.join(output_dir, 'extraction_report.txt')
+ with open(report_path, 'w', encoding='utf-8') as f:
+ f.write("EPUB Extraction Report\n")
+ f.write("=" * 50 + "\n\n")
+
+ f.write(f"EXTRACTION MODE: {metadata.get('extraction_mode', 'unknown').upper()}\n\n")
+
+ f.write("METADATA:\n")
+ for key, value in metadata.items():
+ if key not in ['chapter_titles', 'extracted_resources', 'extraction_mode']:
+ f.write(f" {key}: {value}\n")
+
+ f.write(f"\nCHAPTERS ({len(chapters)}):\n")
+
+ text_chapters = []
+ image_only_chapters = []
+ mixed_chapters = []
+
+ for chapter in chapters:
+ if chapter.get('has_images') and chapter.get('file_size', 0) < 500:
+ image_only_chapters.append(chapter)
+ elif chapter.get('has_images') and chapter.get('file_size', 0) >= 500:
+ mixed_chapters.append(chapter)
+ else:
+ text_chapters.append(chapter)
+
+ if text_chapters:
+ f.write(f"\n TEXT CHAPTERS ({len(text_chapters)}):\n")
+ for c in text_chapters:
+ f.write(f" {c['num']:3d}. {c['title']} ({c['detection_method']})\n")
+ if c.get('original_html_file'):
+ f.write(f" → {c['original_html_file']}\n")
+
+ if image_only_chapters:
+ f.write(f"\n IMAGE-ONLY CHAPTERS ({len(image_only_chapters)}):\n")
+ for c in image_only_chapters:
+ f.write(f" {c['num']:3d}. {c['title']} (images: {c.get('image_count', 0)})\n")
+ if c.get('original_html_file'):
+ f.write(f" → {c['original_html_file']}\n")
+ if 'body' in c:
+ try:
+ soup = BeautifulSoup(c['body'], 'html.parser')
+ images = soup.find_all('img')
+ for img in images[:3]:
+ src = img.get('src', 'unknown')
+ f.write(f" • Image: {src}\n")
+ if len(images) > 3:
+ f.write(f" • ... and {len(images) - 3} more images\n")
+ except:
+ pass
+
+ if mixed_chapters:
+ f.write(f"\n MIXED CONTENT CHAPTERS ({len(mixed_chapters)}):\n")
+ for c in mixed_chapters:
+ f.write(f" {c['num']:3d}. {c['title']} (text: {c.get('file_size', 0)} chars, images: {c.get('image_count', 0)})\n")
+ if c.get('original_html_file'):
+ f.write(f" → {c['original_html_file']}\n")
+
+ f.write(f"\nRESOURCES EXTRACTED:\n")
+ for resource_type, files in extracted_resources.items():
+ if files:
+ if resource_type == 'epub_structure':
+ f.write(f" EPUB Structure: {len(files)} files\n")
+ for file in files:
+ f.write(f" - {file}\n")
+ else:
+ f.write(f" {resource_type.title()}: {len(files)} files\n")
+ for file in files[:5]:
+ f.write(f" - {file}\n")
+ if len(files) > 5:
+ f.write(f" ... and {len(files) - 5} more\n")
+
+ f.write(f"\nHTML FILES WRITTEN:\n")
+ html_files_written = metadata.get('html_files_written', 0)
+ f.write(f" Total: {html_files_written} files\n")
+ f.write(f" Location: Main directory and 'originals' subdirectory\n")
+
+ f.write(f"\nPOTENTIAL ISSUES:\n")
+ issues = []
+
+ if image_only_chapters:
+ issues.append(f" • {len(image_only_chapters)} chapters contain only images (may need OCR)")
+
+ missing_html = sum(1 for c in chapters if not c.get('original_html_file'))
+ if missing_html > 0:
+ issues.append(f" • {missing_html} chapters failed to write HTML files")
+
+ if not extracted_resources.get('epub_structure'):
+ issues.append(" • No EPUB structure files found (may affect reconstruction)")
+
+ if not issues:
+ f.write(" None detected - extraction appears successful!\n")
+ else:
+ for issue in issues:
+ f.write(issue + "\n")
+
+ print(f"📄 Saved extraction report to: {report_path}")
+
+ def _log_extraction_summary(self, chapters, extracted_resources, detected_language, html_files_written=0):
+ """Log final extraction summary with HTML file information"""
+ extraction_mode = chapters[0].get('extraction_mode', 'unknown') if chapters else 'unknown'
+
+ print(f"\n✅ {extraction_mode.capitalize()} extraction complete!")
+ print(f" 📚 Chapters: {len(chapters)}")
+ print(f" 📄 HTML files written: {html_files_written}")
+ print(f" 🎨 Resources: {sum(len(files) for files in extracted_resources.values())}")
+ print(f" 🌍 Language: {detected_language}")
+
+ image_only_count = sum(1 for c in chapters if c.get('has_images') and c.get('file_size', 0) < 500)
+ if image_only_count > 0:
+ print(f" 📸 Image-only chapters: {image_only_count}")
+
+ epub_files = extracted_resources.get('epub_structure', [])
+ if epub_files:
+ print(f" 📋 EPUB Structure: {len(epub_files)} files ({', '.join(epub_files)})")
+ else:
+ print(f" ⚠️ No EPUB structure files extracted!")
+
+ print(f"\n🔍 Pre-flight check readiness:")
+ print(f" ✅ HTML files: {'READY' if html_files_written > 0 else 'NOT READY'}")
+ print(f" ✅ Metadata: READY")
+ print(f" ✅ Resources: READY")
+
+# =====================================================
+# UNIFIED TRANSLATION PROCESSOR
+# =====================================================
+
+class TranslationProcessor:
+ """Handles the translation of individual chapters"""
+
+ def __init__(self, config, client, out_dir, log_callback=None, stop_callback=None, uses_zero_based=False, is_text_file=False):
+ self.config = config
+ self.client = client
+ self.out_dir = out_dir
+ self.log_callback = log_callback
+ self.stop_callback = stop_callback
+ self.chapter_splitter = ChapterSplitter(model_name=config.MODEL)
+ self.uses_zero_based = uses_zero_based
+ self.is_text_file = is_text_file
+
+ # Check and log multi-key status
+ if hasattr(self.client, 'use_multi_keys') and self.client.use_multi_keys:
+ stats = self.client.get_stats()
+ self._log(f"🔑 Multi-key mode active: {stats.get('total_keys', 0)} keys")
+ self._log(f" Active keys: {stats.get('active_keys', 0)}")
+
+ def _log(self, message):
+ """Log a message"""
+ if self.log_callback:
+ self.log_callback(message)
+ else:
+ print(message)
+
+ def report_key_status(self):
+ """Report multi-key status if available"""
+ if hasattr(self.client, 'get_stats'):
+ stats = self.client.get_stats()
+ if stats.get('multi_key_mode', False):
+ self._log(f"\n📊 API Key Status:")
+ self._log(f" Active Keys: {stats.get('active_keys', 0)}/{stats.get('total_keys', 0)}")
+ self._log(f" Success Rate: {stats.get('success_rate', 0):.1%}")
+ self._log(f" Total Requests: {stats.get('total_requests', 0)}\n")
+
+ def check_stop(self):
+ """Check if translation should stop"""
+ if self.stop_callback and self.stop_callback():
+ print("❌ Translation stopped by user request.")
+ return True
+
+ def check_duplicate_content(self, result, idx, prog, out, actual_num=None):
+ """Check if translated content is duplicate - with mode selection"""
+
+ # Get detection mode from config
+ detection_mode = getattr(self.config, 'DUPLICATE_DETECTION_MODE', 'basic')
+ print(f" 🔍 DEBUG: Detection mode = '{detection_mode}'")
+ print(f" 🔍 DEBUG: Lookback chapters = {self.config.DUPLICATE_LOOKBACK_CHAPTERS}")
+
+ # Extract content_hash if available from progress
+ content_hash = None
+ if detection_mode == 'ai-hunter':
+ # Try to get content_hash from the current chapter info
+ # Use actual_num if provided, otherwise fallback to idx+1
+ if actual_num is not None:
+ chapter_key = str(actual_num)
+ else:
+ chapter_key = str(idx + 1)
+ if chapter_key in prog.get("chapters", {}):
+ chapter_info = prog["chapters"][chapter_key]
+ content_hash = chapter_info.get("content_hash")
+ print(f" 🔍 DEBUG: Found content_hash for chapter {idx}: {content_hash}")
+
+ if detection_mode == 'ai-hunter':
+ print(" 🤖 DEBUG: Routing to AI Hunter detection...")
+ # Check if AI Hunter method is available (injected by the wrapper)
+ if hasattr(self, '_check_duplicate_ai_hunter'):
+ return self._check_duplicate_ai_hunter(result, idx, prog, out, content_hash)
+ else:
+ print(" ⚠️ AI Hunter method not available, falling back to basic detection")
+ return self._check_duplicate_basic(result, idx, prog, out)
+ elif detection_mode == 'cascading':
+ print(" 🔄 DEBUG: Routing to Cascading detection...")
+ return self._check_duplicate_cascading(result, idx, prog, out)
+ else:
+ print(" 📋 DEBUG: Routing to Basic detection...")
+ return self._check_duplicate_basic(result, idx, prog, out)
+
+ def _check_duplicate_basic(self, result, idx, prog, out):
+ """Original basic duplicate detection"""
+ try:
+ result_clean = re.sub(r'<[^>]+>', '', result).strip().lower()
+ result_sample = result_clean[:1000]
+
+ lookback_chapters = self.config.DUPLICATE_LOOKBACK_CHAPTERS
+
+ for prev_idx in range(max(0, idx - lookback_chapters), idx):
+ prev_key = str(prev_idx)
+ if prev_key in prog["chapters"] and prog["chapters"][prev_key].get("output_file"):
+ prev_file = prog["chapters"][prev_key]["output_file"]
+ prev_path = os.path.join(out, prev_file)
+
+ if os.path.exists(prev_path):
+ try:
+ with open(prev_path, 'r', encoding='utf-8') as f:
+ prev_content = f.read()
+ prev_clean = re.sub(r'<[^>]+>', '', prev_content).strip().lower()
+ prev_sample = prev_clean[:1000]
+
+ # Use SequenceMatcher for similarity comparison
+ similarity = SequenceMatcher(None, result_sample, prev_sample).ratio()
+
+ if similarity >= 0.85: # 85% threshold
+ print(f" 🚀 Basic detection: Duplicate found ({int(similarity*100)}%)")
+ return True, int(similarity * 100)
+
+ except Exception as e:
+ print(f" Warning: Failed to read {prev_path}: {e}")
+ continue
+
+ return False, 0
+
+ except Exception as e:
+ print(f" Warning: Failed to check duplicate content: {e}")
+ return False, 0
+
+
+ def _check_duplicate_cascading(self, result, idx, prog, out):
+ """Cascading detection - basic first, then AI Hunter for borderline cases"""
+ # Step 1: Basic
+ is_duplicate_basic, similarity_basic = self._check_duplicate_basic(result, idx, prog, out)
+
+ if is_duplicate_basic:
+ return True, similarity_basic
+
+ # Step 2: If basic detection finds moderate similarity, use AI Hunter
+ if similarity_basic >= 60: # Configurable threshold
+ print(f" 🤖 Moderate similarity ({similarity_basic}%) - running AI Hunter analysis...")
+ if hasattr(self, '_check_duplicate_ai_hunter'):
+ is_duplicate_ai, similarity_ai = self._check_duplicate_ai_hunter(result, idx, prog, out)
+ if is_duplicate_ai:
+ return True, similarity_ai
+ else:
+ print(" ⚠️ AI Hunter method not available for cascading analysis")
+
+ return False, max(similarity_basic, 0)
+
+ def _extract_text_features(self, text):
+ """Extract multiple features from text for AI Hunter analysis"""
+ features = {
+ 'semantic': {},
+ 'structural': {},
+ 'characters': [],
+ 'patterns': {}
+ }
+
+ # Semantic fingerprint
+ lines = text.split('\n')
+
+ # Character extraction (names that appear 3+ times)
+ words = re.findall(r'\b[A-Z][a-z]+\b', text)
+ word_freq = Counter(words)
+ features['characters'] = [name for name, count in word_freq.items() if count >= 3]
+
+ # Dialogue patterns
+ dialogue_patterns = re.findall(r'"([^"]+)"', text)
+ features['semantic']['dialogue_count'] = len(dialogue_patterns)
+ features['semantic']['dialogue_lengths'] = [len(d) for d in dialogue_patterns[:10]]
+
+ # Speaker patterns
+ speaker_patterns = re.findall(r'(\w+)\s+(?:said|asked|replied|shouted|whispered)', text.lower())
+ features['semantic']['speakers'] = list(set(speaker_patterns[:20]))
+
+ # Number extraction
+ numbers = re.findall(r'\b\d+\b', text)
+ features['patterns']['numbers'] = numbers[:20]
+
+ # Structural signature
+ para_lengths = []
+ dialogue_count = 0
+ for para in text.split('\n\n'):
+ if para.strip():
+ para_lengths.append(len(para))
+ if '"' in para:
+ dialogue_count += 1
+
+ features['structural']['para_count'] = len(para_lengths)
+ features['structural']['avg_para_length'] = sum(para_lengths) / max(1, len(para_lengths))
+ features['structural']['dialogue_ratio'] = dialogue_count / max(1, len(para_lengths))
+
+ # Create structural pattern string
+ pattern = []
+ for para in text.split('\n\n')[:20]: # First 20 paragraphs
+ if para.strip():
+ if '"' in para:
+ pattern.append('D') # Dialogue
+ elif len(para) > 300:
+ pattern.append('L') # Long
+ elif len(para) < 100:
+ pattern.append('S') # Short
+ else:
+ pattern.append('M') # Medium
+ features['structural']['pattern'] = ''.join(pattern)
+
+ return features
+
+ def _calculate_exact_similarity(self, text1, text2):
+ """Calculate exact text similarity"""
+ return SequenceMatcher(None, text1.lower(), text2.lower()).ratio()
+
+ def _calculate_smart_similarity(self, text1, text2):
+ """Smart similarity with length-aware sampling"""
+ # Check length ratio first
+ len_ratio = len(text1) / max(1, len(text2))
+ if len_ratio < 0.7 or len_ratio > 1.3:
+ return 0.0
+
+ # Smart sampling for large texts
+ if len(text1) > 10000:
+ sample_size = 3000
+ samples1 = [
+ text1[:sample_size],
+ text1[len(text1)//2 - sample_size//2:len(text1)//2 + sample_size//2],
+ text1[-sample_size:]
+ ]
+ samples2 = [
+ text2[:sample_size],
+ text2[len(text2)//2 - sample_size//2:len(text2)//2 + sample_size//2],
+ text2[-sample_size:]
+ ]
+ similarities = [SequenceMatcher(None, s1.lower(), s2.lower()).ratio()
+ for s1, s2 in zip(samples1, samples2)]
+ return sum(similarities) / len(similarities)
+ else:
+ # Use first 2000 chars for smaller texts
+ return SequenceMatcher(None, text1[:2000].lower(), text2[:2000].lower()).ratio()
+
+ def _calculate_semantic_similarity(self, sem1, sem2):
+ """Calculate semantic fingerprint similarity"""
+ score = 0.0
+ max_score = 0.0
+
+ # Compare dialogue counts
+ if 'dialogue_count' in sem1 and 'dialogue_count' in sem2:
+ max_score += 1.0
+ ratio = min(sem1['dialogue_count'], sem2['dialogue_count']) / max(1, max(sem1['dialogue_count'], sem2['dialogue_count']))
+ score += ratio * 0.3
+
+ # Compare speakers
+ if 'speakers' in sem1 and 'speakers' in sem2:
+ max_score += 1.0
+ if sem1['speakers'] and sem2['speakers']:
+ overlap = len(set(sem1['speakers']) & set(sem2['speakers']))
+ total = len(set(sem1['speakers']) | set(sem2['speakers']))
+ score += (overlap / max(1, total)) * 0.4
+
+ # Compare dialogue lengths pattern
+ if 'dialogue_lengths' in sem1 and 'dialogue_lengths' in sem2:
+ max_score += 1.0
+ if sem1['dialogue_lengths'] and sem2['dialogue_lengths']:
+ # Compare dialogue length patterns
+ len1 = sem1['dialogue_lengths'][:10]
+ len2 = sem2['dialogue_lengths'][:10]
+ if len1 and len2:
+ avg1 = sum(len1) / len(len1)
+ avg2 = sum(len2) / len(len2)
+ ratio = min(avg1, avg2) / max(1, max(avg1, avg2))
+ score += ratio * 0.3
+
+ return score / max(1, max_score)
+
+ def _calculate_structural_similarity(self, struct1, struct2):
+ """Calculate structural signature similarity"""
+ score = 0.0
+
+ # Compare paragraph patterns
+ if 'pattern' in struct1 and 'pattern' in struct2:
+ pattern_sim = SequenceMatcher(None, struct1['pattern'], struct2['pattern']).ratio()
+ score += pattern_sim * 0.4
+
+ # Compare paragraph statistics
+ if all(k in struct1 for k in ['para_count', 'avg_para_length', 'dialogue_ratio']) and \
+ all(k in struct2 for k in ['para_count', 'avg_para_length', 'dialogue_ratio']):
+
+ # Paragraph count ratio
+ para_ratio = min(struct1['para_count'], struct2['para_count']) / max(1, max(struct1['para_count'], struct2['para_count']))
+ score += para_ratio * 0.2
+
+ # Average length ratio
+ avg_ratio = min(struct1['avg_para_length'], struct2['avg_para_length']) / max(1, max(struct1['avg_para_length'], struct2['avg_para_length']))
+ score += avg_ratio * 0.2
+
+ # Dialogue ratio similarity
+ dialogue_diff = abs(struct1['dialogue_ratio'] - struct2['dialogue_ratio'])
+ score += (1 - dialogue_diff) * 0.2
+
+ return score
+
+ def _calculate_character_similarity(self, chars1, chars2):
+ """Calculate character name similarity"""
+ if not chars1 or not chars2:
+ return 0.0
+
+ # Find overlapping characters
+ set1 = set(chars1)
+ set2 = set(chars2)
+ overlap = len(set1 & set2)
+ total = len(set1 | set2)
+
+ return overlap / max(1, total)
+
+ def _calculate_pattern_similarity(self, pat1, pat2):
+ """Calculate pattern-based similarity"""
+ score = 0.0
+
+ # Compare numbers (they rarely change in translations)
+ if 'numbers' in pat1 and 'numbers' in pat2:
+ nums1 = set(pat1['numbers'])
+ nums2 = set(pat2['numbers'])
+ if nums1 and nums2:
+ overlap = len(nums1 & nums2)
+ total = len(nums1 | nums2)
+ score = overlap / max(1, total)
+
+ return score
+
+ def generate_rolling_summary(self, history_manager, chapter_num, base_system_content=None, source_text=None):
+ """Generate rolling summary after a chapter for context continuity.
+ Uses a dedicated summary system prompt (with glossary) distinct from translation.
+ Writes the summary to rolling_summary.txt and returns the summary string.
+ """
+ if not self.config.USE_ROLLING_SUMMARY:
+ return None
+
+
+ current_history = history_manager.load_history()
+ messages_to_include = self.config.ROLLING_SUMMARY_EXCHANGES * 2
+
+ # Prefer directly provided source text (e.g., just-translated chapter) when available
+ assistant_responses = []
+ if source_text and isinstance(source_text, str) and source_text.strip():
+ assistant_responses = [source_text]
+ else:
+ if len(current_history) >= 2:
+ recent_messages = current_history[-messages_to_include:] if messages_to_include > 0 else current_history
+ for h in recent_messages:
+ if h.get("role") == "assistant":
+ assistant_responses.append(h["content"])
+
+ # If still empty, skip quietly
+ if not assistant_responses:
+ return None
+
+ # Build a dedicated summary system prompt (do NOT reuse main translation system prompt)
+ # Append glossary to keep terminology consistent
+ summary_system_template = os.getenv("ROLLING_SUMMARY_SYSTEM_PROMPT", "You create concise summaries for continuity.").strip()
+ try:
+ glossary_path = find_glossary_file(self.out_dir)
+ except Exception:
+ glossary_path = None
+ system_prompt = build_system_prompt(summary_system_template, glossary_path)
+ # Add explicit instruction for clarity
+ system_prompt += "\n\n[Instruction: Generate a concise rolling summary of the previous chapter. Use glossary terms consistently. Do not include warnings or explanations.]"
+
+ user_prompt_template = os.getenv(
+ "ROLLING_SUMMARY_USER_PROMPT",
+ "Summarize the key events, characters, tone, and important details from these translations. "
+ "Focus on: character names/relationships, plot developments, and any special terminology used.\n\n"
+ "{translations}"
+ )
+
+ translations_text = "\n---\n".join(assistant_responses)
+ user_prompt = user_prompt_template.replace("{translations}", translations_text)
+
+ summary_msgs = [
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": f"[Rolling Summary of Chapter {chapter_num}]\n" + user_prompt}
+ ]
+
+
+ try:
+ summary_resp, _ = send_with_interrupt(
+ summary_msgs, self.client, self.config.TEMP,
+ min(2000, self.config.MAX_OUTPUT_TOKENS),
+ self.check_stop,
+ context='summary'
+ )
+
+ # Save the summary to the output folder
+ summary_file = os.path.join(self.out_dir, "rolling_summary.txt")
+ header = f"=== Rolling Summary of Chapter {chapter_num} ===\n(This is a summary of the previous chapter for context)\n"
+
+ mode = "a" if self.config.ROLLING_SUMMARY_MODE == "append" else "w"
+ with open(summary_file, mode, encoding="utf-8") as sf:
+ if mode == "a":
+ sf.write("\n\n")
+ sf.write(header)
+ sf.write(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}]\n")
+ sf.write(summary_resp.strip())
+
+ # If in append mode, trim to retain only the last N entries if configured
+ try:
+ if self.config.ROLLING_SUMMARY_MODE == "append":
+ max_entries = int(getattr(self.config, "ROLLING_SUMMARY_MAX_ENTRIES", 0) or 0)
+ if max_entries > 0:
+ with open(summary_file, 'r', encoding='utf-8') as rf:
+ content = rf.read()
+ # Find the start of each summary block by header line
+ headers = [m.start() for m in re.finditer(r"(?m)^===\s*Rolling Summary.*$", content)]
+ if len(headers) > max_entries:
+ # Keep only the last max_entries blocks
+ keep_starts = headers[-max_entries:]
+ blocks = []
+ for i, s in enumerate(keep_starts):
+ e = keep_starts[i + 1] if i + 1 < len(keep_starts) else len(content)
+ block = content[s:e].strip()
+ if block:
+ blocks.append(block)
+ trimmed_content = ("\n\n".join(blocks) + "\n") if blocks else ""
+ with open(summary_file, 'w', encoding='utf-8') as wf:
+ wf.write(trimmed_content)
+ # Optional log showing retained count
+ try:
+ self._log(f"📚 Total summaries in memory: {len(blocks)} (trimmed to last {max_entries})")
+ except Exception:
+ pass
+ except Exception as _trim_err:
+ try:
+ self._log(f"⚠️ Failed to trim rolling summaries: {_trim_err}")
+ except Exception:
+ pass
+
+ # Log to GUI if available, otherwise console
+ try:
+ self._log(f"📝 Generated rolling summary for Chapter {chapter_num} ({'append' if mode=='a' else 'replace'} mode)")
+ self._log(f" ➜ Saved to: {summary_file} ({len(summary_resp.strip())} chars)")
+ except Exception:
+ print(f"📝 Generated rolling summary for Chapter {chapter_num} ({'append' if mode=='a' else 'replace'} mode)")
+ print(f" ➜ Saved to: {summary_file} ({len(summary_resp.strip())} chars)")
+ return summary_resp.strip()
+
+ except Exception as e:
+ try:
+ self._log(f"⚠️ Failed to generate rolling summary: {e}")
+ except Exception:
+ print(f"⚠️ Failed to generate rolling summary: {e}")
+ return None
+
+ def translate_with_retry(self, msgs, chunk_html, c, chunk_idx, total_chunks):
+ """Handle translation with retry logic"""
+
+ # CRITICAL FIX: Reset client state for each chunk
+ if hasattr(self.client, 'reset_cleanup_state'):
+ self.client.reset_cleanup_state()
+
+ # Also ensure we're not in cleanup mode from previous operations
+ if hasattr(self.client, '_in_cleanup'):
+ self.client._in_cleanup = False
+ if hasattr(self.client, '_cancelled'):
+ self.client._cancelled = False
+
+
+ retry_count = 0
+
+ # Get retry attempts from AI Hunter config if available
+ ai_config = {}
+ try:
+ # Try to get AI Hunter config from environment variable first
+ ai_hunter_config_str = os.getenv('AI_HUNTER_CONFIG')
+ if ai_hunter_config_str:
+ ai_config = json.loads(ai_hunter_config_str)
+ else:
+ # Fallback to config attribute
+ ai_config = getattr(self.config, 'ai_hunter_config', {})
+ except (json.JSONDecodeError, AttributeError):
+ ai_config = {}
+
+ if isinstance(ai_config, dict):
+ max_retries = ai_config.get('retry_attempts', 3)
+ max_duplicate_retries = ai_config.get('retry_attempts', 6) # Use same setting for duplicate retries
+ else:
+ max_retries = 3
+ max_duplicate_retries = 6
+
+ duplicate_retry_count = 0
+ timeout_retry_count = 0
+ max_timeout_retries = 2
+ history_purged = False
+
+ original_max_tokens = self.config.MAX_OUTPUT_TOKENS
+ original_temp = self.config.TEMP
+ original_user_prompt = msgs[-1]["content"]
+
+ chunk_timeout = None
+ if self.config.RETRY_TIMEOUT:
+ chunk_timeout = self.config.CHUNK_TIMEOUT
+
+ result = None
+ finish_reason = None
+
+ while True:
+ if self.check_stop():
+ return None, None
+
+ try:
+ current_max_tokens = self.config.MAX_OUTPUT_TOKENS
+ current_temp = self.config.TEMP
+
+ total_tokens = sum(self.chapter_splitter.count_tokens(m["content"]) for m in msgs)
+ # Determine file reference
+ if c.get('is_chunk', False):
+ file_ref = f"Section_{c['num']}"
+ else:
+ # Check if this is a text file - need to access from self
+ is_text_source = self.is_text_file or c.get('filename', '').endswith('.txt')
+ terminology = "Section" if is_text_source else "Chapter"
+ file_ref = c.get('original_basename', f'{terminology}_{c["num"]}')
+
+ print(f"[DEBUG] Chunk {chunk_idx}/{total_chunks} tokens = {total_tokens:,} / {self.get_token_budget_str()} [File: {file_ref}]")
+
+ self.client.context = 'translation'
+
+ # Generate filename for chunks
+ if chunk_idx and total_chunks > 1:
+ # This is a chunk - use chunk naming format
+ fname = f"response_{c['num']:03d}_chunk_{chunk_idx}.html"
+ else:
+ # Not a chunk - use regular naming
+ fname = FileUtilities.create_chapter_filename(c, c.get('actual_chapter_num', c['num']))
+
+ # Set output filename BEFORE the API call
+ if hasattr(self.client, 'set_output_filename'):
+ self.client.set_output_filename(fname)
+
+ # Track the filename so truncation logs know which file this is
+ if hasattr(self.client, '_current_output_file'):
+ self.client._current_output_file = fname
+
+ # Generate unique request ID for this chunk
+ #request_id = f"{c['num']:03d}_chunk{chunk_idx}_{uuid.uuid4().hex[:8]}"
+
+ result, finish_reason = send_with_interrupt(
+ msgs, self.client, current_temp, current_max_tokens,
+ self.check_stop, chunk_timeout
+ )
+ # Enhanced mode workflow:
+ # 1. Original HTML -> html2text -> Markdown/plain text (during extraction)
+ # 2. Markdown sent to translation API (better for translation quality)
+ # 3. Translated markdown -> HTML conversion (here)
+ if result and c.get("enhanced_extraction", False):
+ print(f"🔄 Converting translated markdown back to HTML...")
+ result = convert_enhanced_text_to_html(result, c)
+ retry_needed = False
+ retry_reason = ""
+ is_duplicate_retry = False
+
+ # ENHANCED: Force re-read environment variable for latest setting
+ retry_truncated_enabled = os.getenv("RETRY_TRUNCATED", "0") == "1"
+
+ # Debug logging to verify the toggle state
+ #print(f" DEBUG: finish_reason='{finish_reason}', RETRY_TRUNCATED={retry_truncated_enabled}, config.RETRY_TRUNCATED={self.config.RETRY_TRUNCATED}")
+ #print(f" DEBUG: Current tokens={self.config.MAX_OUTPUT_TOKENS}, Min retry tokens={self.config.MAX_RETRY_TOKENS}, retry_count={retry_count}")
+
+ if finish_reason == "length" and (retry_truncated_enabled or self.config.RETRY_TRUNCATED):
+ if retry_count < max_retries:
+ # For truncated responses, ensure we never go below the minimum retry tokens
+ proposed_limit = self.config.MAX_OUTPUT_TOKENS * 2
+
+ # Always enforce minimum - never retry with tokens below the constraint
+ new_token_limit = max(proposed_limit, self.config.MAX_RETRY_TOKENS)
+
+ if new_token_limit != self.config.MAX_OUTPUT_TOKENS:
+ retry_needed = True
+ retry_reason = "truncated output"
+ old_limit = self.config.MAX_OUTPUT_TOKENS
+ self.config.MAX_OUTPUT_TOKENS = new_token_limit
+ retry_count += 1
+
+ if old_limit < self.config.MAX_RETRY_TOKENS:
+ print(f" 🔄 TRUNCATION RETRY: Boosting tokens {old_limit} → {new_token_limit} (enforcing minimum: {self.config.MAX_RETRY_TOKENS})")
+ else:
+ print(f" 🔄 TRUNCATION RETRY: Doubling tokens {old_limit} → {new_token_limit} (above minimum: {self.config.MAX_RETRY_TOKENS})")
+ else:
+ print(f" ⚠️ TRUNCATION DETECTED: Token adjustment not needed - already at maximum {self.config.MAX_OUTPUT_TOKENS}")
+ else:
+ print(f" ⚠️ TRUNCATION DETECTED: Max retries ({max_retries}) reached - accepting truncated response")
+ elif finish_reason == "length" and not (retry_truncated_enabled or self.config.RETRY_TRUNCATED):
+ print(f" ⏭️ TRUNCATION DETECTED: Auto-retry is DISABLED - accepting truncated response")
+ elif finish_reason == "length":
+ print(f" ⚠️ TRUNCATION DETECTED: Unexpected condition - check logic")
+
+ if not retry_needed:
+ # Force re-read the environment variable to ensure we have current setting
+ duplicate_enabled = os.getenv("RETRY_DUPLICATE_BODIES", "0") == "1"
+
+ if duplicate_enabled and duplicate_retry_count < max_duplicate_retries:
+ idx = c.get('__index', 0)
+ prog = c.get('__progress', {})
+ print(f" 🔍 Checking for duplicate content...")
+ # Get actual chapter number for duplicate detection
+ actual_num = c.get('actual_chapter_num', c.get('num', idx + 1))
+ is_duplicate, similarity = self.check_duplicate_content(result, idx, prog, self.out_dir, actual_num)
+
+ if is_duplicate:
+ retry_needed = True
+ is_duplicate_retry = True
+ retry_reason = f"duplicate content (similarity: {similarity}%)"
+ duplicate_retry_count += 1
+
+ # Check if temperature change is disabled
+ disable_temp_change = ai_config.get('disable_temperature_change', False) if isinstance(ai_config, dict) else False
+
+ if duplicate_retry_count >= 3 and not history_purged:
+ print(f" 🧹 Clearing history after 3 attempts...")
+ if 'history_manager' in c:
+ c['history_manager'].save_history([])
+ history_purged = True
+ if not disable_temp_change:
+ self.config.TEMP = original_temp
+ else:
+ print(f" 🌡️ Temperature change disabled - keeping current temp: {self.config.TEMP}")
+
+ elif duplicate_retry_count == 1:
+ if disable_temp_change:
+ print(f" 🔄 First duplicate retry - temperature change disabled")
+ else:
+ print(f" 🔄 First duplicate retry - same temperature")
+
+ elif history_purged:
+ if not disable_temp_change:
+ attempts_since_purge = duplicate_retry_count - 3
+ self.config.TEMP = min(original_temp + (0.1 * attempts_since_purge), 1.0)
+ print(f" 🌡️ Post-purge temp: {self.config.TEMP}")
+ else:
+ print(f" 🌡️ Temperature change disabled - keeping temp: {self.config.TEMP}")
+
+ else:
+ if not disable_temp_change:
+ self.config.TEMP = min(original_temp + (0.1 * (duplicate_retry_count - 1)), 1.0)
+ print(f" 🌡️ Gradual temp increase: {self.config.TEMP}")
+ else:
+ print(f" 🌡️ Temperature change disabled - keeping temp: {self.config.TEMP}")
+
+ if duplicate_retry_count == 1:
+ user_prompt = f"[RETRY] Chapter {c['num']}: Ensure unique translation.\n{chunk_html}"
+ elif duplicate_retry_count <= 3:
+ user_prompt = f"[ATTEMPT {duplicate_retry_count}] Translate uniquely:\n{chunk_html}"
+ else:
+ user_prompt = f"Chapter {c['num']}:\n{chunk_html}"
+
+ msgs[-1] = {"role": "user", "content": user_prompt}
+ elif not duplicate_enabled:
+ print(f" ⏭️ Duplicate detection is DISABLED - skipping check")
+
+ if retry_needed:
+ if is_duplicate_retry:
+ print(f" 🔄 Duplicate retry {duplicate_retry_count}/{max_duplicate_retries}")
+ else:
+ print(f" 🔄 Retry {retry_count}/{max_retries}: {retry_reason}")
+
+ time.sleep(2)
+ continue
+
+ break
+
+ except UnifiedClientError as e:
+ error_msg = str(e)
+
+ if "stopped by user" in error_msg:
+ print("❌ Translation stopped by user during API call")
+ return None, None
+
+ if "took" in error_msg and "timeout:" in error_msg:
+ if timeout_retry_count < max_timeout_retries:
+ timeout_retry_count += 1
+ print(f" ⏱️ Chunk took too long, retry {timeout_retry_count}/{max_timeout_retries}")
+ print(f" 🔄 Retrying")
+ time.sleep(2)
+ continue
+ else:
+ print(f" ❌ Max timeout retries reached")
+ raise UnifiedClientError("Translation failed after timeout retries")
+
+ elif "timed out" in error_msg and "timeout:" not in error_msg:
+ print(f"⚠️ {error_msg}, retrying...")
+ time.sleep(5)
+ continue
+
+ elif getattr(e, "error_type", None) == "rate_limit" or getattr(e, "http_status", None) == 429:
+ # Rate limit errors - clean handling without traceback
+ print("⚠️ Rate limited, sleeping 60s…")
+ for i in range(60):
+ if self.check_stop():
+ print("❌ Translation stopped during rate limit wait")
+ return None, None
+ time.sleep(1)
+ continue
+
+ else:
+ # For unexpected errors, show the error message but suppress traceback in most cases
+ if getattr(e, "error_type", None) in ["api_error", "validation", "prohibited_content"]:
+ print(f"❌ API Error: {error_msg}")
+ raise UnifiedClientError(f"API Error: {error_msg}")
+ else:
+ raise
+
+ except Exception as e:
+ print(f"❌ Unexpected error during API call: {e}")
+ raise
+
+ self.config.MAX_OUTPUT_TOKENS = original_max_tokens
+ self.config.TEMP = original_temp
+
+ if retry_count > 0 or duplicate_retry_count > 0 or timeout_retry_count > 0:
+ if duplicate_retry_count > 0:
+ print(f" 🔄 Restored original temperature: {self.config.TEMP} (after {duplicate_retry_count} duplicate retries)")
+ elif timeout_retry_count > 0:
+ print(f" 🔄 Restored original settings after {timeout_retry_count} timeout retries")
+ elif retry_count > 0:
+ print(f" 🔄 Restored original settings after {retry_count} retries")
+
+ if duplicate_retry_count >= max_duplicate_retries:
+ print(f" ⚠️ WARNING: Duplicate content issue persists after {max_duplicate_retries} attempts")
+
+ return result, finish_reason
+
+ def get_token_budget_str(self):
+ """Get token budget as string"""
+ _tok_env = os.getenv("MAX_INPUT_TOKENS", "1000000").strip()
+ max_tokens_limit, budget_str = parse_token_limit(_tok_env)
+ return budget_str
+
+# =====================================================
+# BATCH TRANSLATION PROCESSOR
+# =====================================================
+class BatchTranslationProcessor:
+ """Handles batch/parallel translation processing"""
+
+ def __init__(self, config, client, base_msg, out_dir, progress_lock,
+ save_progress_fn, update_progress_fn, check_stop_fn,
+ image_translator=None, is_text_file=False):
+ self.config = config
+ self.client = client
+ self.base_msg = base_msg
+ self.out_dir = out_dir
+ self.progress_lock = progress_lock
+ self.save_progress_fn = save_progress_fn
+ self.update_progress_fn = update_progress_fn
+ self.check_stop_fn = check_stop_fn
+ self.image_translator = image_translator
+ self.chapters_completed = 0
+ self.chunks_completed = 0
+ self.is_text_file = is_text_file
+
+ # Optionally log multi-key status
+ if hasattr(self.client, 'use_multi_keys') and self.client.use_multi_keys:
+ stats = self.client.get_stats()
+ print(f"🔑 Batch processor using multi-key mode: {stats.get('total_keys', 0)} keys")
+
+ def process_single_chapter(self, chapter_data):
+ """Process a single chapter (runs in thread)"""
+ # APPLY INTERRUPTIBLE THREADING DELAY FIRST
+ thread_delay = float(os.getenv("THREAD_SUBMISSION_DELAY_SECONDS", "0.5"))
+ if thread_delay > 0:
+ # Check if we need to wait (same logic as unified_api_client)
+ if hasattr(self.client, '_thread_submission_lock') and hasattr(self.client, '_last_thread_submission_time'):
+ with self.client._thread_submission_lock:
+ current_time = time.time()
+ time_since_last = current_time - self.client._last_thread_submission_time
+
+ if time_since_last < thread_delay:
+ sleep_time = thread_delay - time_since_last
+ thread_name = threading.current_thread().name
+
+ # PRINT BEFORE THE DELAY STARTS
+ idx, chapter = chapter_data # Extract chapter info for better logging
+ print(f"🧵 [{thread_name}] Applying thread delay: {sleep_time:.1f}s for Chapter {idx+1}")
+
+ # Interruptible sleep - check stop flag every 0.1 seconds
+ elapsed = 0
+ check_interval = 0.1
+ while elapsed < sleep_time:
+ if self.check_stop_fn():
+ print(f"🛑 Threading delay interrupted by stop flag")
+ raise Exception("Translation stopped by user during threading delay")
+
+ sleep_chunk = min(check_interval, sleep_time - elapsed)
+ time.sleep(sleep_chunk)
+ elapsed += sleep_chunk
+
+ self.client._last_thread_submission_time = time.time()
+ if not hasattr(self.client, '_thread_submission_count'):
+ self.client._thread_submission_count = 0
+ self.client._thread_submission_count += 1
+
+ idx, chapter = chapter_data
+ chap_num = chapter["num"]
+
+ # Use the pre-calculated actual_chapter_num from the main loop
+ actual_num = chapter.get('actual_chapter_num')
+
+ # Fallback if not set (common in batch mode where first pass might be skipped)
+ if actual_num is None:
+ # Try to extract it using the same logic as non-batch mode
+ raw_num = FileUtilities.extract_actual_chapter_number(chapter, patterns=None, config=self.config)
+
+ # Apply offset if configured
+ offset = self.config.CHAPTER_NUMBER_OFFSET if hasattr(self.config, 'CHAPTER_NUMBER_OFFSET') else 0
+ raw_num += offset
+
+ # Check if zero detection is disabled
+ if hasattr(self.config, 'DISABLE_ZERO_DETECTION') and self.config.DISABLE_ZERO_DETECTION:
+ actual_num = raw_num
+ elif hasattr(self.config, '_uses_zero_based') and self.config._uses_zero_based:
+ # This is a 0-based novel, adjust the number
+ actual_num = raw_num + 1
+ else:
+ # Default to raw number (1-based or unknown)
+ actual_num = raw_num
+
+ print(f" 📖 Extracted actual chapter number: {actual_num} (from raw: {raw_num})")
+
+ try:
+ # Check if this is from a text file
+ ai_features = None
+ is_text_source = self.is_text_file or chapter.get('filename', '').endswith('.txt') or chapter.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+ print(f"🔄 Starting #{idx+1} (Internal: {terminology} {chap_num}, Actual: {terminology} {actual_num}) (thread: {threading.current_thread().name}) [File: {chapter.get('original_basename', f'{terminology}_{chap_num}')}]")
+
+ content_hash = chapter.get("content_hash") or ContentProcessor.get_content_hash(chapter["body"])
+ with self.progress_lock:
+ self.update_progress_fn(idx, actual_num, content_hash, None, status="in_progress")
+ self.save_progress_fn()
+
+ chapter_body = chapter["body"]
+ if chapter.get('has_images') and self.image_translator and self.config.ENABLE_IMAGE_TRANSLATION:
+ print(f"🖼️ Processing images for Chapter {actual_num}...")
+ self.image_translator.set_current_chapter(actual_num)
+ chapter_body, image_translations = process_chapter_images(
+ chapter_body,
+ actual_num,
+ self.image_translator,
+ self.check_stop_fn
+ )
+ if image_translations:
+ # Create a copy of the processed body
+ from bs4 import BeautifulSoup
+ c = chapter
+ soup_for_text = BeautifulSoup(c["body"], 'html.parser')
+
+ # Remove all translated content
+ for trans_div in soup_for_text.find_all('div', class_='translated-text-only'):
+ trans_div.decompose()
+
+ # Use this cleaned version for text translation
+ text_to_translate = str(soup_for_text)
+ final_body_with_images = c["body"]
+ else:
+ text_to_translate = c["body"]
+ image_translations = {}
+ print(f"✅ Processed {len(image_translations)} images for Chapter {actual_num}")
+
+ chapter_msgs = self.base_msg + [{"role": "user", "content": chapter_body}]
+
+ # Generate filename before API call
+ fname = FileUtilities.create_chapter_filename(chapter, actual_num)
+ self.client.set_output_filename(fname)
+
+ if hasattr(self.client, '_current_output_file'):
+ self.client._current_output_file = fname
+
+ print(f"📤 Sending Chapter {actual_num} to API...")
+ result, finish_reason = send_with_interrupt(
+ chapter_msgs, self.client, self.config.TEMP,
+ self.config.MAX_OUTPUT_TOKENS, self.check_stop_fn
+ )
+
+ print(f"📥 Received Chapter {actual_num} response, finish_reason: {finish_reason}")
+
+ # Enhanced mode workflow (same as non-batch):
+ # 1. Original HTML -> html2text -> Markdown/plain text (during extraction)
+ # 2. Markdown sent to translation API (better for translation quality)
+ # 3. Translated markdown -> HTML conversion (here)
+ if result and chapter.get("enhanced_extraction", False):
+ print(f"🔄 Converting translated markdown back to HTML...")
+ result = convert_enhanced_text_to_html(result, chapter)
+
+ if finish_reason in ["length", "max_tokens"]:
+ print(f"⚠️ Chapter {actual_num} response was TRUNCATED!")
+
+ if self.config.REMOVE_AI_ARTIFACTS:
+ result = ContentProcessor.clean_ai_artifacts(result, True)
+
+ result = ContentProcessor.clean_memory_artifacts(result)
+
+ cleaned = re.sub(r"^```(?:html)?\s*\n?", "", result, count=1, flags=re.MULTILINE)
+ cleaned = re.sub(r"\n?```\s*$", "", cleaned, count=1, flags=re.MULTILINE)
+ cleaned = ContentProcessor.clean_ai_artifacts(cleaned, remove_artifacts=self.config.REMOVE_AI_ARTIFACTS)
+
+ fname = FileUtilities.create_chapter_filename(chapter, actual_num)
+
+ if self.is_text_file:
+ # For text files, save as plain text
+ fname_txt = fname.replace('.html', '.txt') if fname.endswith('.html') else fname
+
+ # Extract text from HTML
+ from bs4 import BeautifulSoup
+ soup = BeautifulSoup(cleaned, 'html.parser')
+ text_content = soup.get_text(strip=True)
+
+ # Merge image translations back with text translation
+ if 'final_body_with_images' in locals() and image_translations:
+ # Parse both versions
+ soup_with_images = BeautifulSoup(final_body_with_images, 'html.parser')
+ soup_with_text = BeautifulSoup(cleaned, 'html.parser')
+
+ # Get the translated text content (without images)
+ body_content = soup_with_text.body
+
+ # Add image translations to the translated content
+ for trans_div in soup_with_images.find_all('div', class_='translated-text-only'):
+ body_content.insert(0, trans_div)
+
+ final_html = str(soup_with_text)
+ cleaned = final_html
+
+ with open(os.path.join(self.out_dir, fname), 'w', encoding='utf-8') as f:
+ f.write(cleaned)
+
+ # Update with .txt filename
+ with self.progress_lock:
+ self.update_progress_fn(idx, actual_num, content_hash, fname_txt, status="completed", ai_features=ai_features)
+ self.save_progress_fn()
+ else:
+ # Original code for EPUB files
+ with open(os.path.join(self.out_dir, fname), 'w', encoding='utf-8') as f:
+ f.write(cleaned)
+
+ print(f"💾 Saved Chapter {actual_num}: {fname} ({len(cleaned)} chars)")
+
+ # Initialize ai_features at the beginning to ensure it's always defined
+ if ai_features is None:
+ ai_features = None
+
+ # Extract and save AI features for future duplicate detection
+ if (self.config.RETRY_DUPLICATE_BODIES and
+ hasattr(self.config, 'DUPLICATE_DETECTION_MODE') and
+ self.config.DUPLICATE_DETECTION_MODE in ['ai-hunter', 'cascading']):
+ try:
+ # Extract features from the translated content
+ cleaned_text = re.sub(r'<[^>]+>', '', cleaned).strip()
+ # Note: self.translator doesn't exist, so we can't extract features here
+ # The features will need to be extracted during regular processing
+ print(f" ⚠️ AI features extraction not available in batch mode")
+ except Exception as e:
+ print(f" ⚠️ Failed to extract AI features: {e}")
+
+ with self.progress_lock:
+ # Check for QA failures with comprehensive detection
+ if is_qa_failed_response(cleaned):
+ chapter_status = "qa_failed"
+ failure_reason = get_failure_reason(cleaned)
+ print(f"⚠️ Batch: Chapter {actual_num} marked as qa_failed: {failure_reason}")
+ # Update progress to qa_failed status
+ self.update_progress_fn(idx, actual_num, content_hash, fname, status=chapter_status, ai_features=ai_features)
+ self.save_progress_fn()
+ # DO NOT increment chapters_completed for qa_failed
+ # Return False to indicate failure
+ return False, actual_num
+ else:
+ chapter_status = "completed"
+ # Update progress to completed status
+ self.update_progress_fn(idx, actual_num, content_hash, fname, status=chapter_status, ai_features=ai_features)
+ self.save_progress_fn()
+ # Only increment chapters_completed for successful chapters
+ self.chapters_completed += 1
+ self.chunks_completed += 1
+
+ print(f"✅ Chapter {actual_num} completed successfully")
+ return True, actual_num
+
+ except Exception as e:
+ print(f"❌ Chapter {actual_num} failed: {e}")
+ with self.progress_lock:
+ self.update_progress_fn(idx, actual_num, content_hash, None, status="failed")
+ self.save_progress_fn()
+ return False, actual_num
+
+# =====================================================
+# GLOSSARY MANAGER - TRUE CSV FORMAT WITH FUZZY MATCHING
+# =====================================================
+
+class GlossaryManager:
+ """Unified glossary management with true CSV format, fuzzy matching, and parallel processing"""
+
+ # Class-level shared lock for API submission timing
+ _api_submission_lock = threading.Lock()
+ _last_api_submission_time = 0
+
+ def __init__(self):
+ self.pattern_manager = PatternManager()
+ self._results_lock = threading.Lock() # Thread lock for collecting results
+ self._file_write_lock = threading.Lock() # Thread lock for file operations
+
+ def _atomic_write_file(self, filepath, content, encoding='utf-8'):
+ """Atomically write to a file to prevent corruption from concurrent writes"""
+
+ # Create temp file in same directory to ensure same filesystem
+ dir_path = os.path.dirname(filepath)
+
+ with self._file_write_lock:
+ try:
+ # Write to temporary file first
+ with tempfile.NamedTemporaryFile(mode='w', encoding=encoding,
+ dir=dir_path, delete=False) as tmp_file:
+ tmp_file.write(content)
+ tmp_path = tmp_file.name
+
+ # Atomic rename (on same filesystem)
+ if os.name == 'nt': # Windows
+ # Windows doesn't support atomic rename if target exists
+ if os.path.exists(filepath):
+ os.remove(filepath)
+ os.rename(tmp_path, filepath)
+ else: # Unix/Linux/Mac
+ os.rename(tmp_path, filepath)
+
+ return True
+
+ except Exception as e:
+ print(f"⚠️ Atomic write failed: {e}")
+ # Cleanup temp file if it exists
+ if 'tmp_path' in locals() and os.path.exists(tmp_path):
+ try:
+ os.remove(tmp_path)
+ except:
+ pass
+
+ # Fallback to direct write with lock
+ try:
+ with open(filepath, 'w', encoding=encoding) as f:
+ f.write(content)
+ return True
+ except Exception as e2:
+ print(f"⚠️ Fallback write also failed: {e2}")
+ return False
+
+ def save_glossary(self, output_dir, chapters, instructions, language="korean"):
+ """Targeted glossary generator with true CSV format output and parallel processing"""
+ print("📑 Targeted Glossary Generator v6.0 (CSV Format + Parallel)")
+
+ # Check stop flag at start
+ # Ensure output directory exists
+ try:
+ os.makedirs(output_dir, exist_ok=True)
+ except Exception as _e:
+ print(f"⚠️ Could not ensure output directory exists: {output_dir} ({_e})")
+ if is_stop_requested():
+ print("📑 ❌ Glossary generation stopped by user")
+ return {}
+
+ # Check if glossary already exists; if so, we'll MERGE it later (do not return early)
+ glossary_path = os.path.join(output_dir, "glossary.csv")
+ existing_glossary_content = None
+ if os.path.exists(glossary_path):
+ print(f"📑 Existing glossary detected (will merge): {glossary_path}")
+ try:
+ with open(glossary_path, 'r', encoding='utf-8') as f:
+ existing_glossary_content = f.read()
+ except Exception as e:
+ print(f"⚠️ Could not read existing glossary: {e}")
+
+ # Rest of the method continues as before...
+ print("📑 Extracting names and terms with configurable options")
+
+ # Check stop flag before processing
+ if is_stop_requested():
+ print("📑 ❌ Glossary generation stopped by user")
+ return {}
+
+ # Check for manual glossary first (CSV only)
+ manual_glossary_path = os.getenv("MANUAL_GLOSSARY")
+ existing_glossary = None
+ if manual_glossary_path and os.path.exists(manual_glossary_path):
+ print(f"📑 Manual glossary detected: {os.path.basename(manual_glossary_path)}")
+ try:
+ with open(manual_glossary_path, 'r', encoding='utf-8') as f:
+ content = f.read()
+ # Treat as CSV text and stage it for merge; also copy to output for visibility
+ target_path = os.path.join(output_dir, "glossary.csv")
+ with open(target_path, 'w', encoding='utf-8') as f:
+ f.write(content)
+ print(f"📑 ✅ Manual CSV glossary copied to: {target_path}")
+ existing_glossary = content
+ except Exception as e:
+ print(f"⚠️ Could not copy manual glossary: {e}")
+ print(f"📑 Proceeding with automatic generation...")
+
+ # Check for existing glossary from manual extraction
+ glossary_folder_path = os.path.join(output_dir, "Glossary")
+ # existing_glossary may already be set by MANUAL_GLOSSARY above
+
+ if os.path.exists(glossary_folder_path):
+ for file in os.listdir(glossary_folder_path):
+ if file.endswith("_glossary.json"):
+ existing_path = os.path.join(glossary_folder_path, file)
+ try:
+ with open(existing_path, 'r', encoding='utf-8') as f:
+ existing_content = f.read()
+ existing_glossary = existing_content
+ print(f"📑 Found existing glossary from manual extraction: {file}")
+ break
+ except Exception as e:
+ print(f"⚠️ Could not load existing glossary: {e}")
+
+ # Get configuration from environment variables
+ min_frequency = int(os.getenv("GLOSSARY_MIN_FREQUENCY", "2"))
+ max_names = int(os.getenv("GLOSSARY_MAX_NAMES", "50"))
+ max_titles = int(os.getenv("GLOSSARY_MAX_TITLES", "30"))
+ batch_size = int(os.getenv("GLOSSARY_BATCH_SIZE", "50"))
+ strip_honorifics = os.getenv("GLOSSARY_STRIP_HONORIFICS", "1") == "1"
+ fuzzy_threshold = float(os.getenv("GLOSSARY_FUZZY_THRESHOLD", "0.90"))
+ max_text_size = int(os.getenv("GLOSSARY_MAX_TEXT_SIZE", "50000"))
+
+ print(f"📑 Settings: Min frequency: {min_frequency}, Max names: {max_names}, Max titles: {max_titles}")
+ print(f"📑 Strip honorifics: {'✅ Yes' if strip_honorifics else '❌ No'}")
+ print(f"📑 Fuzzy matching threshold: {fuzzy_threshold}")
+
+ # Get custom prompt from environment
+ custom_prompt = os.getenv("AUTO_GLOSSARY_PROMPT", "").strip()
+
+ def clean_html(html_text):
+ """Remove HTML tags to get clean text"""
+ soup = BeautifulSoup(html_text, 'html.parser')
+ return soup.get_text()
+
+ # Check stop before processing chapters
+ if is_stop_requested():
+ print("📑 ❌ Glossary generation stopped by user")
+ return {}
+
+ # Get chapter split threshold and filter mode
+ chapter_split_threshold = int(os.getenv("GLOSSARY_CHAPTER_SPLIT_THRESHOLD", "100000"))
+ filter_mode = os.getenv("GLOSSARY_FILTER_MODE", "all") # all, only_with_honorifics, only_without_honorifics
+
+ # Check if parallel extraction is enabled for automatic glossary
+ extraction_workers = int(os.getenv("EXTRACTION_WORKERS", "1"))
+ batch_translation = os.getenv("BATCH_TRANSLATION", "0") == "1"
+ api_batch_size = int(os.getenv("BATCH_SIZE", "5"))
+
+ # Log the settings
+ print(f"📑 Filter mode: {filter_mode}")
+ if extraction_workers > 1:
+ print(f"📑 Parallel extraction enabled: {extraction_workers} workers")
+ if batch_translation:
+ print(f"📑 Batch API calls enabled: {api_batch_size} chunks per batch")
+
+ all_text = ' '.join(clean_html(chapter["body"]) for chapter in chapters)
+ print(f"📑 Processing {len(all_text):,} characters of text")
+
+ # Apply smart filtering FIRST to check actual size needed
+ use_smart_filter = os.getenv("GLOSSARY_USE_SMART_FILTER", "1") == "1"
+ effective_text_size = len(all_text)
+
+ filtered_text_cache = None
+ if use_smart_filter and custom_prompt: # Only apply for AI extraction
+ print(f"📑 Smart filtering enabled - checking effective text size after filtering...")
+ # Perform filtering ONCE and reuse for chunking
+ filtered_sample, _ = self._filter_text_for_glossary(all_text, min_frequency)
+ filtered_text_cache = filtered_sample
+ effective_text_size = len(filtered_sample)
+ print(f"📑 Effective text size after filtering: {effective_text_size:,} chars (from {len(all_text):,})")
+
+ # Check if we need to split into chunks based on EFFECTIVE size after filtering
+ if chapter_split_threshold > 0 and effective_text_size > chapter_split_threshold:
+ print(f"📑 Effective text exceeds {chapter_split_threshold:,} chars, will process in chunks...")
+
+ # If using smart filter, we need to split the FILTERED text, not raw text
+ if use_smart_filter and custom_prompt:
+ # Split the filtered text into chunks (reuse cached filtered text)
+ filtered_text = filtered_text_cache if filtered_text_cache is not None else self._filter_text_for_glossary(all_text, min_frequency)[0]
+ chunks_to_process = []
+
+ # Split filtered text into chunks of appropriate size
+ chunk_size = chapter_split_threshold
+ for i in range(0, len(filtered_text), chunk_size):
+ chunk_text = filtered_text[i:i + chunk_size]
+ chunks_to_process.append((len(chunks_to_process) + 1, chunk_text))
+
+ print(f"📑 Split filtered text into {len(chunks_to_process)} chunks")
+ all_glossary_entries = []
+ else:
+ # Original logic for unfiltered text
+ all_glossary_entries = []
+ chunk_size = 0
+ chunk_chapters = []
+ chunks_to_process = []
+
+ for idx, chapter in enumerate(chapters):
+ if is_stop_requested():
+ print("📑 ❌ Glossary generation stopped by user")
+ return all_glossary_entries
+
+ chapter_text = clean_html(chapter["body"])
+ chunk_size += len(chapter_text)
+ chunk_chapters.append(chapter)
+
+ # Process chunk when it reaches threshold or last chapter
+ if chunk_size >= chapter_split_threshold or idx == len(chapters) - 1:
+ chunk_text = ' '.join(clean_html(ch["body"]) for ch in chunk_chapters)
+ chunks_to_process.append((len(chunks_to_process) + 1, chunk_text))
+
+ # Reset for next chunk
+ chunk_size = 0
+ chunk_chapters = []
+
+ print(f"📑 Split into {len(chunks_to_process)} chunks for processing")
+
+ # Batch toggle decides concurrency: ON => parallel API calls; OFF => strict sequential
+ if batch_translation and custom_prompt and len(chunks_to_process) > 1:
+ print(f"📑 Processing chunks in batch mode with {api_batch_size} chunks per batch...")
+ # Set fast mode for batch processing
+ os.environ["GLOSSARY_SKIP_ALL_VALIDATION"] = "1"
+
+ # Use batch API calls for AI extraction
+ all_csv_lines = self._process_chunks_batch_api(
+ chunks_to_process, custom_prompt, language,
+ min_frequency, max_names, max_titles,
+ output_dir, strip_honorifics, fuzzy_threshold,
+ filter_mode, api_batch_size, extraction_workers
+ )
+
+ # Reset validation mode
+ os.environ["GLOSSARY_SKIP_ALL_VALIDATION"] = "0"
+
+ print(f"📑 All chunks completed. Aggregated raw lines: {len(all_csv_lines)}")
+
+ # Process all collected entries at once (even if empty)
+ # Add header so downstream steps can work uniformly
+ all_csv_lines.insert(0, "type,raw_name,translated_name")
+
+ # Merge with any on-disk glossary first (to avoid overwriting user edits)
+ on_disk_path = os.path.join(output_dir, "glossary.csv")
+ if os.path.exists(on_disk_path):
+ try:
+ with open(on_disk_path, 'r', encoding='utf-8') as f:
+ on_disk_content = f.read()
+ all_csv_lines = self._merge_csv_entries(all_csv_lines, on_disk_content, strip_honorifics, language)
+ print("📑 Merged with existing on-disk glossary")
+ except Exception as e:
+ print(f"⚠️ Failed to merge with existing on-disk glossary: {e}")
+
+ # Apply filter mode if needed
+ if filter_mode == "only_with_honorifics":
+ filtered = [all_csv_lines[0]] # Keep header
+ for line in all_csv_lines[1:]:
+ parts = line.split(',', 2)
+ if len(parts) >= 3 and parts[0] == "character":
+ filtered.append(line)
+ all_csv_lines = filtered
+ print(f"📑 Filter applied: {len(all_csv_lines)-1} character entries with honorifics kept")
+
+ # Apply fuzzy deduplication (deferred until after all chunks)
+ try:
+ print(f"📑 Applying fuzzy deduplication (threshold: {fuzzy_threshold})...")
+ all_csv_lines = self._deduplicate_glossary_with_fuzzy(all_csv_lines, fuzzy_threshold)
+ except Exception as e:
+ print(f"⚠️ Deduplication error: {e} — continuing without dedup")
+
+ # Sort by type and name
+ print(f"📑 Sorting glossary by type and name...")
+ header = all_csv_lines[0]
+ entries = all_csv_lines[1:]
+ if entries:
+ entries.sort(key=lambda x: (0 if x.startswith('character,') else 1, x.split(',')[1].lower()))
+ all_csv_lines = [header] + entries
+
+ # Save
+ # Check format preference
+ use_legacy_format = os.getenv('GLOSSARY_USE_LEGACY_CSV', '0') == '1'
+
+ if not use_legacy_format:
+ # Convert to token-efficient format
+ all_csv_lines = self._convert_to_token_efficient_format(all_csv_lines)
+
+ # Final sanitize to prevent stray headers
+ all_csv_lines = self._sanitize_final_glossary_lines(all_csv_lines, use_legacy_format)
+
+ # Save
+ csv_content = '\n'.join(all_csv_lines)
+ glossary_path = os.path.join(output_dir, "glossary.csv")
+ self._atomic_write_file(glossary_path, csv_content)
+
+ # Verify file exists; fallback direct write if needed
+ if not os.path.exists(glossary_path):
+ try:
+ with open(glossary_path, 'w', encoding='utf-8') as f:
+ f.write(csv_content)
+ print("📑 Fallback write succeeded for glossary.csv")
+ except Exception as e:
+ print(f"❌ Failed to write glossary.csv: {e}")
+
+ print(f"\n📑 ✅ GLOSSARY SAVED!")
+ print(f"📑 ✅ AI GLOSSARY SAVED!")
+ c_count, t_count, total = self._count_glossary_entries(all_csv_lines, use_legacy_format)
+ print(f"📑 Character entries: {c_count}")
+ print(f"📑 Term entries: {t_count}")
+ print(f"📑 Total entries: {total}")
+
+ return self._parse_csv_to_dict(csv_content)
+ else:
+ # Strict sequential processing (one API call at a time)
+ _prev_defer = os.getenv("GLOSSARY_DEFER_SAVE")
+ _prev_filtered = os.getenv("_CHUNK_ALREADY_FILTERED")
+ _prev_force_disable = os.getenv("GLOSSARY_FORCE_DISABLE_SMART_FILTER")
+ os.environ["GLOSSARY_DEFER_SAVE"] = "1"
+ # Tell the extractor each chunk is already filtered to avoid re-running smart filter per chunk
+ os.environ["_CHUNK_ALREADY_FILTERED"] = "1"
+ os.environ["GLOSSARY_FORCE_DISABLE_SMART_FILTER"] = "1"
+ try:
+ for chunk_idx, chunk_text in chunks_to_process:
+ if is_stop_requested():
+ break
+
+ print(f"📑 Processing chunk {chunk_idx}/{len(chunks_to_process)} ({len(chunk_text):,} chars)...")
+
+ if custom_prompt:
+ chunk_glossary = self._extract_with_custom_prompt(
+ custom_prompt, chunk_text, language,
+ min_frequency, max_names, max_titles,
+ None, output_dir, # Don't pass existing glossary to chunks
+ strip_honorifics, fuzzy_threshold, filter_mode
+ )
+ else:
+ chunk_glossary = self._extract_with_patterns(
+ chunk_text, language, min_frequency,
+ max_names, max_titles, batch_size,
+ None, output_dir, # Don't pass existing glossary to chunks
+ strip_honorifics, fuzzy_threshold, filter_mode
+ )
+
+ # Normalize to CSV lines and aggregate
+ chunk_lines = []
+ if isinstance(chunk_glossary, list):
+ for line in chunk_glossary:
+ if line and not line.startswith('type,'):
+ all_glossary_entries.append(line)
+ chunk_lines.append(line)
+ else:
+ for raw_name, translated_name in chunk_glossary.items():
+ entry_type = "character" if self._has_honorific(raw_name) else "term"
+ line = f"{entry_type},{raw_name},{translated_name}"
+ all_glossary_entries.append(line)
+ chunk_lines.append(line)
+
+ # Incremental update
+ try:
+ self._incremental_update_glossary(output_dir, chunk_lines, strip_honorifics, language, filter_mode)
+ print(f"📑 Incremental write: +{len(chunk_lines)} entries")
+ except Exception as e2:
+ print(f"⚠️ Incremental write failed: {e2}")
+ finally:
+ if _prev_defer is None:
+ if "GLOSSARY_DEFER_SAVE" in os.environ:
+ del os.environ["GLOSSARY_DEFER_SAVE"]
+ else:
+ os.environ["GLOSSARY_DEFER_SAVE"] = _prev_defer
+ if _prev_filtered is None:
+ os.environ.pop("_CHUNK_ALREADY_FILTERED", None)
+ else:
+ os.environ["_CHUNK_ALREADY_FILTERED"] = _prev_filtered
+ if _prev_force_disable is None:
+ os.environ.pop("GLOSSARY_FORCE_DISABLE_SMART_FILTER", None)
+ else:
+ os.environ["GLOSSARY_FORCE_DISABLE_SMART_FILTER"] = _prev_force_disable
+
+ # Build CSV from aggregated entries
+ csv_lines = ["type,raw_name,translated_name"] + all_glossary_entries
+
+ # Merge with any provided existing glossary AND on-disk glossary to avoid overwriting
+ on_disk_path = os.path.join(output_dir, "glossary.csv")
+ merge_sources = []
+ if existing_glossary:
+ merge_sources.append(existing_glossary)
+ if os.path.exists(on_disk_path):
+ try:
+ with open(on_disk_path, 'r', encoding='utf-8') as f:
+ merge_sources.append(f.read())
+ print("📑 Found existing on-disk glossary to merge")
+ except Exception as e:
+ print(f"⚠️ Failed to read on-disk glossary for merging: {e}")
+ # Also merge the main on-disk glossary if it was present at start
+ if existing_glossary_content:
+ csv_lines = self._merge_csv_entries(csv_lines, existing_glossary_content, strip_honorifics, language)
+ for src in merge_sources:
+ csv_lines = self._merge_csv_entries(csv_lines, src, strip_honorifics, language)
+
+ # Apply filter mode to final results
+ csv_lines = self._filter_csv_by_mode(csv_lines, filter_mode)
+
+ # Apply fuzzy deduplication (deferred until after all chunks)
+ print(f"📑 Applying fuzzy deduplication (threshold: {fuzzy_threshold})...")
+ original_count = len(csv_lines) - 1
+ csv_lines = self._deduplicate_glossary_with_fuzzy(csv_lines, fuzzy_threshold)
+ deduped_count = len(csv_lines) - 1
+ if original_count > deduped_count:
+ print(f"📑 Removed {original_count - deduped_count} duplicate entries")
+
+ # Sort by type and name
+ print(f"📑 Sorting glossary by type and name...")
+ header = csv_lines[0]
+ entries = csv_lines[1:]
+ entries.sort(key=lambda x: (0 if x.startswith('character,') else 1, x.split(',')[1].lower() if ',' in x else x.lower()))
+ csv_lines = [header] + entries
+
+ # Token-efficient format if enabled
+ use_legacy_format = os.getenv('GLOSSARY_USE_LEGACY_CSV', '0') == '1'
+ if not use_legacy_format:
+ csv_lines = self._convert_to_token_efficient_format(csv_lines)
+
+ # Final sanitize to prevent stray headers and section titles at end
+ csv_lines = self._sanitize_final_glossary_lines(csv_lines, use_legacy_format)
+
+ try:
+ # Save
+ csv_content = '\n'.join(csv_lines)
+ glossary_path = os.path.join(output_dir, "glossary.csv")
+ self._atomic_write_file(glossary_path, csv_content)
+
+ # Verify file exists; fallback direct write if needed
+ if not os.path.exists(glossary_path):
+ try:
+ with open(glossary_path, 'w', encoding='utf-8') as f:
+ f.write(csv_content)
+ print("📑 Fallback write succeeded for glossary.csv")
+ except Exception as e:
+ print(f"❌ Failed to write glossary.csv: {e}")
+ finally:
+ print(f"\n📑 ✅ CHUNKED GLOSSARY SAVED!")
+ print(f"📑 ✅ AI GLOSSARY SAVED!")
+ print(f"📑 File: {glossary_path}")
+ c_count, t_count, total = self._count_glossary_entries(csv_lines, use_legacy_format)
+ print(f"📑 Character entries: {c_count}")
+ print(f"📑 Term entries: {t_count}")
+ print(f"📑 Total entries: {total}")
+
+ return self._parse_csv_to_dict(csv_content)
+
+ # Original single-text processing
+ if custom_prompt:
+ return self._extract_with_custom_prompt(custom_prompt, all_text, language,
+ min_frequency, max_names, max_titles,
+ existing_glossary, output_dir,
+ strip_honorifics, fuzzy_threshold, filter_mode)
+ else:
+ return self._extract_with_patterns(all_text, language, min_frequency,
+ max_names, max_titles, batch_size,
+ existing_glossary, output_dir,
+ strip_honorifics, fuzzy_threshold, filter_mode)
+
+ total_time = time.time() - total_start_time
+ print(f"\n📑 ========== GLOSSARY GENERATION COMPLETE ==========")
+ print(f"📑 Total time: {total_time:.1f}s")
+ print(f"📑 Performance breakdown:")
+ print(f"📑 - Extraction: {getattr(self, '_extraction_time', 0):.1f}s")
+ print(f"📑 - API calls: {getattr(self, '_api_time', 0):.1f}s")
+ print(f"📑 - Frequency checking: {getattr(self, '_freq_check_time', 0):.1f}s")
+ print(f"📑 - Deduplication: {getattr(self, '_dedup_time', 0):.1f}s")
+ print(f"📑 - File I/O: {getattr(self, '_io_time', 0):.1f}s")
+ print(f"📑 ================================================")
+
+ return result # This is the existing return statement
+
+ def _convert_to_token_efficient_format(self, csv_lines):
+ """Convert CSV lines to token-efficient format with sections and asterisks"""
+ if len(csv_lines) <= 1:
+ return csv_lines
+
+ header = csv_lines[0]
+ entries = csv_lines[1:]
+
+ # Group by type (only from valid CSV lines)
+ import re as _re
+ grouped = {}
+ for line in entries:
+ if not line.strip():
+ continue
+ # Only accept proper CSV rows: at least 3 fields and a sane type token
+ parts_full = [p.strip() for p in line.split(',')]
+ if len(parts_full) < 3:
+ continue
+ entry_type = parts_full[0].lower()
+ if not _re.match(r'^[a-z_]+$', entry_type):
+ continue
+ if entry_type not in grouped:
+ grouped[entry_type] = []
+ grouped[entry_type].append(line)
+
+ # Rebuild with token-efficient format
+ result = []
+ result.append("Glossary: Characters, Terms, and Important Elements\n")
+
+ # Process in order: character first, then term, then others
+ type_order = ['character', 'term'] + [t for t in grouped.keys() if t not in ['character', 'term']]
+
+ for entry_type in type_order:
+ if entry_type not in grouped:
+ continue
+
+ entries = grouped[entry_type]
+
+ # Add section header
+ section_name = entry_type.upper() + 'S' if not entry_type.upper().endswith('S') else entry_type.upper()
+ result.append(f"=== {section_name} ===")
+
+ # Add entries in new format
+ for line in entries:
+ parts = [p.strip() for p in line.split(',')]
+ if len(parts) >= 3:
+ raw_name = parts[1]
+ translated_name = parts[2]
+
+ # Format: * TranslatedName (RawName)
+ entry_line = f"* {translated_name} ({raw_name})"
+
+ # Add gender if present and not Unknown
+ if len(parts) > 3 and parts[3] and parts[3] != 'Unknown':
+ entry_line += f" [{parts[3]}]"
+
+ # Add any additional fields as description
+ if len(parts) > 4:
+ description = ', '.join(parts[4:])
+ if description.strip():
+ entry_line += f": {description}"
+
+ result.append(entry_line)
+
+ result.append("") # Blank line between sections
+
+ return result
+
+ def _count_glossary_entries(self, lines, use_legacy_format=False):
+ """Return (char_count, term_count, total_count) for either format."""
+ if not lines:
+ return 0, 0, 0
+ if use_legacy_format:
+ data = lines[1:] if lines and lines[0].lower().startswith('type,raw_name') else lines
+ char_count = sum(1 for ln in data if ln.startswith('character,'))
+ term_count = sum(1 for ln in data if ln.startswith('term,'))
+ total = sum(1 for ln in data if ln and ',' in ln)
+ return char_count, term_count, total
+ # token-efficient
+ current = None
+ char_count = term_count = total = 0
+ for ln in lines:
+ s = ln.strip()
+ if s.startswith('=== ') and 'CHARACTER' in s.upper():
+ current = 'character'
+ continue
+ if s.startswith('=== ') and 'TERM' in s.upper():
+ current = 'term'
+ continue
+ if s.startswith('* '):
+ total += 1
+ if current == 'character':
+ char_count += 1
+ elif current == 'term':
+ term_count += 1
+ return char_count, term_count, total
+
+ def _sanitize_final_glossary_lines(self, lines, use_legacy_format=False):
+ """Remove stray CSV headers and normalize header placement before saving.
+ - In legacy CSV mode, ensure exactly one header at the very top.
+ - In token-efficient mode, remove any CSV header lines entirely.
+ """
+ header_norm = "type,raw_name,translated_name"
+ if not lines:
+ return lines
+
+ if use_legacy_format:
+ sanitized = []
+ header_seen = False
+ for ln in lines:
+ txt = ln.strip()
+ if txt.lower().startswith("type,raw_name"):
+ if not header_seen:
+ sanitized.append(header_norm)
+ header_seen = True
+ # skip duplicates
+ else:
+ sanitized.append(ln)
+ # ensure header at top
+ if sanitized and not sanitized[0].strip().lower().startswith("type,raw_name"):
+ sanitized.insert(0, header_norm)
+ return sanitized
+ else:
+ # remove any CSV header lines anywhere and duplicate top headers/sections
+ cleaned = []
+ glossary_header_seen = False
+ for i, ln in enumerate(lines):
+ txt = ln.strip()
+ low = txt.lower()
+ # Drop CSV headers
+ if low.startswith("type,raw_name"):
+ continue
+ # Keep only the first main glossary header
+ if low.startswith("glossary:"):
+ if glossary_header_seen:
+ continue
+ glossary_header_seen = True
+ cleaned.append(ln)
+ continue
+ # Remove bogus section like '=== GLOSSARY: ... ==='
+ if low.startswith("=== glossary:"):
+ continue
+ cleaned.append(ln)
+ return cleaned
+
+ def _process_chunks_batch_api(self, chunks_to_process, custom_prompt, language,
+ min_frequency, max_names, max_titles,
+ output_dir, strip_honorifics, fuzzy_threshold,
+ filter_mode, api_batch_size, extraction_workers):
+ """Process chunks using batch API calls for AI extraction with thread delay"""
+
+ print(f"📑 Using batch API mode with {api_batch_size} chunks per batch")
+
+ # Ensure we defer saving and heavy merging when processing chunks
+ _prev_defer = os.getenv("GLOSSARY_DEFER_SAVE")
+ os.environ["GLOSSARY_DEFER_SAVE"] = "1"
+
+ # Get thread submission delay
+ thread_delay = float(os.getenv("THREAD_SUBMISSION_DELAY_SECONDS", "0.5"))
+ if thread_delay > 0:
+ print(f"📑 Thread submission delay: {thread_delay}s between parallel calls")
+
+ # CHANGE: Collect raw CSV lines instead of dictionary
+ all_csv_lines = [] # Collect all entries as CSV lines
+ total_chunks = len(chunks_to_process)
+ completed_chunks = 0
+
+ # Ensure per-chunk smart filtering is disabled globally during batch processing
+ _prev_filtered = os.getenv("_CHUNK_ALREADY_FILTERED")
+ _prev_force_disable = os.getenv("GLOSSARY_FORCE_DISABLE_SMART_FILTER")
+ os.environ["_CHUNK_ALREADY_FILTERED"] = "1"
+ os.environ["GLOSSARY_FORCE_DISABLE_SMART_FILTER"] = "1"
+
+ # Process in API batches
+ for batch_start in range(0, len(chunks_to_process), api_batch_size):
+ if is_stop_requested():
+ break
+
+ batch_end = min(batch_start + api_batch_size, len(chunks_to_process))
+ batch_chunks = chunks_to_process[batch_start:batch_end]
+
+ print(f"📑 Processing API batch {batch_start//api_batch_size + 1}: chunks {batch_start+1}-{batch_end}")
+
+ # Use ThreadPoolExecutor for parallel API calls within batch
+ # Batch mode: issue multiple API calls in parallel within each batch (one worker per chunk)
+ with ThreadPoolExecutor(max_workers=len(batch_chunks)) as executor:
+ futures = {}
+ last_submission_time = 0
+
+ for chunk_idx, chunk_text in batch_chunks:
+ if is_stop_requested():
+ break
+
+ # Apply thread submission delay
+ if thread_delay > 0 and last_submission_time > 0:
+ time_since_last = time.time() - last_submission_time
+ if time_since_last < thread_delay:
+ sleep_time = thread_delay - time_since_last
+ print(f"🧵 Thread delay: {sleep_time:.1f}s for chunk {chunk_idx}")
+ time.sleep(sleep_time)
+
+ future = executor.submit(
+ self._extract_with_custom_prompt,
+ custom_prompt, chunk_text, language,
+ min_frequency, max_names, max_titles,
+ None, output_dir, strip_honorifics,
+ fuzzy_threshold, filter_mode
+ )
+ futures[future] = chunk_idx
+ last_submission_time = time.time()
+
+ # Collect results
+ for future in as_completed(futures):
+ if is_stop_requested():
+ break
+
+ try:
+ chunk_glossary = future.result()
+ print(f"📑 DEBUG: Chunk {futures[future]} returned type={type(chunk_glossary)}, len={len(chunk_glossary)}")
+
+ # Normalize to CSV lines (without header)
+ chunk_lines = []
+ if isinstance(chunk_glossary, dict):
+ for raw_name, translated_name in chunk_glossary.items():
+ entry_type = "character" if self._has_honorific(raw_name) else "term"
+ chunk_lines.append(f"{entry_type},{raw_name},{translated_name}")
+ elif isinstance(chunk_glossary, list):
+ for line in chunk_glossary:
+ if line and not line.startswith('type,'):
+ chunk_lines.append(line)
+
+ # Aggregate for end-of-run
+ all_csv_lines.extend(chunk_lines)
+
+ # Incremental update of glossary.csv in token-efficient format
+ try:
+ self._incremental_update_glossary(output_dir, chunk_lines, strip_honorifics, language, filter_mode)
+ print(f"📑 Incremental write: +{len(chunk_lines)} entries")
+ except Exception as e2:
+ print(f"⚠️ Incremental write failed: {e2}")
+
+ completed_chunks += 1
+
+ # Print progress for GUI
+ progress_percent = (completed_chunks / total_chunks) * 100
+ print(f"📑 Progress: {completed_chunks}/{total_chunks} chunks ({progress_percent:.0f}%)")
+ print(f"📑 Chunk {futures[future]} completed and aggregated")
+
+ except Exception as e:
+ print(f"⚠️ API call for chunk {futures[future]} failed: {e}")
+ completed_chunks += 1
+ progress_percent = (completed_chunks / total_chunks) * 100
+ print(f"📑 Progress: {completed_chunks}/{total_chunks} chunks ({progress_percent:.0f}%)")
+
+ # Add delay between API batches
+ if batch_end < len(chunks_to_process):
+ api_delay = float(os.getenv("SEND_INTERVAL_SECONDS", "2"))
+ print(f"⏱️ Waiting {api_delay}s before next API batch...")
+ time.sleep(api_delay)
+
+ # CHANGE: Return CSV lines instead of dictionary
+
+ # Restore per-chunk filter disabling envs
+ if _prev_filtered is None:
+ os.environ.pop("_CHUNK_ALREADY_FILTERED", None)
+ else:
+ os.environ["_CHUNK_ALREADY_FILTERED"] = _prev_filtered
+ if _prev_force_disable is None:
+ os.environ.pop("GLOSSARY_FORCE_DISABLE_SMART_FILTER", None)
+ else:
+ os.environ["GLOSSARY_FORCE_DISABLE_SMART_FILTER"] = _prev_force_disable
+
+ # Restore previous defer setting
+ if _prev_defer is None:
+ # Default back to not deferring if it wasn't set
+ if "GLOSSARY_DEFER_SAVE" in os.environ:
+ del os.environ["GLOSSARY_DEFER_SAVE"]
+ else:
+ os.environ["GLOSSARY_DEFER_SAVE"] = _prev_defer
+
+ return all_csv_lines
+
+ def _incremental_update_glossary(self, output_dir, chunk_lines, strip_honorifics, language, filter_mode):
+ """Incrementally update glossary.csv (token-efficient) using an on-disk CSV aggregator.
+ This keeps glossary.csv present and growing after each chunk while preserving
+ token-efficient format for the visible file.
+ """
+ if not chunk_lines:
+ return
+ # Paths
+ agg_path = os.path.join(output_dir, "glossary.incremental.csv")
+ vis_path = os.path.join(output_dir, "glossary.csv")
+ # Ensure output dir
+ os.makedirs(output_dir, exist_ok=True)
+ # Compose CSV with header for merging
+ new_csv_lines = ["type,raw_name,translated_name"] + chunk_lines
+ # Load existing aggregator content, if any
+ existing_csv = None
+ if os.path.exists(agg_path):
+ try:
+ with open(agg_path, 'r', encoding='utf-8') as f:
+ existing_csv = f.read()
+ except Exception as e:
+ print(f"⚠️ Incremental: cannot read aggregator: {e}")
+ # Merge (exact merge, no fuzzy to keep this fast)
+ merged_csv_lines = self._merge_csv_entries(new_csv_lines, existing_csv or "", strip_honorifics, language)
+ # Optional filter mode
+ merged_csv_lines = self._filter_csv_by_mode(merged_csv_lines, filter_mode)
+ # Save aggregator (CSV)
+ self._atomic_write_file(agg_path, "\n".join(merged_csv_lines))
+ # Convert to token-efficient format for visible glossary.csv
+ token_lines = self._convert_to_token_efficient_format(merged_csv_lines)
+ token_lines = self._sanitize_final_glossary_lines(token_lines, use_legacy_format=False)
+ self._atomic_write_file(vis_path, "\n".join(token_lines))
+ if not os.path.exists(vis_path):
+ with open(vis_path, 'w', encoding='utf-8') as f:
+ f.write("\n".join(token_lines))
+
+ def _process_single_chunk(self, chunk_idx, chunk_text, custom_prompt, language,
+ min_frequency, max_names, max_titles, batch_size,
+ output_dir, strip_honorifics, fuzzy_threshold, filter_mode,
+ already_filtered=False):
+ """Process a single chunk - wrapper for parallel execution"""
+ print(f"📑 Worker processing chunk {chunk_idx} ({len(chunk_text):,} chars)...")
+
+ if custom_prompt:
+ # Pass flag to indicate if text is already filtered
+ os.environ["_CHUNK_ALREADY_FILTERED"] = "1" if already_filtered else "0"
+ _prev_defer = os.getenv("GLOSSARY_DEFER_SAVE")
+ os.environ["GLOSSARY_DEFER_SAVE"] = "1"
+ try:
+ result = self._extract_with_custom_prompt(
+ custom_prompt, chunk_text, language,
+ min_frequency, max_names, max_titles,
+ None, output_dir,
+ strip_honorifics, fuzzy_threshold, filter_mode
+ )
+ finally:
+ os.environ["_CHUNK_ALREADY_FILTERED"] = "0" # Reset
+ if _prev_defer is None:
+ if "GLOSSARY_DEFER_SAVE" in os.environ:
+ del os.environ["GLOSSARY_DEFER_SAVE"]
+ else:
+ os.environ["GLOSSARY_DEFER_SAVE"] = _prev_defer
+ return result
+ else:
+ return self._extract_with_patterns(
+ chunk_text, language, min_frequency,
+ max_names, max_titles, batch_size,
+ None, output_dir,
+ strip_honorifics, fuzzy_threshold, filter_mode
+ )
+
+ def _apply_final_filter(self, entries, filter_mode):
+ """Apply final filtering based on mode to ensure only requested types are included"""
+ if filter_mode == "only_with_honorifics":
+ # Filter to keep only entries that look like they have honorifics
+ filtered = {}
+ for key, value in entries.items():
+ # Check if the key contains known honorific patterns
+ if self._has_honorific(key):
+ filtered[key] = value
+ print(f"📑 Final filter: Kept {len(filtered)} entries with honorifics (from {len(entries)} total)")
+ return filtered
+ elif filter_mode == "only_without_honorifics":
+ # Filter to keep only entries without honorifics
+ filtered = {}
+ for key, value in entries.items():
+ if not self._has_honorific(key):
+ filtered[key] = value
+ print(f"📑 Final filter: Kept {len(filtered)} entries without honorifics (from {len(entries)} total)")
+ return filtered
+ else:
+ return entries
+
+ def _looks_like_name(self, text):
+ """Check if text looks like a character name"""
+ if not text:
+ return False
+
+ # Check for various name patterns
+ # Korean names (2-4 hangul characters)
+ if all(0xAC00 <= ord(char) <= 0xD7AF for char in text) and 2 <= len(text) <= 4:
+ return True
+
+ # Japanese names (mix of kanji/kana, 2-6 chars)
+ has_kanji = any(0x4E00 <= ord(char) <= 0x9FFF for char in text)
+ has_kana = any((0x3040 <= ord(char) <= 0x309F) or (0x30A0 <= ord(char) <= 0x30FF) for char in text)
+ if (has_kanji or has_kana) and 2 <= len(text) <= 6:
+ return True
+
+ # Chinese names (2-4 Chinese characters)
+ if all(0x4E00 <= ord(char) <= 0x9FFF for char in text) and 2 <= len(text) <= 4:
+ return True
+
+ # English names (starts with capital, mostly letters)
+ if text[0].isupper() and sum(1 for c in text if c.isalpha()) >= len(text) * 0.8:
+ return True
+
+ return False
+
+ def _has_honorific(self, term):
+ """Check if a term contains an honorific using PatternManager's comprehensive list"""
+ if not term:
+ return False
+
+ term_lower = term.lower()
+
+ # Check all language honorifics from PatternManager
+ for language, honorifics_list in self.pattern_manager.CJK_HONORIFICS.items():
+ for honorific in honorifics_list:
+ # For romanized/English honorifics with spaces or dashes
+ if honorific.startswith(' ') or honorific.startswith('-'):
+ if term_lower.endswith(honorific.lower()):
+ return True
+ # For CJK honorifics (no separator)
+ else:
+ if honorific in term:
+ return True
+
+ return False
+
+ def _strip_all_honorifics(self, term, language='korean'):
+ """Strip all honorifics from a term using PatternManager's lists"""
+ if not term:
+ return term
+
+ result = term
+
+ # Get honorifics for the specific language and English romanizations
+ honorifics_to_strip = []
+ if language in self.pattern_manager.CJK_HONORIFICS:
+ honorifics_to_strip.extend(self.pattern_manager.CJK_HONORIFICS[language])
+ honorifics_to_strip.extend(self.pattern_manager.CJK_HONORIFICS.get('english', []))
+
+ # Sort by length (longest first) to avoid partial matches
+ honorifics_to_strip.sort(key=len, reverse=True)
+
+ # Strip honorifics
+ for honorific in honorifics_to_strip:
+ if honorific.startswith(' ') or honorific.startswith('-'):
+ # For romanized honorifics with separators
+ if result.lower().endswith(honorific.lower()):
+ result = result[:-len(honorific)]
+ else:
+ # For CJK honorifics (no separator)
+ if result.endswith(honorific):
+ result = result[:-len(honorific)]
+
+ return result.strip()
+
+ def _convert_to_csv_format(self, data):
+ """Convert various glossary formats to CSV string format with enforced 3 columns"""
+ csv_lines = ["type,raw_name,translated_name"]
+
+ if isinstance(data, str):
+ # Already CSV string
+ if data.strip().startswith('type,raw_name'):
+ return data
+ # Try to parse as JSON
+ try:
+ data = json.loads(data)
+ except:
+ return data
+
+ if isinstance(data, list):
+ for item in data:
+ if isinstance(item, dict):
+ if 'type' in item and 'raw_name' in item:
+ # Already in correct format
+ line = f"{item['type']},{item['raw_name']},{item.get('translated_name', item['raw_name'])}"
+ csv_lines.append(line)
+ else:
+ # Old format - default to 'term' type
+ entry_type = 'term'
+ raw_name = item.get('original_name', '')
+ translated_name = item.get('name', raw_name)
+ if raw_name and translated_name:
+ csv_lines.append(f"{entry_type},{raw_name},{translated_name}")
+
+ elif isinstance(data, dict):
+ if 'entries' in data:
+ # Has metadata wrapper, extract entries
+ for original, translated in data['entries'].items():
+ csv_lines.append(f"term,{original},{translated}")
+ else:
+ # Plain dictionary - default to 'term' type
+ for original, translated in data.items():
+ csv_lines.append(f"term,{original},{translated}")
+
+ return '\n'.join(csv_lines)
+
+ def _parse_csv_to_dict(self, csv_content):
+ """Parse CSV content to dictionary for backward compatibility"""
+ result = {}
+ lines = csv_content.strip().split('\n')
+
+ for line in lines[1:]: # Skip header
+ if not line.strip():
+ continue
+ parts = [p.strip() for p in line.split(',')]
+ if len(parts) >= 3:
+ result[parts[1]] = parts[2] # raw_name -> translated_name
+
+ return result
+
+ def _fuzzy_match(self, term1, term2, threshold=0.90):
+ """Check if two terms match using fuzzy matching"""
+ ratio = SequenceMatcher(None, term1.lower(), term2.lower()).ratio()
+ return ratio >= threshold
+
+ def _fuzzy_match_rapidfuzz(self, term_lower, text_lower, threshold, term_len):
+ """Use rapidfuzz library for MUCH faster fuzzy matching"""
+ from rapidfuzz import fuzz
+
+ print(f"📑 Using RapidFuzz (C++ speed)...")
+ start_time = time.time()
+
+ matches_count = 0
+ threshold_percent = threshold * 100 # rapidfuzz uses 0-100 scale
+
+ # Can use smaller step because rapidfuzz is so fast
+ step = 1 # Check every position - rapidfuzz can handle it
+
+ # Process text
+ for i in range(0, len(text_lower) - term_len + 1, step):
+ # Check stop flag every 10000 positions
+ if i > 0 and i % 10000 == 0:
+ if is_stop_requested():
+ print(f"📑 RapidFuzz stopped at position {i}")
+ return matches_count
+
+ window = text_lower[i:i + term_len]
+
+ # rapidfuzz is fast enough we can check every position
+ if fuzz.ratio(term_lower, window) >= threshold_percent:
+ matches_count += 1
+
+ elapsed = time.time() - start_time
+ print(f"📑 RapidFuzz found {matches_count} matches in {elapsed:.2f}s")
+ return matches_count
+
+ def _batch_compute_frequencies(self, terms, all_text, fuzzy_threshold=0.90, min_frequency=2):
+ """Compute frequencies for all terms at once - MUCH faster than individual checking"""
+ print(f"📑 Computing frequencies for {len(terms)} terms in batch mode...")
+ start_time = time.time()
+
+ # Result dictionary
+ term_frequencies = {}
+
+ # First pass: exact matching (very fast)
+ print(f"📑 Phase 1: Exact matching...")
+ text_lower = all_text.lower()
+ for term in terms:
+ if is_stop_requested():
+ return term_frequencies
+ term_lower = term.lower()
+ count = text_lower.count(term_lower)
+ term_frequencies[term] = count
+
+ exact_time = time.time() - start_time
+ high_freq_terms = sum(1 for count in term_frequencies.values() if count >= min_frequency)
+ print(f"📑 Exact matching complete: {high_freq_terms}/{len(terms)} terms meet threshold ({exact_time:.1f}s)")
+
+ # If fuzzy matching is disabled, we're done
+ if fuzzy_threshold >= 1.0:
+ return term_frequencies
+
+ # Second pass: fuzzy matching ONLY for low-frequency terms
+ low_freq_terms = [term for term, count in term_frequencies.items() if count < min_frequency]
+
+ if low_freq_terms:
+ print(f"📑 Phase 2: Fuzzy matching for {len(low_freq_terms)} low-frequency terms...")
+
+ # Try to use RapidFuzz batch processing
+ try:
+ from rapidfuzz import process, fuzz
+
+ # For very large texts, sample it for fuzzy matching
+ if len(text_lower) > 500000:
+ print(f"📑 Text too large ({len(text_lower):,} chars), sampling for fuzzy matching...")
+ # Sample every Nth character to reduce size
+ sample_rate = max(1, len(text_lower) // 100000)
+ sampled_text = text_lower[::sample_rate]
+ else:
+ sampled_text = text_lower
+
+ # Create chunks of text for fuzzy matching
+ chunk_size = 1000 # Process text in chunks
+ text_chunks = [sampled_text[i:i+chunk_size] for i in range(0, len(sampled_text), chunk_size//2)] # Overlapping chunks
+
+ print(f"📑 Processing {len(text_chunks)} text chunks...")
+ threshold_percent = fuzzy_threshold * 100
+
+ # Process in batches to avoid memory issues
+ batch_size = 100 # Process 100 terms at a time
+ for batch_start in range(0, len(low_freq_terms), batch_size):
+ if is_stop_requested():
+ break
+
+ batch_end = min(batch_start + batch_size, len(low_freq_terms))
+ batch_terms = low_freq_terms[batch_start:batch_end]
+
+ for term in batch_terms:
+ if is_stop_requested():
+ break
+
+ # Quick fuzzy search in chunks
+ fuzzy_count = 0
+ for chunk in text_chunks[:50]: # Limit to first 50 chunks for speed
+ if fuzz.partial_ratio(term.lower(), chunk) >= threshold_percent:
+ fuzzy_count += 1
+
+ if fuzzy_count > 0:
+ # Scale up based on sampling
+ if len(text_lower) > 500000:
+ fuzzy_count *= (len(text_lower) // len(sampled_text))
+ term_frequencies[term] += fuzzy_count
+
+ if (batch_end % 500 == 0) or (batch_end == len(low_freq_terms)):
+ elapsed = time.time() - start_time
+ print(f"📑 Processed {batch_end}/{len(low_freq_terms)} terms ({elapsed:.1f}s)")
+
+ except ImportError:
+ print("📑 RapidFuzz not available, skipping fuzzy matching")
+
+ total_time = time.time() - start_time
+ final_high_freq = sum(1 for count in term_frequencies.values() if count >= min_frequency)
+ print(f"📑 Batch frequency computation complete: {final_high_freq}/{len(terms)} terms accepted ({total_time:.1f}s)")
+
+ return term_frequencies
+
+ def _find_fuzzy_matches(self, term, text, threshold=0.90):
+ """Find fuzzy matches of a term in text using efficient method with parallel processing"""
+ start_time = time.time()
+
+ term_lower = term.lower()
+ text_lower = text.lower()
+ term_len = len(term)
+
+ # Only log for debugging if explicitly enabled
+ debug_search = os.getenv("GLOSSARY_DEBUG_SEARCH", "0") == "1"
+ if debug_search and len(text) > 100000:
+ print(f"📑 Searching for '{term}' in {len(text):,} chars (threshold: {threshold})")
+
+ # Strategy 1: Use exact matching first for efficiency
+ exact_start = time.time()
+ matches_count = text_lower.count(term_lower)
+ exact_time = time.time() - exact_start
+
+ if matches_count > 0:
+ if debug_search and len(text) > 100000:
+ print(f"📑 Found {matches_count} exact matches in {exact_time:.3f}s")
+ return matches_count
+
+ # Strategy 2: Try rapidfuzz if available (much faster)
+ if matches_count == 0 and threshold < 1.0:
+ try:
+ from rapidfuzz import fuzz
+ return self._fuzzy_match_rapidfuzz(term_lower, text_lower, threshold, term_len)
+ except ImportError:
+ pass # Fall back to parallel/sequential
+
+ # Strategy 3: Fall back to parallel/sequential if rapidfuzz not available
+ # Check if parallel processing is enabled
+ extraction_workers = int(os.getenv("EXTRACTION_WORKERS", "1"))
+
+ if extraction_workers > 1 and len(text) > 50000: # Use parallel for large texts
+ return self._parallel_fuzzy_search(term_lower, text_lower, threshold, term_len, extraction_workers)
+ else:
+ return self._sequential_fuzzy_search(term_lower, text_lower, threshold, term_len)
+ # Check if parallel processing is enabled
+ extraction_workers = int(os.getenv("EXTRACTION_WORKERS", "1"))
+
+ if extraction_workers > 1 and len(text) > 50000: # Use parallel for large texts
+ return self._parallel_fuzzy_search(term_lower, text_lower, threshold, term_len, extraction_workers)
+ else:
+ return self._sequential_fuzzy_search(term_lower, text_lower, threshold, term_len)
+
+ return matches_count
+
+ def _parallel_fuzzy_search(self, term_lower, text_lower, threshold, term_len, num_workers):
+ """Parallel fuzzy search using ThreadPoolExecutor"""
+ print(f"📑 Starting parallel fuzzy search with {num_workers} workers...")
+
+ text_len = len(text_lower)
+ matches_count = 0
+
+ # Split text into overlapping chunks for parallel processing
+ chunk_size = max(text_len // num_workers, term_len * 100)
+ chunks = []
+
+ for i in range(0, text_len, chunk_size):
+ # Add overlap to avoid missing matches at boundaries
+ end = min(i + chunk_size + term_len - 1, text_len)
+ chunks.append((i, text_lower[i:end]))
+
+ print(f"📑 Split into {len(chunks)} chunks of ~{chunk_size:,} chars each")
+
+ # Process chunks in parallel
+ with ThreadPoolExecutor(max_workers=num_workers) as executor:
+ futures = []
+
+ for chunk_idx, (start_pos, chunk_text) in enumerate(chunks):
+ if is_stop_requested():
+ return matches_count
+
+ future = executor.submit(
+ self._fuzzy_search_chunk,
+ term_lower, chunk_text, threshold, term_len, chunk_idx, len(chunks)
+ )
+ futures.append(future)
+
+ # Collect results
+ for future in as_completed(futures):
+ if is_stop_requested():
+ executor.shutdown(wait=False)
+ return matches_count
+
+ try:
+ chunk_matches = future.result()
+ matches_count += chunk_matches
+ except Exception as e:
+ print(f"📑 ⚠️ Chunk processing error: {e}")
+
+ print(f"📑 Parallel fuzzy search found {matches_count} matches")
+ return matches_count
+
+ def _fuzzy_search_chunk(self, term_lower, chunk_text, threshold, term_len, chunk_idx, total_chunks):
+ """Process a single chunk for fuzzy matches"""
+ chunk_matches = 0
+
+ # Use a more efficient step size - no need to check every position
+ step = max(1, term_len // 3) # Check every third of term length
+
+ for i in range(0, len(chunk_text) - term_len + 1, step):
+ # Check stop flag periodically
+ if i > 0 and i % 1000 == 0:
+ if is_stop_requested():
+ return chunk_matches
+
+ window = chunk_text[i:i + term_len]
+
+ # Use SequenceMatcher for fuzzy matching
+ if SequenceMatcher(None, term_lower, window).ratio() >= threshold:
+ chunk_matches += 1
+
+ # Log progress for this chunk
+ if total_chunks > 1:
+ print(f"📑 Chunk {chunk_idx + 1}/{total_chunks} completed: {chunk_matches} matches")
+
+ return chunk_matches
+
+ def _sequential_fuzzy_search(self, term_lower, text_lower, threshold, term_len):
+ """Sequential fuzzy search (fallback for small texts or single worker)"""
+ print(f"📑 Starting sequential fuzzy search...")
+ fuzzy_start = time.time()
+
+ matches_count = 0
+
+ # More efficient step size
+ step = max(1, term_len // 3)
+ total_windows = (len(text_lower) - term_len + 1) // step
+
+ print(f"📑 Checking ~{total_windows:,} windows with step size {step}")
+
+ windows_checked = 0
+ for i in range(0, len(text_lower) - term_len + 1, step):
+ # Check stop flag frequently
+ if i > 0 and i % (step * 100) == 0:
+ if is_stop_requested():
+ return matches_count
+
+ # Progress log for very long operations
+ if windows_checked % 1000 == 0 and windows_checked > 0:
+ elapsed = time.time() - fuzzy_start
+ rate = windows_checked / elapsed if elapsed > 0 else 0
+ eta = (total_windows - windows_checked) / rate if rate > 0 else 0
+ print(f"📑 Progress: {windows_checked}/{total_windows} windows, {rate:.0f} w/s, ETA: {eta:.1f}s")
+
+ window = text_lower[i:i + term_len]
+ if SequenceMatcher(None, term_lower, window).ratio() >= threshold:
+ matches_count += 1
+
+ windows_checked += 1
+
+ fuzzy_time = time.time() - fuzzy_start
+ print(f"📑 Sequential fuzzy search completed in {fuzzy_time:.2f}s, found {matches_count} matches")
+
+ return matches_count
+
+ def _fuzzy_match(self, term1, term2, threshold=0.90):
+ """Check if two terms match using fuzzy matching (unchanged)"""
+ ratio = SequenceMatcher(None, term1.lower(), term2.lower()).ratio()
+ return ratio >= threshold
+
+ def _strip_honorific(self, term, language_hint='unknown'):
+ """Strip honorific from a term if present"""
+ if not term:
+ return term
+
+ # Get honorifics for the detected language
+ honorifics_to_check = []
+ if language_hint in self.pattern_manager.CJK_HONORIFICS:
+ honorifics_to_check.extend(self.pattern_manager.CJK_HONORIFICS[language_hint])
+ honorifics_to_check.extend(self.pattern_manager.CJK_HONORIFICS.get('english', []))
+
+ # Check and remove honorifics
+ for honorific in honorifics_to_check:
+ if honorific.startswith('-') or honorific.startswith(' '):
+ # English-style suffix
+ if term.endswith(honorific):
+ return term[:-len(honorific)].strip()
+ else:
+ # CJK-style suffix (no separator)
+ if term.endswith(honorific):
+ return term[:-len(honorific)]
+
+ return term
+
+ def _translate_chunk_traditional(self, chunk_text, chunk_index, total_chunks, chapter_title=""):
+ """Simplified translation for traditional APIs (DeepL, Google Translate)"""
+
+ print(f"📝 Using traditional translation API for chunk {chunk_index}/{total_chunks}")
+
+ # Traditional APIs don't use complex prompts, just need the text
+ messages = []
+
+ # Add minimal system context for language detection
+ profile = self.active_profile
+ if profile == 'korean':
+ lang_hint = "Translating from Korean to English"
+ elif profile == 'japanese':
+ lang_hint = "Translating from Japanese to English"
+ elif profile == 'chinese':
+ lang_hint = "Translating from Chinese to English"
+ else:
+ lang_hint = "Translating to English"
+
+ messages.append({
+ "role": "system",
+ "content": lang_hint
+ })
+
+ # For traditional APIs, we need to handle glossary differently
+ # Apply glossary terms as preprocessing if available
+ processed_text = chunk_text
+
+ if hasattr(self, 'glossary_manager') and self.glossary_manager and self.glossary_manager.entries:
+ # Pre-process: Mark glossary terms with placeholders
+ glossary_placeholders = {}
+ placeholder_index = 0
+
+ for entry in self.glossary_manager.entries:
+ source = entry.get('source', '')
+ target = entry.get('target', '')
+
+ if source and target and source in processed_text:
+ # Create unique placeholder
+ placeholder = f"[[GLOSS_{placeholder_index}]]"
+ glossary_placeholders[placeholder] = target
+ processed_text = processed_text.replace(source, placeholder)
+ placeholder_index += 1
+
+ print(f"📚 Applied {len(glossary_placeholders)} glossary placeholders")
+
+ # Add the text to translate
+ messages.append({
+ "role": "user",
+ "content": processed_text
+ })
+
+ # Send to API
+ try:
+ response = self.client.send(messages)
+
+ if response and response.content:
+ translated_text = response.content
+
+ # Post-process: Replace placeholders with glossary terms
+ if 'glossary_placeholders' in locals():
+ for placeholder, target in glossary_placeholders.items():
+ translated_text = translated_text.replace(placeholder, target)
+ print(f"✅ Restored {len(glossary_placeholders)} glossary terms")
+
+ # Log detected language if available
+ if hasattr(response, 'usage') and response.usage:
+ detected_lang = response.usage.get('detected_source_lang')
+ if detected_lang:
+ print(f"🔍 Detected source language: {detected_lang}")
+
+ return translated_text
+ else:
+ print("❌ No translation received from traditional API")
+ return None
+
+ except Exception as e:
+ print(f"❌ Traditional API translation error: {e}")
+ return None
+
+ def _filter_text_for_glossary(self, text, min_frequency=2):
+ """Filter text to extract only meaningful content for glossary extraction"""
+ import re
+ from collections import Counter
+ from concurrent.futures import ThreadPoolExecutor, as_completed
+ import time
+
+ filter_start_time = time.time()
+ print(f"📑 Starting smart text filtering...")
+ print(f"📑 Input text size: {len(text):,} characters")
+
+ # Clean HTML if present
+ print(f"📑 Step 1/7: Cleaning HTML tags...")
+ from bs4 import BeautifulSoup
+ soup = BeautifulSoup(text, 'html.parser')
+ clean_text = soup.get_text()
+ print(f"📑 Clean text size: {len(clean_text):,} characters")
+
+ # Detect primary language for better filtering
+ print(f"📑 Step 2/7: Detecting primary language...")
+ def detect_primary_language(text_sample):
+ sample = text_sample[:1000]
+ korean_chars = sum(1 for char in sample if 0xAC00 <= ord(char) <= 0xD7AF)
+ japanese_kana = sum(1 for char in sample if (0x3040 <= ord(char) <= 0x309F) or (0x30A0 <= ord(char) <= 0x30FF))
+ chinese_chars = sum(1 for char in sample if 0x4E00 <= ord(char) <= 0x9FFF)
+
+ if korean_chars > 50:
+ return 'korean'
+ elif japanese_kana > 20:
+ return 'japanese'
+ elif chinese_chars > 50 and japanese_kana < 10:
+ return 'chinese'
+ else:
+ return 'english'
+
+ primary_lang = detect_primary_language(clean_text)
+ print(f"📑 Detected primary language: {primary_lang}")
+
+ # Split into sentences for better context
+ print(f"📑 Step 3/7: Splitting text into sentences...")
+ sentences = re.split(r'[.!?。！？]+', clean_text)
+ print(f"📑 Found {len(sentences):,} sentences")
+
+ # Extract potential terms (words/phrases that appear multiple times)
+ print(f"📑 Step 4/7: Setting up extraction patterns and exclusion rules...")
+ word_freq = Counter()
+
+ # Pattern for detecting potential names/terms based on capitalization or special characters
+ # Korean names: 2-4 hangul characters WITHOUT honorifics
+ korean_pattern = r'[가-힣]{2,4}'
+ # Japanese names: kanji/hiragana/katakana combinations
+ japanese_pattern = r'[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff]{2,6}'
+ # Chinese names: 2-4 Chinese characters
+ chinese_pattern = r'[\u4e00-\u9fff]{2,4}'
+ # English proper nouns: Capitalized words
+ english_pattern = r'\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b'
+
+ # Combine patterns
+ combined_pattern = f'({korean_pattern}|{japanese_pattern}|{chinese_pattern}|{english_pattern})'
+ print(f"📑 Using combined regex pattern for {primary_lang} text")
+
+ # Get honorifics and title patterns for the detected language
+ honorifics_to_exclude = set()
+ if primary_lang in self.pattern_manager.CJK_HONORIFICS:
+ honorifics_to_exclude.update(self.pattern_manager.CJK_HONORIFICS[primary_lang])
+ # Also add English romanizations
+ honorifics_to_exclude.update(self.pattern_manager.CJK_HONORIFICS.get('english', []))
+
+ # Compile title patterns for the language
+ title_patterns = []
+ if primary_lang in self.pattern_manager.TITLE_PATTERNS:
+ for pattern in self.pattern_manager.TITLE_PATTERNS[primary_lang]:
+ title_patterns.append(re.compile(pattern))
+
+ # Function to check if a term should be excluded
+ def should_exclude_term(term):
+ term_lower = term.lower()
+
+ # Check if it's a common word
+ if term in self.pattern_manager.COMMON_WORDS or term_lower in self.pattern_manager.COMMON_WORDS:
+ return True
+
+ # Check if it contains honorifics
+ for honorific in honorifics_to_exclude:
+ if honorific in term or (honorific.startswith('-') and term.endswith(honorific[1:])):
+ return True
+
+ # Check if it matches title patterns
+ for pattern in title_patterns:
+ if pattern.search(term):
+ return True
+
+ # Check if it's a number (including Chinese numbers)
+ if term in self.pattern_manager.CHINESE_NUMS:
+ return True
+
+ # Check if it's just digits
+ if term.isdigit():
+ return True
+
+ return False
+
+ # Extract potential terms from each sentence
+ print(f"📑 Step 5/7: Extracting and filtering terms from sentences...")
+
+ # Check if we should use parallel processing
+ extraction_workers = int(os.getenv("EXTRACTION_WORKERS", "1"))
+ # Auto-detect optimal workers if not set
+ if extraction_workers == 1 and len(sentences) > 1000:
+ # Use more cores for better parallelization
+ cpu_count = os.cpu_count() or 4
+ extraction_workers = min(cpu_count, 12) # Use up to 12 cores
+ print(f"📑 Auto-detected {cpu_count} CPU cores, using {extraction_workers} workers")
+
+ use_parallel = extraction_workers > 1 and len(sentences) > 100
+
+ if use_parallel:
+ print(f"📑 Using parallel processing with {extraction_workers} workers")
+ print(f"📑 Estimated speedup: {extraction_workers}x faster")
+
+ important_sentences = []
+ seen_contexts = set()
+ processed_count = 0
+ total_sentences = len(sentences)
+ last_progress_time = time.time()
+
+ def process_sentence_batch(batch_sentences, batch_idx):
+ """Process a batch of sentences"""
+ local_word_freq = Counter()
+ local_important = []
+ local_seen = set()
+
+ for sentence in batch_sentences:
+ sentence = sentence.strip()
+ if len(sentence) < 10 or len(sentence) > 500:
+ continue
+
+ # Find all potential terms in this sentence
+ matches = re.findall(combined_pattern, sentence)
+
+ if matches:
+ # Filter out excluded terms
+ filtered_matches = []
+ for match in matches:
+ if not should_exclude_term(match):
+ local_word_freq[match] += 1
+ filtered_matches.append(match)
+
+ # Keep sentences with valid potential terms
+ if filtered_matches:
+ sentence_key = ' '.join(sorted(filtered_matches))
+ if sentence_key not in local_seen:
+ local_important.append(sentence)
+ local_seen.add(sentence_key)
+
+ return local_word_freq, local_important, local_seen, batch_idx
+
+ if use_parallel:
+ # Force SMALL batches for real parallelization
+ # We want MANY small batches, not few large ones!
+
+ # Calculate based on total sentences
+ total_sentences = len(sentences)
+
+ if total_sentences < 1000:
+ # Small dataset: 50-100 sentences per batch
+ optimal_batch_size = 100
+ elif total_sentences < 10000:
+ # Medium dataset: 200 sentences per batch
+ optimal_batch_size = 200
+ elif total_sentences < 50000:
+ # Large dataset: 300 sentences per batch
+ optimal_batch_size = 300
+ else:
+ # Very large dataset: 400 sentences per batch max
+ optimal_batch_size = 400
+
+ # Ensure we have enough batches for all workers
+ min_batches = extraction_workers * 3 # At least 3 batches per worker
+ max_batch_size = max(50, total_sentences // min_batches)
+ optimal_batch_size = min(optimal_batch_size, max_batch_size)
+
+ print(f"📑 Total sentences: {total_sentences:,}")
+ print(f"📑 Target batch size: {optimal_batch_size} sentences")
+
+ # Calculate expected number of batches
+ expected_batches = (total_sentences + optimal_batch_size - 1) // optimal_batch_size
+ print(f"📑 Expected batches: {expected_batches} (for {extraction_workers} workers)")
+ print(f"📑 Batches per worker: ~{expected_batches // extraction_workers} batches")
+
+ batches = [sentences[i:i + optimal_batch_size] for i in range(0, len(sentences), optimal_batch_size)]
+ print(f"📑 Processing {len(batches)} batches of ~{optimal_batch_size} sentences each")
+ print(f"📑 Expected speedup: {min(extraction_workers, len(batches))}x (using {extraction_workers} workers)")
+
+ # Decide between ThreadPoolExecutor and ProcessPoolExecutor
+ import multiprocessing
+ in_subprocess = multiprocessing.current_process().name != 'MainProcess'
+
+ # Use ProcessPoolExecutor for better parallelism on larger datasets
+ use_process_pool = (not in_subprocess and len(sentences) > 5000)
+
+ if use_process_pool:
+ print(f"📑 Using ProcessPoolExecutor for maximum performance (true parallelism)")
+ executor_class = ProcessPoolExecutor
+ else:
+ print(f"📑 Using ThreadPoolExecutor for sentence processing")
+ executor_class = ThreadPoolExecutor
+
+ with executor_class(max_workers=extraction_workers) as executor:
+ futures = []
+
+ # Prepare data for ProcessPoolExecutor if needed
+ if use_process_pool:
+ # Serialize exclusion check data for process pool
+ exclude_check_data = (
+ list(honorifics_to_exclude),
+ [p.pattern for p in title_patterns], # Convert regex to strings
+ self.pattern_manager.COMMON_WORDS,
+ self.pattern_manager.CHINESE_NUMS
+ )
+
+ for idx, batch in enumerate(batches):
+ if use_process_pool:
+ # Use module-level function for ProcessPoolExecutor
+ future = executor.submit(_process_sentence_batch_for_extraction,
+ (batch, idx, combined_pattern, exclude_check_data))
+ else:
+ # Use local function for ThreadPoolExecutor
+ future = executor.submit(process_sentence_batch, batch, idx)
+
+ futures.append(future)
+ # Yield to GUI when submitting futures
+ if idx % 10 == 0:
+ time.sleep(0.001)
+
+ # Collect results with progress
+ completed_batches = 0
+ batch_start_time = time.time()
+ for future in as_completed(futures):
+ # Get result without timeout - as_completed already handles waiting
+ local_word_freq, local_important, local_seen, batch_idx = future.result()
+
+ # Merge results
+ word_freq.update(local_word_freq)
+ for sentence in local_important:
+ sentence_key = ' '.join(sorted(re.findall(combined_pattern, sentence)))
+ if sentence_key not in seen_contexts:
+ important_sentences.append(sentence)
+ seen_contexts.add(sentence_key)
+
+ processed_count += len(batches[batch_idx])
+ completed_batches += 1
+
+ # Show progress every 10 batches or at key milestones
+ if completed_batches % 10 == 0 or completed_batches == len(batches):
+ progress = (processed_count / total_sentences) * 100
+ elapsed = time.time() - batch_start_time
+ rate = (processed_count / elapsed) if elapsed > 0 else 0
+ print(f"📑 Progress: {processed_count:,}/{total_sentences:,} sentences ({progress:.1f}%) | Batch {completed_batches}/{len(batches)} | {rate:.0f} sent/sec")
+
+ # Yield to GUI after each batch completes
+ time.sleep(0.001)
+ else:
+ # Sequential processing with progress
+ for idx, sentence in enumerate(sentences):
+ sentence = sentence.strip()
+ if len(sentence) < 10 or len(sentence) > 500:
+ continue
+
+ # Find all potential terms in this sentence
+ matches = re.findall(combined_pattern, sentence)
+
+ if matches:
+ # Filter out excluded terms
+ filtered_matches = []
+ for match in matches:
+ if not should_exclude_term(match):
+ word_freq[match] += 1
+ filtered_matches.append(match)
+
+ # Keep sentences with valid potential terms
+ if filtered_matches:
+ sentence_key = ' '.join(sorted(filtered_matches))
+ if sentence_key not in seen_contexts:
+ important_sentences.append(sentence)
+ seen_contexts.add(sentence_key)
+
+ # Show progress every 1000 sentences or 2 seconds
+ if idx % 1000 == 0 or (time.time() - last_progress_time > 2):
+ progress = ((idx + 1) / total_sentences) * 100
+ print(f"📑 Processing sentences: {idx + 1:,}/{total_sentences:,} ({progress:.1f}%)")
+ last_progress_time = time.time()
+ # Yield to GUI thread every 1000 sentences
+ time.sleep(0.001) # Tiny sleep to let GUI update
+ # Yield to GUI thread every 1000 sentences
+ time.sleep(0.001) # Tiny sleep to let GUI update
+
+ print(f"📑 Found {len(important_sentences):,} sentences with potential glossary terms")
+
+ # Step 6/7: Deduplicate and normalize terms
+ print(f"📑 Step 6/7: Normalizing and deduplicating {len(word_freq):,} unique terms...")
+
+ # Since should_exclude_term already filters honorifics, we just need to deduplicate
+ # based on normalized forms (lowercase, etc.)
+ combined_freq = Counter()
+ term_count = 0
+
+ for term, count in word_freq.items():
+ # Normalize term for deduplication (but keep original form)
+ normalized = term.lower().strip()
+
+ # Keep the version with highest count
+ if normalized in combined_freq:
+ # If we already have this normalized form, keep the one with higher count
+ if count > combined_freq[normalized]:
+ # Remove old entry and add new one
+ del combined_freq[normalized]
+ combined_freq[term] = count
+ else:
+ combined_freq[term] = count
+
+ term_count += 1
+ # Yield to GUI every 1000 terms
+ if term_count % 1000 == 0:
+ time.sleep(0.001)
+
+ print(f"📑 Deduplicated to {len(combined_freq):,} unique terms")
+
+ # Filter to keep only terms that appear at least min_frequency times
+ frequent_terms = {term: count for term, count in combined_freq.items() if count >= min_frequency}
+
+ # Build filtered text focusing on sentences containing frequent terms
+ print(f"📑 Step 7/7: Building filtered text from relevant sentences...")
+
+ # OPTIMIZATION: Skip sentences that already passed filtering in step 5
+ # These sentences already contain glossary terms, no need to check again!
+ # We just need to limit the sample size
+
+ filtered_sentences = important_sentences # Already filtered!
+ print(f"📑 Using {len(filtered_sentences):,} pre-filtered sentences (already contain glossary terms)")
+
+ # For extremely large datasets, we can optionally do additional filtering
+ if len(filtered_sentences) > 10000 and len(frequent_terms) > 1000:
+ print(f"📑 Large dataset detected - applying frequency-based filtering...")
+ print(f"📑 Filtering {len(filtered_sentences):,} sentences for top frequent terms...")
+
+ # Sort terms by frequency to prioritize high-frequency ones
+ sorted_terms = sorted(frequent_terms.items(), key=lambda x: x[1], reverse=True)
+ top_terms = dict(sorted_terms[:1000]) # Focus on top 1000 most frequent terms
+
+ print(f"📑 Using top {len(top_terms):,} most frequent terms for final filtering")
+
+ # Use parallel processing only if really needed
+ if use_parallel and len(filtered_sentences) > 5000:
+ import multiprocessing
+ in_subprocess = multiprocessing.current_process().name != 'MainProcess'
+
+ # Create a simple set of terms for fast lookup (no variations needed)
+ term_set = set(top_terms.keys())
+
+ print(f"📑 Using parallel filtering with {extraction_workers} workers...")
+
+ # Optimize batch size
+ check_batch_size = 500 # Larger batches since we're doing simpler checks
+ check_batches = [filtered_sentences[i:i + check_batch_size]
+ for i in range(0, len(filtered_sentences), check_batch_size)]
+
+ print(f"📑 Processing {len(check_batches)} batches of ~{check_batch_size} sentences")
+
+ # Simple function to check if sentence contains any top term
+ def check_batch_simple(batch):
+ result = []
+ for sentence in batch:
+ # Simple substring check - much faster than regex
+ for term in term_set:
+ if term in sentence:
+ result.append(sentence)
+ break
+ return result
+
+ new_filtered = []
+ with ThreadPoolExecutor(max_workers=extraction_workers) as executor:
+ futures = [executor.submit(check_batch_simple, batch) for batch in check_batches]
+
+ for future in as_completed(futures):
+ new_filtered.extend(future.result())
+
+ filtered_sentences = new_filtered
+ print(f"📑 Filtered to {len(filtered_sentences):,} sentences containing top terms")
+ else:
+ # For smaller datasets, simple sequential filtering
+ print(f"📑 Using sequential filtering...")
+ new_filtered = []
+ for i, sentence in enumerate(filtered_sentences):
+ for term in top_terms:
+ if term in sentence:
+ new_filtered.append(sentence)
+ break
+ if i % 1000 == 0:
+ print(f"📑 Progress: {i:,}/{len(filtered_sentences):,} sentences")
+ time.sleep(0.001)
+
+ filtered_sentences = new_filtered
+ print(f"📑 Filtered to {len(filtered_sentences):,} sentences containing top terms")
+
+ print(f"📑 Selected {len(filtered_sentences):,} sentences containing frequent terms")
+
+ # Limit the number of sentences to reduce token usage
+ max_sentences = int(os.getenv("GLOSSARY_MAX_SENTENCES", "200"))
+ if len(filtered_sentences) > max_sentences:
+ print(f"📑 Limiting to {max_sentences} representative sentences (from {len(filtered_sentences):,})")
+ # Take a representative sample
+ step = len(filtered_sentences) // max_sentences
+ filtered_sentences = filtered_sentences[::step][:max_sentences]
+
+ filtered_text = ' '.join(filtered_sentences)
+
+ # Calculate and display filtering statistics
+ filter_end_time = time.time()
+ filter_duration = filter_end_time - filter_start_time
+
+ original_length = len(clean_text)
+ filtered_length = len(filtered_text)
+ reduction_percent = ((original_length - filtered_length) / original_length * 100) if original_length > 0 else 0
+
+ print(f"\n📑 === FILTERING COMPLETE ===")
+ print(f"📑 Duration: {filter_duration:.1f} seconds")
+ print(f"📑 Text reduction: {original_length:,} → {filtered_length:,} chars ({reduction_percent:.1f}% reduction)")
+ print(f"📑 Terms found: {len(frequent_terms):,} unique terms (min frequency: {min_frequency})")
+ print(f"📑 Final output: {len(filtered_sentences)} sentences, {filtered_length:,} characters")
+ print(f"📑 Performance: {(original_length / filter_duration / 1000):.1f}K chars/second")
+ print(f"📑 ========================\n")
+
+ return filtered_text, frequent_terms
+
+ def _extract_with_custom_prompt(self, custom_prompt, all_text, language,
+ min_frequency, max_names, max_titles,
+ existing_glossary, output_dir,
+ strip_honorifics=True, fuzzy_threshold=0.90, filter_mode='all'):
+ """Extract glossary using custom AI prompt with proper filtering"""
+ print("📑 Using custom automatic glossary prompt")
+ extraction_start = time.time()
+
+ # Check stop flag
+ if is_stop_requested():
+ print("📑 ❌ Glossary extraction stopped by user")
+ return {}
+
+ # Note: Filter mode can be controlled via the configurable prompt environment variable
+ # No hardcoded filter instructions are added here
+
+ try:
+ MODEL = os.getenv("MODEL", "gemini-2.0-flash")
+ API_KEY = (os.getenv("API_KEY") or
+ os.getenv("OPENAI_API_KEY") or
+ os.getenv("OPENAI_OR_Gemini_API_KEY") or
+ os.getenv("GEMINI_API_KEY"))
+
+ if is_traditional_translation_api(MODEL):
+ return self._translate_chunk_traditional(chunk_text, chunk_index, total_chunks, chapter_title)
+
+ elif not API_KEY:
+ print(f"📑 No API key found, falling back to pattern-based extraction")
+ return self._extract_with_patterns(all_text, language, min_frequency,
+ max_names, max_titles, 50,
+ existing_glossary, output_dir,
+ strip_honorifics, fuzzy_threshold, filter_mode)
+ else:
+ print(f"📑 Using AI-assisted extraction with custom prompt")
+
+ from unified_api_client import UnifiedClient, UnifiedClientError
+ client = UnifiedClient(model=MODEL, api_key=API_KEY, output_dir=output_dir)
+ if hasattr(client, 'reset_cleanup_state'):
+ client.reset_cleanup_state()
+
+ # Apply thread submission delay using the client's method
+ thread_delay = float(os.getenv("THREAD_SUBMISSION_DELAY_SECONDS", "0.5"))
+ if thread_delay > 0:
+ client._apply_thread_submission_delay()
+
+ # Check if cancelled during delay
+ if hasattr(client, '_cancelled') and client._cancelled:
+ print("📑 ❌ Glossary extraction stopped during delay")
+ return {}
+
+ # Check if text is already filtered (from chunking)
+ already_filtered = os.getenv("_CHUNK_ALREADY_FILTERED", "0") == "1"
+
+ if already_filtered:
+ print("📑 Text already filtered during chunking, skipping re-filtering")
+ text_sample = all_text # Use as-is since it's already filtered
+ detected_terms = {}
+ else:
+ # Apply smart filtering to reduce noise and focus on meaningful content
+ force_disable = os.getenv("GLOSSARY_FORCE_DISABLE_SMART_FILTER", "0") == "1"
+ use_smart_filter = (os.getenv("GLOSSARY_USE_SMART_FILTER", "1") == "1") and not force_disable
+
+ if use_smart_filter:
+ print("📑 Applying smart text filtering to reduce noise...")
+ text_sample, detected_terms = self._filter_text_for_glossary(all_text, min_frequency)
+ else:
+ print("📑 Smart filter disabled - using raw text sample")
+ # Fallback to simple truncation
+ max_text_size = int(os.getenv("GLOSSARY_MAX_TEXT_SIZE", "50000"))
+ text_sample = all_text[:max_text_size] if len(all_text) > max_text_size and max_text_size > 0 else all_text
+ detected_terms = {}
+
+ # Replace placeholders in prompt
+ prompt = custom_prompt.replace('{language}', language)
+ prompt = prompt.replace('{min_frequency}', str(min_frequency))
+ prompt = prompt.replace('{max_names}', str(max_names))
+ prompt = prompt.replace('{max_titles}', str(max_titles))
+
+ # Get the format instructions from environment variable
+ format_instructions = os.getenv("GLOSSARY_FORMAT_INSTRUCTIONS", "")
+
+ # If no format instructions are provided, use a default
+ if not format_instructions:
+ format_instructions = """
+Return the results in EXACT CSV format with this header:
+type,raw_name,translated_name
+
+For example:
+character,김상현,Kim Sang-hyu
+character,갈편제,Gale Hardest
+character,디히릿 아데,Dihirit Ade
+
+Only include entries that actually appear in the text.
+Do not use quotes around values unless they contain commas.
+
+Text to analyze:
+{text_sample}"""
+
+ # Replace placeholders in format instructions
+ format_instructions = format_instructions.replace('{text_sample}', text_sample)
+
+ # Combine the user's prompt with format instructions
+ enhanced_prompt = f"{prompt}\n\n{format_instructions}"
+
+ messages = [
+ {"role": "system", "content": "You are a glossary extraction assistant. Return ONLY CSV format with exactly 3 columns: type,raw_name,translated_name. The 'type' column should classify entries (e.g., character, term, location, etc.)."},
+ {"role": "user", "content": enhanced_prompt}
+ ]
+
+ # Check stop before API call
+ if is_stop_requested():
+ print("📑 ❌ Glossary extraction stopped before API call")
+ return {}
+
+ try:
+ temperature = float(os.getenv("TEMPERATURE", "0.3"))
+ max_tokens = int(os.getenv("MAX_OUTPUT_TOKENS", "4096"))
+
+ # Use send_with_interrupt for interruptible API call
+ chunk_timeout = int(os.getenv("CHUNK_TIMEOUT", "900")) # 15 minute default for glossary
+ print(f"📑 Sending AI extraction request (timeout: {chunk_timeout}s, interruptible)...")
+
+ # Before API call
+ api_start = time.time()
+ print(f"📑 Preparing API request (text size: {len(text_sample):,} chars)...")
+ print(f"📑 ⏳ Processing {len(text_sample):,} characters... Please wait, this may take 5-10 minutes")
+
+ response = send_with_interrupt(
+ messages=messages,
+ client=client,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ stop_check_fn=is_stop_requested,
+ chunk_timeout=chunk_timeout
+ )
+ api_time = time.time() - api_start
+ print(f"📑 API call completed in {api_time:.1f}s")
+
+ # Get the actual text from the response
+ if hasattr(response, 'content'):
+ response_text = response.content
+ else:
+ response_text = str(response)
+
+ # Before processing response
+ process_start = time.time()
+ print(f"📑 Processing AI response...")
+
+ # Process response and build CSV
+ csv_lines = self._process_ai_response(response_text, all_text, min_frequency,
+ strip_honorifics, fuzzy_threshold,
+ language, filter_mode)
+
+ print(f"📑 AI extracted {len(csv_lines) - 1} valid terms (header excluded)")
+
+ process_time = time.time() - process_start
+ print(f"📑 Response processing took {process_time:.1f}s")
+
+ # If we're running per-chunk, defer all heavy work and saving
+ if os.getenv("GLOSSARY_DEFER_SAVE", "0") == "1":
+ return csv_lines
+
+ # Check stop before merging
+ if is_stop_requested():
+ print("📑 ❌ Glossary generation stopped before merging")
+ return {}
+
+ # Merge with existing glossary if present
+ if existing_glossary:
+ csv_lines = self._merge_csv_entries(csv_lines, existing_glossary, strip_honorifics, language)
+
+ # Fuzzy matching deduplication
+ skip_frequency_check = os.getenv("GLOSSARY_SKIP_FREQUENCY_CHECK", "0") == "1"
+ if not skip_frequency_check: # Only dedupe if we're checking frequencies
+ # Time the deduplication
+ dedup_start = time.time()
+ original_count = len(csv_lines) - 1 # Exclude header
+
+ csv_lines = self._deduplicate_glossary_with_fuzzy(csv_lines, fuzzy_threshold)
+
+ dedup_time = time.time() - dedup_start
+ final_count = len(csv_lines) - 1 # Exclude header
+ removed_count = original_count - final_count
+
+ print(f"📑 Deduplication completed in {dedup_time:.1f}s")
+ print(f"📑 - Original entries: {original_count}")
+ print(f"📑 - Duplicates removed: {removed_count}")
+ print(f"📑 - Final entries: {final_count}")
+
+ # Store for summary statistics
+ self._dedup_time = getattr(self, '_dedup_time', 0) + dedup_time
+ else:
+ print(f"📑 Skipping deduplication (frequency check disabled)")
+
+ # Apply filter mode to final results
+ csv_lines = self._filter_csv_by_mode(csv_lines, filter_mode)
+
+ # Check if we should use token-efficient format
+ use_legacy_format = os.getenv('GLOSSARY_USE_LEGACY_CSV', '0') == '1'
+
+ if not use_legacy_format:
+ # Convert to token-efficient format
+ csv_lines = self._convert_to_token_efficient_format(csv_lines)
+
+ # Final sanitize to prevent stray headers
+ csv_lines = self._sanitize_final_glossary_lines(csv_lines, use_legacy_format)
+
+ # Create final CSV content
+ csv_content = '\n'.join(csv_lines)
+
+ # Save glossary as CSV with proper extension
+ glossary_path = os.path.join(output_dir, "glossary.csv")
+ self._atomic_write_file(glossary_path, csv_content)
+
+ print(f"\n📑 ✅ AI-ASSISTED GLOSSARY SAVED!")
+ print(f"📑 File: {glossary_path}")
+ c_count, t_count, total = self._count_glossary_entries(csv_lines, use_legacy_format)
+ print(f"📑 Character entries: {c_count}")
+ print(f"📑 Term entries: {t_count}")
+ print(f"📑 Total entries: {total}")
+ total_time = time.time() - extraction_start
+ print(f"📑 Total extraction time: {total_time:.1f}s")
+ return self._parse_csv_to_dict(csv_content)
+
+ except UnifiedClientError as e:
+ if "stopped by user" in str(e).lower():
+ print(f"📑 ❌ AI extraction interrupted by user")
+ return {}
+ else:
+ print(f"⚠️ AI extraction failed: {e}")
+ print("📑 Falling back to pattern-based extraction")
+ return self._extract_with_patterns(all_text, language, min_frequency,
+ max_names, max_titles, 50,
+ existing_glossary, output_dir,
+ strip_honorifics, fuzzy_threshold, filter_mode)
+ except Exception as e:
+ print(f"⚠️ AI extraction failed: {e}")
+ import traceback
+ traceback.print_exc()
+ print("📑 Falling back to pattern-based extraction")
+ return self._extract_with_patterns(all_text, language, min_frequency,
+ max_names, max_titles, 50,
+ existing_glossary, output_dir,
+ strip_honorifics, fuzzy_threshold, filter_mode)
+
+ except Exception as e:
+ print(f"⚠️ Custom prompt processing failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return self._extract_with_patterns(all_text, language, min_frequency,
+ max_names, max_titles, 50,
+ existing_glossary, output_dir,
+ strip_honorifics, fuzzy_threshold, filter_mode)
+
+ def _filter_csv_by_mode(self, csv_lines, filter_mode):
+ """Filter CSV lines based on the filter mode"""
+ if filter_mode == "all":
+ return csv_lines
+
+ filtered = [csv_lines[0]] # Keep header
+
+ for line in csv_lines[1:]:
+ if not line.strip():
+ continue
+
+ parts = [p.strip() for p in line.split(',')]
+ if len(parts) < 3:
+ continue
+
+ entry_type = parts[0].lower()
+ raw_name = parts[1]
+
+ if filter_mode == "only_with_honorifics":
+ # Only keep character entries with honorifics
+ if entry_type == "character" and self._has_honorific(raw_name):
+ filtered.append(line)
+ elif filter_mode == "only_without_honorifics":
+ # Keep terms and characters without honorifics
+ if entry_type == "term" or (entry_type == "character" and not self._has_honorific(raw_name)):
+ filtered.append(line)
+
+ print(f"📑 Filter '{filter_mode}': {len(filtered)-1} entries kept from {len(csv_lines)-1}")
+ return filtered
+
+ def _process_ai_response(self, response_text, all_text, min_frequency,
+ strip_honorifics, fuzzy_threshold, language, filter_mode):
+ """Process AI response and return CSV lines"""
+
+ # option to completely skip frequency validation for speed
+ skip_all_validation = os.getenv("GLOSSARY_SKIP_ALL_VALIDATION", "0") == "1"
+
+ if skip_all_validation:
+ print("📑 ⚡ FAST MODE: Skipping all frequency validation (accepting all AI results)")
+
+ # Clean response text
+ response_text = response_text.strip()
+
+ # Remove string representation artifacts if they wrap the entire response
+ if response_text.startswith('("') and response_text.endswith('")'):
+ response_text = response_text[2:-2]
+ elif response_text.startswith('"') and response_text.endswith('"'):
+ response_text = response_text[1:-1]
+ elif response_text.startswith('(') and response_text.endswith(')'):
+ response_text = response_text[1:-1]
+
+ # Unescape the string
+ response_text = response_text.replace('\\n', '\n')
+ response_text = response_text.replace('\\r', '')
+ response_text = response_text.replace('\\t', '\t')
+ response_text = response_text.replace('\\"', '"')
+ response_text = response_text.replace("\\'", "'")
+ response_text = response_text.replace('\\\\', '\\')
+
+ # Clean up markdown code blocks if present
+ if '```' in response_text:
+ parts = response_text.split('```')
+ for part in parts:
+ if 'csv' in part[:10].lower():
+ response_text = part[part.find('\n')+1:]
+ break
+ elif part.strip() and ('type,raw_name' in part or 'character,' in part or 'term,' in part):
+ response_text = part
+ break
+
+ # Normalize line endings
+ response_text = response_text.replace('\r\n', '\n').replace('\r', '\n')
+ lines = [line.strip() for line in response_text.strip().split('\n') if line.strip()]
+
+ csv_lines = []
+ header_found = False
+
+ # Check if we should skip frequency check
+ skip_frequency_check = os.getenv("GLOSSARY_SKIP_FREQUENCY_CHECK", "0") == "1"
+
+ # Add option to completely skip ALL validation for maximum speed
+ skip_all_validation = os.getenv("GLOSSARY_SKIP_ALL_VALIDATION", "0") == "1"
+
+ if skip_all_validation:
+ print("📑 ⚡ FAST MODE: Skipping all frequency validation (accepting all AI results)")
+
+ # Always use the enforced 3-column header
+ csv_lines.append("type,raw_name,translated_name")
+
+ # Process the AI response
+ for line in lines:
+ # Skip header lines
+ if 'type' in line.lower() and 'raw_name' in line.lower():
+ continue
+
+ # Parse CSV line
+ parts = [p.strip().strip('"\"') for p in line.split(',')]
+
+ if len(parts) >= 3:
+ # Has all 3 columns
+ entry_type = parts[0]
+ raw_name = parts[1]
+ translated_name = parts[2]
+ if raw_name and translated_name:
+ csv_lines.append(f"{entry_type},{raw_name},{translated_name}")
+ elif len(parts) == 2:
+ # Missing type, default to 'term'
+ raw_name = parts[0]
+ translated_name = parts[1]
+ if raw_name and translated_name:
+ csv_lines.append(f"term,{raw_name},{translated_name}")
+
+ print(f"📑 Fast mode: Accepted {len(csv_lines) - 1} entries without validation")
+ return csv_lines
+
+ # For "only_with_honorifics" mode, ALWAYS skip frequency check
+ if filter_mode == "only_with_honorifics":
+ skip_frequency_check = True
+ print("📑 Filter mode 'only_with_honorifics': Bypassing frequency checks")
+
+ print(f"📑 Processing {len(lines)} lines from AI response...")
+ print(f"📑 Text corpus size: {len(all_text):,} chars")
+ print(f"📑 Frequency checking: {'DISABLED' if skip_frequency_check else f'ENABLED (min: {min_frequency})'}")
+ print(f"📑 Fuzzy threshold: {fuzzy_threshold}")
+
+ # Collect all terms first for batch processing
+ all_terms_to_check = []
+ term_info_map = {} # Map term to its full info
+
+ if not skip_frequency_check:
+ # First pass: collect all terms that need frequency checking
+ for line in lines:
+ if 'type' in line.lower() and 'raw_name' in line.lower():
+ continue # Skip header
+
+ parts = [p.strip().strip('"\"') for p in line.split(',')]
+ if len(parts) >= 3:
+ entry_type = parts[0].lower()
+ raw_name = parts[1]
+ translated_name = parts[2]
+ elif len(parts) == 2:
+ entry_type = 'term'
+ raw_name = parts[0]
+ translated_name = parts[1]
+ else:
+ continue
+
+ if raw_name and translated_name:
+ # Store for batch processing
+ original_raw = raw_name
+ if strip_honorifics:
+ raw_name = self._strip_honorific(raw_name, language)
+
+ all_terms_to_check.append(raw_name)
+ term_info_map[raw_name] = {
+ 'entry_type': entry_type,
+ 'original_raw': original_raw,
+ 'translated_name': translated_name,
+ 'line': line
+ }
+
+ # Batch compute all frequencies at once
+ if all_terms_to_check:
+ print(f"📑 Computing frequencies for {len(all_terms_to_check)} terms...")
+ term_frequencies = self._batch_compute_frequencies(
+ all_terms_to_check, all_text, fuzzy_threshold, min_frequency
+ )
+ else:
+ term_frequencies = {}
+
+ # Now process the results using pre-computed frequencies
+ entries_processed = 0
+ entries_accepted = 0
+ # Process based on mode
+ if filter_mode == "only_with_honorifics" or skip_frequency_check:
+ # For these modes, accept all entries
+ csv_lines.append("type,raw_name,translated_name") # Header
+ for line in lines:
+ if 'type' in line.lower() and 'raw_name' in line.lower():
+ continue # Skip header
+
+ parts = [p.strip().strip('"\"') for p in line.split(',')]
+ if len(parts) >= 3:
+ entry_type = parts[0].lower()
+ raw_name = parts[1]
+ translated_name = parts[2]
+ elif len(parts) == 2:
+ entry_type = 'term'
+ raw_name = parts[0]
+ translated_name = parts[1]
+ else:
+ continue
+
+ if raw_name and translated_name:
+ csv_line = f"{entry_type},{raw_name},{translated_name}"
+ csv_lines.append(csv_line)
+ entries_accepted += 1
+
+ print(f"📑 Accepted {entries_accepted} entries (frequency check disabled)")
+
+ else:
+ # Use pre-computed frequencies
+ csv_lines.append("type,raw_name,translated_name") # Header
+
+ for term, info in term_info_map.items():
+ count = term_frequencies.get(term, 0)
+
+ # Also check original form if it was stripped
+ if info['original_raw'] != term:
+ count += term_frequencies.get(info['original_raw'], 0)
+
+ if count >= min_frequency:
+ csv_line = f"{info['entry_type']},{term},{info['translated_name']}"
+ csv_lines.append(csv_line)
+ entries_accepted += 1
+
+ # Log first few examples
+ if entries_accepted <= 5:
+ print(f"📑 ✓ Example: {term} -> {info['translated_name']} (freq: {count})")
+
+ print(f"📑 Frequency filtering complete: {entries_accepted}/{len(term_info_map)} terms accepted")
+
+ # Ensure we have at least the header
+ if len(csv_lines) == 0:
+ csv_lines.append("type,raw_name,translated_name")
+
+ # Print final summary
+ print(f"📑 Processing complete: {entries_accepted} terms accepted")
+
+ return csv_lines
+
+ def _deduplicate_glossary_with_fuzzy(self, csv_lines, fuzzy_threshold):
+ """Apply fuzzy matching to remove duplicate entries from the glossary with stop flag checks"""
+ from difflib import SequenceMatcher
+
+ print(f"📑 Applying fuzzy deduplication (threshold: {fuzzy_threshold})...")
+
+ # Check stop flag at start
+ if is_stop_requested():
+ print(f"📑 ❌ Deduplication stopped by user")
+ return csv_lines
+
+ header_line = csv_lines[0] # Keep header
+ entry_lines = csv_lines[1:] # Data lines
+
+ deduplicated = [header_line]
+ seen_entries = {} # Use dict for O(1) lookups instead of list
+ seen_names_lower = set() # Quick exact match check
+ removed_count = 0
+ total_entries = len(entry_lines)
+
+ # Pre-process all entries for faster comparison
+ print(f"📑 Processing {total_entries} entries for deduplication...")
+
+ for idx, line in enumerate(entry_lines):
+ # Check stop flag every 100 entries
+ if idx > 0 and idx % 100 == 0:
+ if is_stop_requested():
+ print(f"📑 ❌ Deduplication stopped at entry {idx}/{total_entries}")
+ return deduplicated
+
+ # Show progress for large glossaries
+ if total_entries > 500 and idx % 200 == 0:
+ progress = (idx / total_entries) * 100
+ print(f"📑 Deduplication progress: {progress:.1f}% ({idx}/{total_entries})")
+
+ if not line.strip():
+ continue
+
+ parts = [p.strip() for p in line.split(',')]
+ if len(parts) < 3:
+ continue
+
+ entry_type = parts[0]
+ raw_name = parts[1]
+ translated_name = parts[2]
+ raw_name_lower = raw_name.lower()
+
+ # Fast exact duplicate check first
+ if raw_name_lower in seen_names_lower:
+ removed_count += 1
+ continue
+
+ # For fuzzy matching, only check if threshold is less than 1.0
+ is_duplicate = False
+ if fuzzy_threshold < 1.0:
+ # Use a more efficient approach: only check similar length strings
+ name_len = len(raw_name)
+ min_len = int(name_len * 0.7)
+ max_len = int(name_len * 1.3)
+
+ # Only compare with entries of similar length
+ candidates = []
+ for seen_name, (seen_type, seen_trans) in seen_entries.items():
+ if min_len <= len(seen_name) <= max_len:
+ candidates.append(seen_name)
+
+ # Check fuzzy similarity with candidates
+ for seen_name in candidates:
+ # Quick character overlap check before expensive SequenceMatcher
+ char_overlap = len(set(raw_name_lower) & set(seen_name.lower()))
+ if char_overlap < len(raw_name_lower) * 0.5:
+ continue # Too different, skip
+
+ raw_similarity = SequenceMatcher(None, raw_name_lower, seen_name.lower()).ratio()
+
+ if raw_similarity >= fuzzy_threshold:
+ if removed_count < 10: # Only log first few
+ print(f"📑 Removing duplicate: '{raw_name}' ~= '{seen_name}' (similarity: {raw_similarity:.2%})")
+ removed_count += 1
+ is_duplicate = True
+ break
+
+ if not is_duplicate:
+ seen_entries[raw_name] = (entry_type, translated_name)
+ seen_names_lower.add(raw_name_lower)
+ deduplicated.append(line)
+
+ print(f"📑 ✅ Removed {removed_count} duplicates from glossary")
+ print(f"📑 Final glossary size: {len(deduplicated) - 1} unique entries")
+
+ return deduplicated
+
+ def _merge_csv_entries(self, new_csv_lines, existing_glossary, strip_honorifics, language):
+ """Merge CSV entries with existing glossary with stop flag checks"""
+
+ # Check stop flag at start
+ if is_stop_requested():
+ print(f"📑 ❌ Glossary merge stopped by user")
+ return new_csv_lines
+
+ # Parse existing glossary
+ existing_lines = []
+ existing_names = set()
+
+ if isinstance(existing_glossary, str):
+ # Already CSV format
+ lines = existing_glossary.strip().split('\n')
+ total_lines = len(lines)
+
+ for idx, line in enumerate(lines):
+ # Check stop flag every 50 lines
+ if idx > 0 and idx % 50 == 0:
+ if is_stop_requested():
+ print(f"📑 ❌ Merge stopped while processing existing glossary at line {idx}/{total_lines}")
+ return new_csv_lines
+
+ if total_lines > 200:
+ progress = (idx / total_lines) * 100
+ print(f"📑 Processing existing glossary: {progress:.1f}%")
+
+ if 'type,raw_name' in line.lower():
+ continue # Skip header
+
+ line_stripped = line.strip()
+ # Skip token-efficient lines and section/bullet markers
+ if not line_stripped or line_stripped.startswith('===') or line_stripped.startswith('*') or line_stripped.lower().startswith('glossary:'):
+ continue
+
+ parts = [p.strip() for p in line.split(',')]
+ # Require at least 3 fields (type, raw_name, translated_name)
+ if len(parts) < 3:
+ continue
+
+ entry_type = parts[0].strip().lower()
+ # Only accept reasonable type tokens (letters/underscores only)
+ import re as _re
+ if not _re.match(r'^[a-z_]+$', entry_type):
+ continue
+
+ raw_name = parts[1]
+ if strip_honorifics:
+ raw_name = self._strip_honorific(raw_name, language)
+ parts[1] = raw_name
+ if raw_name not in existing_names:
+ existing_lines.append(','.join(parts))
+ existing_names.add(raw_name)
+
+ # Check stop flag before processing new names
+ if is_stop_requested():
+ print(f"📑 ❌ Merge stopped before processing new entries")
+ return new_csv_lines
+
+ # Get new names
+ new_names = set()
+ final_lines = []
+
+ for idx, line in enumerate(new_csv_lines):
+ # Check stop flag every 50 lines
+ if idx > 0 and idx % 50 == 0:
+ if is_stop_requested():
+ print(f"📑 ❌ Merge stopped while processing new entries at line {idx}")
+ return final_lines if final_lines else new_csv_lines
+
+ if 'type,raw_name' in line.lower():
+ final_lines.append(line) # Keep header
+ continue
+ parts = [p.strip() for p in line.split(',')]
+ if len(parts) >= 2:
+ new_names.add(parts[1])
+ final_lines.append(line)
+
+ # Check stop flag before adding existing entries
+ if is_stop_requested():
+ print(f"📑 ❌ Merge stopped before combining entries")
+ return final_lines
+
+ # Add non-duplicate existing entries
+ added_count = 0
+ for idx, line in enumerate(existing_lines):
+ # Check stop flag every 50 additions
+ if idx > 0 and idx % 50 == 0:
+ if is_stop_requested():
+ print(f"📑 ❌ Merge stopped while adding existing entries ({added_count} added)")
+ return final_lines
+
+ parts = [p.strip() for p in line.split(',')]
+ if len(parts) >= 2 and parts[1] not in new_names:
+ final_lines.append(line)
+ added_count += 1
+
+ print(f"📑 Merged {added_count} entries from existing glossary")
+ return final_lines
+
+ def _extract_with_patterns(self, all_text, language, min_frequency,
+ max_names, max_titles, batch_size,
+ existing_glossary, output_dir,
+ strip_honorifics=True, fuzzy_threshold=0.90, filter_mode='all'):
+ """Extract glossary using pattern matching with true CSV format output and stop flag checks"""
+ print("📑 Using pattern-based extraction")
+
+ # Check stop flag at start
+ if is_stop_requested():
+ print("📑 ❌ Pattern-based extraction stopped by user")
+ return {}
+
+ def is_valid_name(name, language_hint='unknown'):
+ """Strict validation for proper names only"""
+ if not name or len(name.strip()) < 1:
+ return False
+
+ name = name.strip()
+
+ if name.lower() in self.pattern_manager.COMMON_WORDS or name in self.pattern_manager.COMMON_WORDS:
+ return False
+
+ if language_hint == 'korean':
+ if not (2 <= len(name) <= 4):
+ return False
+ if not all(0xAC00 <= ord(char) <= 0xD7AF for char in name):
+ return False
+ if len(set(name)) == 1:
+ return False
+
+ elif language_hint == 'japanese':
+ if not (2 <= len(name) <= 6):
+ return False
+ has_kanji = any(0x4E00 <= ord(char) <= 0x9FFF for char in name)
+ has_kana = any((0x3040 <= ord(char) <= 0x309F) or (0x30A0 <= ord(char) <= 0x30FF) for char in name)
+ if not (has_kanji or has_kana):
+ return False
+
+ elif language_hint == 'chinese':
+ if not (2 <= len(name) <= 4):
+ return False
+ if not all(0x4E00 <= ord(char) <= 0x9FFF for char in name):
+ return False
+
+ elif language_hint == 'english':
+ if not name[0].isupper():
+ return False
+ if sum(1 for c in name if c.isalpha()) < len(name) * 0.8:
+ return False
+ if not (2 <= len(name) <= 20):
+ return False
+
+ return True
+
+ def detect_language_hint(text_sample):
+ """Quick language detection for validation purposes"""
+ sample = text_sample[:1000]
+
+ korean_chars = sum(1 for char in sample if 0xAC00 <= ord(char) <= 0xD7AF)
+ japanese_kana = sum(1 for char in sample if (0x3040 <= ord(char) <= 0x309F) or (0x30A0 <= ord(char) <= 0x30FF))
+ chinese_chars = sum(1 for char in sample if 0x4E00 <= ord(char) <= 0x9FFF)
+ latin_chars = sum(1 for char in sample if 0x0041 <= ord(char) <= 0x007A)
+
+ if korean_chars > 50:
+ return 'korean'
+ elif japanese_kana > 20:
+ return 'japanese'
+ elif chinese_chars > 50 and japanese_kana < 10:
+ return 'chinese'
+ elif latin_chars > 100:
+ return 'english'
+ else:
+ return 'unknown'
+
+ language_hint = detect_language_hint(all_text)
+ print(f"📑 Detected primary language: {language_hint}")
+
+ # Check stop flag after language detection
+ if is_stop_requested():
+ print("📑 ❌ Extraction stopped after language detection")
+ return {}
+
+ honorifics_to_use = []
+ if language_hint in self.pattern_manager.CJK_HONORIFICS:
+ honorifics_to_use.extend(self.pattern_manager.CJK_HONORIFICS[language_hint])
+ honorifics_to_use.extend(self.pattern_manager.CJK_HONORIFICS.get('english', []))
+
+ print(f"📑 Using {len(honorifics_to_use)} honorifics for {language_hint}")
+
+ names_with_honorifics = {}
+ standalone_names = {}
+
+ # Check if parallel processing is enabled
+ extraction_workers = int(os.getenv("EXTRACTION_WORKERS", "1"))
+
+ # PARALLEL HONORIFIC PROCESSING
+ if extraction_workers > 1 and len(honorifics_to_use) > 3:
+ print(f"📑 Scanning for names with honorifics (parallel with {extraction_workers} workers)...")
+
+ # Create a wrapper function that can be called in parallel
+ def process_honorific(args):
+ """Process a single honorific in a worker thread"""
+ honorific, idx, total = args
+
+ # Check stop flag
+ if is_stop_requested():
+ return None, None
+
+ print(f"📑 Worker processing honorific {idx}/{total}: '{honorific}'")
+
+ # Local dictionaries for this worker
+ local_names_with = {}
+ local_standalone = {}
+
+ # Call the extraction method
+ self._extract_names_for_honorific(
+ honorific, all_text, language_hint,
+ min_frequency, local_names_with,
+ local_standalone, is_valid_name, fuzzy_threshold
+ )
+
+ return local_names_with, local_standalone
+
+ # Prepare arguments for parallel processing
+ honorific_args = [
+ (honorific, idx + 1, len(honorifics_to_use))
+ for idx, honorific in enumerate(honorifics_to_use)
+ ]
+
+ # Process honorifics in parallel
+ with ThreadPoolExecutor(max_workers=min(extraction_workers, len(honorifics_to_use))) as executor:
+ futures = []
+
+ for args in honorific_args:
+ if is_stop_requested():
+ executor.shutdown(wait=False)
+ return {}
+
+ future = executor.submit(process_honorific, args)
+ futures.append(future)
+
+ # Collect results as they complete
+ completed = 0
+ for future in as_completed(futures):
+ if is_stop_requested():
+ executor.shutdown(wait=False)
+ return {}
+
+ try:
+ result = future.result()
+ if result and result[0] is not None:
+ local_names_with, local_standalone = result
+
+ # Merge results (thread-safe since we're in main thread)
+ for name, count in local_names_with.items():
+ if name not in names_with_honorifics:
+ names_with_honorifics[name] = count
+ else:
+ names_with_honorifics[name] = max(names_with_honorifics[name], count)
+
+ for name, count in local_standalone.items():
+ if name not in standalone_names:
+ standalone_names[name] = count
+ else:
+ standalone_names[name] = max(standalone_names[name], count)
+
+ completed += 1
+ if completed % 5 == 0 or completed == len(honorifics_to_use):
+ print(f"📑 Honorific processing: {completed}/{len(honorifics_to_use)} completed")
+
+ except Exception as e:
+ print(f"⚠️ Failed to process honorific: {e}")
+ completed += 1
+
+ print(f"📑 Parallel honorific processing completed: found {len(names_with_honorifics)} names")
+
+ else:
+ # SEQUENTIAL PROCESSING (fallback)
+ print("📑 Scanning for names with honorifics...")
+
+ # Extract names with honorifics
+ total_honorifics = len(honorifics_to_use)
+ for idx, honorific in enumerate(honorifics_to_use):
+ # Check stop flag before each honorific
+ if is_stop_requested():
+ print(f"📑 ❌ Extraction stopped at honorific {idx}/{total_honorifics}")
+ return {}
+
+ print(f"📑 Processing honorific {idx + 1}/{total_honorifics}: '{honorific}'")
+
+ self._extract_names_for_honorific(honorific, all_text, language_hint,
+ min_frequency, names_with_honorifics,
+ standalone_names, is_valid_name, fuzzy_threshold)
+
+ # Check stop flag before processing terms
+ if is_stop_requested():
+ print("📑 ❌ Extraction stopped before processing terms")
+ return {}
+
+ # Apply filter mode
+ filtered_names = {}
+ if filter_mode == 'only_with_honorifics':
+ # Only keep names that have honorifics (no standalone names)
+ filtered_names = names_with_honorifics.copy()
+ print(f"📑 Filter: Keeping only names with honorifics ({len(filtered_names)} names)")
+ elif filter_mode == 'only_without_honorifics':
+ # Keep standalone names that were NOT found with honorifics
+ for name, count in standalone_names.items():
+ # Check if this name also appears with honorifics
+ appears_with_honorific = False
+ for honorific_name in names_with_honorifics.keys():
+ if self._strip_honorific(honorific_name, language_hint) == name:
+ appears_with_honorific = True
+ break
+
+ # Only add if it doesn't appear with honorifics
+ if not appears_with_honorific:
+ filtered_names[name] = count
+
+ print(f"📑 Filter: Keeping only names without honorifics ({len(filtered_names)} names)")
+ else: # 'all' mode
+ # Keep all names (both with and without honorifics)
+ filtered_names = names_with_honorifics.copy()
+ # Also add standalone names
+ for name, count in standalone_names.items():
+ if name not in filtered_names and not any(
+ self._strip_honorific(n, language_hint) == name for n in filtered_names.keys()
+ ):
+ filtered_names[name] = count
+ print(f"📑 Filter: Keeping all names ({len(filtered_names)} names)")
+
+ # Process extracted terms
+ final_terms = {}
+
+ term_count = 0
+ total_terms = len(filtered_names)
+ for term, count in filtered_names.items():
+ term_count += 1
+
+ # Check stop flag every 20 terms
+ if term_count % 20 == 0:
+ if is_stop_requested():
+ print(f"📑 ❌ Term processing stopped at {term_count}/{total_terms}")
+ return {}
+
+ if strip_honorifics:
+ clean_term = self._strip_honorific(term, language_hint)
+ if clean_term in final_terms:
+ final_terms[clean_term] = final_terms[clean_term] + count
+ else:
+ final_terms[clean_term] = count
+ else:
+ final_terms[term] = count
+
+ # Check stop flag before finding titles
+ if is_stop_requested():
+ print("📑 ❌ Extraction stopped before finding titles")
+ return {}
+
+ # Find titles (but respect filter mode)
+ print("📑 Scanning for titles...")
+ found_titles = {}
+
+ # Extract titles for all modes EXCEPT "only_with_honorifics"
+ # (titles are included in "only_without_honorifics" since titles typically don't have honorifics)
+ if filter_mode != 'only_with_honorifics':
+ title_patterns_to_use = []
+ if language_hint in self.pattern_manager.TITLE_PATTERNS:
+ title_patterns_to_use.extend(self.pattern_manager.TITLE_PATTERNS[language_hint])
+ title_patterns_to_use.extend(self.pattern_manager.TITLE_PATTERNS.get('english', []))
+
+ total_patterns = len(title_patterns_to_use)
+ for pattern_idx, pattern in enumerate(title_patterns_to_use):
+ # Check stop flag before each pattern
+ if is_stop_requested():
+ print(f"📑 ❌ Title extraction stopped at pattern {pattern_idx}/{total_patterns}")
+ return {}
+
+ print(f"📑 Processing title pattern {pattern_idx + 1}/{total_patterns}")
+
+ matches = list(re.finditer(pattern, all_text, re.IGNORECASE if 'english' in pattern else 0))
+
+ for match_idx, match in enumerate(matches):
+ # Check stop flag every 50 matches
+ if match_idx > 0 and match_idx % 50 == 0:
+ if is_stop_requested():
+ print(f"📑 ❌ Title extraction stopped at match {match_idx}")
+ return {}
+
+ title = match.group(0)
+
+ # Skip if this title is already in names
+ if title in filtered_names or title in names_with_honorifics:
+ continue
+
+ count = self._find_fuzzy_matches(title, all_text, fuzzy_threshold)
+
+ # Check if stopped during fuzzy matching
+ if is_stop_requested():
+ print(f"📑 ❌ Title extraction stopped during fuzzy matching")
+ return {}
+
+ if count >= min_frequency:
+ if re.match(r'[A-Za-z]', title):
+ title = title.title()
+
+ if strip_honorifics:
+ title = self._strip_honorific(title, language_hint)
+
+ if title not in found_titles:
+ found_titles[title] = count
+
+ if filter_mode == 'only_without_honorifics':
+ print(f"📑 Found {len(found_titles)} titles (included in 'without honorifics' mode)")
+ else:
+ print(f"📑 Found {len(found_titles)} unique titles")
+ else:
+ print(f"📑 Skipping title extraction (filter mode: only_with_honorifics)")
+
+ # Check stop flag before sorting and translation
+ if is_stop_requested():
+ print("📑 ❌ Extraction stopped before sorting terms")
+ return {}
+
+ # Combine and sort
+ sorted_names = sorted(final_terms.items(), key=lambda x: x[1], reverse=True)[:max_names]
+ sorted_titles = sorted(found_titles.items(), key=lambda x: x[1], reverse=True)[:max_titles]
+
+ all_terms = []
+ for name, count in sorted_names:
+ all_terms.append(name)
+ for title, count in sorted_titles:
+ all_terms.append(title)
+
+ print(f"📑 Total terms to translate: {len(all_terms)}")
+
+ # Check stop flag before translation
+ if is_stop_requested():
+ print("📑 ❌ Extraction stopped before translation")
+ return {}
+
+ # Translate terms
+ if os.getenv("DISABLE_GLOSSARY_TRANSLATION", "0") == "1":
+ print("📑 Translation disabled - keeping original terms")
+ translations = {term: term for term in all_terms}
+ else:
+ print(f"📑 Translating {len(all_terms)} terms...")
+ translations = self._translate_terms_batch(all_terms, language_hint, batch_size, output_dir)
+
+ # Check if translation was stopped
+ if is_stop_requested():
+ print("📑 ❌ Extraction stopped after translation")
+ return translations # Return partial results
+
+ # Build CSV lines
+ csv_lines = ["type,raw_name,translated_name"]
+
+ for name, _ in sorted_names:
+ if name in translations:
+ csv_lines.append(f"character,{name},{translations[name]}")
+
+ for title, _ in sorted_titles:
+ if title in translations:
+ csv_lines.append(f"term,{title},{translations[title]}")
+
+ # Check stop flag before merging
+ if is_stop_requested():
+ print("📑 ❌ Extraction stopped before merging with existing glossary")
+ # Still save what we have
+ csv_content = '\n'.join(csv_lines)
+ glossary_path = os.path.join(output_dir, "glossary.json")
+ self._atomic_write_file(glossary_path, csv_content)
+ return self._parse_csv_to_dict(csv_content)
+
+ # Merge with existing glossary
+ if existing_glossary:
+ csv_lines = self._merge_csv_entries(csv_lines, existing_glossary, strip_honorifics, language_hint)
+
+ # Check stop flag before deduplication
+ if is_stop_requested():
+ print("📑 ❌ Extraction stopped before deduplication")
+ csv_content = '\n'.join(csv_lines)
+ glossary_path = os.path.join(output_dir, "glossary.json")
+ self._atomic_write_file(glossary_path, csv_content)
+ return self._parse_csv_to_dict(csv_content)
+
+ # Fuzzy matching deduplication
+ csv_lines = self._deduplicate_glossary_with_fuzzy(csv_lines, fuzzy_threshold)
+
+ # Create CSV content
+ csv_content = '\n'.join(csv_lines)
+ # Save glossary as CSV
+ glossary_path = os.path.join(output_dir, "glossary.csv")
+ self._atomic_write_file(glossary_path, csv_content)
+
+ print(f"\n📑 ✅ TARGETED GLOSSARY SAVED!")
+ print(f"📑 File: {glossary_path}")
+ print(f"📑 Total entries: {len(csv_lines) - 1}") # Exclude header
+
+ return self._parse_csv_to_dict(csv_content)
+
+ def _translate_terms_batch(self, term_list, profile_name, batch_size=50, output_dir=None):
+ """Use fully configurable prompts for translation with interrupt support"""
+ if not term_list or os.getenv("DISABLE_GLOSSARY_TRANSLATION", "0") == "1":
+ print(f"📑 Glossary translation disabled or no terms to translate")
+ return {term: term for term in term_list}
+
+ # Check stop flag
+ if is_stop_requested():
+ print("📑 ❌ Glossary translation stopped by user")
+ return {term: term for term in term_list}
+
+ try:
+ MODEL = os.getenv("MODEL", "gemini-1.5-flash")
+ API_KEY = (os.getenv("API_KEY") or
+ os.getenv("OPENAI_API_KEY") or
+ os.getenv("OPENAI_OR_Gemini_API_KEY") or
+ os.getenv("GEMINI_API_KEY"))
+
+ if is_traditional_translation_api(MODEL):
+ return
+
+ if not API_KEY:
+ print(f"📑 No API key found, skipping translation")
+ return {term: term for term in term_list}
+
+ print(f"📑 Translating {len(term_list)} {profile_name} terms to English using batch size {batch_size}...")
+
+ from unified_api_client import UnifiedClient, UnifiedClientError
+ client = UnifiedClient(model=MODEL, api_key=API_KEY, output_dir=output_dir)
+ if hasattr(client, 'reset_cleanup_state'):
+ client.reset_cleanup_state()
+
+ # Get custom translation prompt from environment
+ translation_prompt_template = os.getenv("GLOSSARY_TRANSLATION_PROMPT", "")
+
+ if not translation_prompt_template:
+ translation_prompt_template = """You are translating {language} character names and important terms to English.
+ For character names, provide English transliterations or keep as romanized.
+ Keep honorifics/suffixes only if they are integral to the name.
+ Respond with the same numbered format.
+
+ Terms to translate:
+ {terms_list}
+
+ Provide translations in the same numbered format."""
+
+ all_translations = {}
+ chunk_timeout = int(os.getenv("CHUNK_TIMEOUT", "300")) # 5 minute default
+
+ for i in range(0, len(term_list), batch_size):
+ # Check stop flag before each batch
+ if is_stop_requested():
+ print(f"📑 ❌ Translation stopped at batch {(i // batch_size) + 1}")
+ # Return partial translations
+ for term in term_list:
+ if term not in all_translations:
+ all_translations[term] = term
+ return all_translations
+
+ batch = term_list[i:i + batch_size]
+ batch_num = (i // batch_size) + 1
+ total_batches = (len(term_list) + batch_size - 1) // batch_size
+
+ print(f"📑 Processing batch {batch_num}/{total_batches} ({len(batch)} terms)...")
+
+ # Format terms list
+ terms_text = ""
+ for idx, term in enumerate(batch, 1):
+ terms_text += f"{idx}. {term}\n"
+
+ # Replace placeholders in prompt
+ prompt = translation_prompt_template.replace('{language}', profile_name)
+ prompt = prompt.replace('{terms_list}', terms_text.strip())
+ prompt = prompt.replace('{batch_size}', str(len(batch)))
+
+ messages = [
+ {"role": "user", "content": prompt}
+ ]
+
+ try:
+ temperature = float(os.getenv("TEMPERATURE", "0.3"))
+ max_tokens = int(os.getenv("MAX_OUTPUT_TOKENS", "4096"))
+
+ # Use send_with_interrupt for interruptible API call
+ print(f"📑 Sending translation request for batch {batch_num} (interruptible)...")
+
+ response = send_with_interrupt(
+ messages=messages,
+ client=client,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ stop_check_fn=is_stop_requested,
+ chunk_timeout=chunk_timeout
+ )
+
+ # Handle response properly
+ if hasattr(response, 'content'):
+ response_text = response.content
+ else:
+ response_text = str(response)
+
+ batch_translations = self._parse_translation_response(response_text, batch)
+ all_translations.update(batch_translations)
+
+ print(f"📑 Batch {batch_num} completed: {len(batch_translations)} translations")
+
+ # Small delay between batches to avoid rate limiting (configurable)
+ if i + batch_size < len(term_list):
+ # Check stop before sleep
+ if is_stop_requested():
+ print(f"📑 ❌ Translation stopped after batch {batch_num}")
+ # Fill in missing translations
+ for term in term_list:
+ if term not in all_translations:
+ all_translations[term] = term
+ return all_translations
+ # Use configurable batch delay or default to 0.1s (much faster than 0.5s)
+ batch_delay = float(os.getenv("GLOSSARY_BATCH_DELAY", "0.001"))
+ if batch_delay > 0:
+ time.sleep(batch_delay)
+
+ except UnifiedClientError as e:
+ if "stopped by user" in str(e).lower():
+ print(f"📑 ❌ Translation interrupted by user at batch {batch_num}")
+ # Fill in remaining terms with originals
+ for term in term_list:
+ if term not in all_translations:
+ all_translations[term] = term
+ return all_translations
+ else:
+ print(f"⚠️ Translation failed for batch {batch_num}: {e}")
+ for term in batch:
+ all_translations[term] = term
+ except Exception as e:
+ print(f"⚠️ Translation failed for batch {batch_num}: {e}")
+ for term in batch:
+ all_translations[term] = term
+
+ # Ensure all terms have translations
+ for term in term_list:
+ if term not in all_translations:
+ all_translations[term] = term
+
+ translated_count = sum(1 for term, translation in all_translations.items()
+ if translation != term and translation.strip())
+
+ print(f"📑 Successfully translated {translated_count}/{len(term_list)} terms")
+ return all_translations
+
+ except Exception as e:
+ print(f"⚠️ Glossary translation failed: {e}")
+ return {term: term for term in term_list}
+
+
+ def _extract_names_for_honorific(self, honorific, all_text, language_hint,
+ min_frequency, names_with_honorifics,
+ standalone_names, is_valid_name, fuzzy_threshold=0.90):
+ """Extract names for a specific honorific with fuzzy matching and stop flag checks"""
+
+ # Check stop flag at start
+ if is_stop_requested():
+ print(f"📑 ❌ Name extraction for '{honorific}' stopped by user")
+ return
+
+ if language_hint == 'korean' and not honorific.startswith('-'):
+ pattern = r'([\uac00-\ud7af]{2,4})(?=' + re.escape(honorific) + r'(?:\s|[,.\!?]|$))'
+
+ matches = list(re.finditer(pattern, all_text))
+ total_matches = len(matches)
+
+ for idx, match in enumerate(matches):
+ # Check stop flag every 50 matches
+ if idx > 0 and idx % 50 == 0:
+ if is_stop_requested():
+ print(f"📑 ❌ Korean name extraction stopped at {idx}/{total_matches}")
+ return
+
+ # Show progress for large sets
+ if total_matches > 500:
+ progress = (idx / total_matches) * 100
+ print(f"📑 Processing Korean names: {progress:.1f}% ({idx}/{total_matches})")
+
+ potential_name = match.group(1)
+
+ if is_valid_name(potential_name, 'korean'):
+ full_form = potential_name + honorific
+
+ # Use fuzzy matching for counting with stop check
+ count = self._find_fuzzy_matches(full_form, all_text, fuzzy_threshold)
+
+ # Check if stopped during fuzzy matching
+ if is_stop_requested():
+ print(f"📑 ❌ Name extraction stopped during fuzzy matching")
+ return
+
+ if count >= min_frequency:
+ context_patterns = [
+ full_form + r'[은는이가]',
+ full_form + r'[을를]',
+ full_form + r'[에게한테]',
+ r'["]' + full_form,
+ full_form + r'[,]',
+ ]
+
+ context_count = 0
+ for ctx_pattern in context_patterns:
+ context_count += len(re.findall(ctx_pattern, all_text))
+
+ if context_count > 0:
+ names_with_honorifics[full_form] = count
+ standalone_names[potential_name] = count
+
+ elif language_hint == 'japanese' and not honorific.startswith('-'):
+ pattern = r'([\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff]{2,5})(?=' + re.escape(honorific) + r'(?:\s|[、。！？]|$))'
+
+ matches = list(re.finditer(pattern, all_text))
+ total_matches = len(matches)
+
+ for idx, match in enumerate(matches):
+ # Check stop flag every 50 matches
+ if idx > 0 and idx % 50 == 0:
+ if is_stop_requested():
+ print(f"📑 ❌ Japanese name extraction stopped at {idx}/{total_matches}")
+ return
+
+ if total_matches > 500:
+ progress = (idx / total_matches) * 100
+ print(f"📑 Processing Japanese names: {progress:.1f}% ({idx}/{total_matches})")
+
+ potential_name = match.group(1)
+
+ if is_valid_name(potential_name, 'japanese'):
+ full_form = potential_name + honorific
+ count = self._find_fuzzy_matches(full_form, all_text, fuzzy_threshold)
+
+ if is_stop_requested():
+ print(f"📑 ❌ Name extraction stopped during fuzzy matching")
+ return
+
+ if count >= min_frequency:
+ names_with_honorifics[full_form] = count
+ standalone_names[potential_name] = count
+
+ elif language_hint == 'chinese' and not honorific.startswith('-'):
+ pattern = r'([\u4e00-\u9fff]{2,4})(?=' + re.escape(honorific) + r'(?:\s|[，。！？]|$))'
+
+ matches = list(re.finditer(pattern, all_text))
+ total_matches = len(matches)
+
+ for idx, match in enumerate(matches):
+ # Check stop flag every 50 matches
+ if idx > 0 and idx % 50 == 0:
+ if is_stop_requested():
+ print(f"📑 ❌ Chinese name extraction stopped at {idx}/{total_matches}")
+ return
+
+ if total_matches > 500:
+ progress = (idx / total_matches) * 100
+ print(f"📑 Processing Chinese names: {progress:.1f}% ({idx}/{total_matches})")
+
+ potential_name = match.group(1)
+
+ if is_valid_name(potential_name, 'chinese'):
+ full_form = potential_name + honorific
+ count = self._find_fuzzy_matches(full_form, all_text, fuzzy_threshold)
+
+ if is_stop_requested():
+ print(f"📑 ❌ Name extraction stopped during fuzzy matching")
+ return
+
+ if count >= min_frequency:
+ names_with_honorifics[full_form] = count
+ standalone_names[potential_name] = count
+
+ elif honorific.startswith('-') or honorific.startswith(' '):
+ is_space_separated = honorific.startswith(' ')
+
+ if is_space_separated:
+ pattern_english = r'\b([A-Z][a-zA-Z]+)' + re.escape(honorific) + r'(?=\s|[,.\!?]|$)'
+ else:
+ pattern_english = r'\b([A-Z][a-zA-Z]+)' + re.escape(honorific) + r'\b'
+
+ matches = list(re.finditer(pattern_english, all_text))
+ total_matches = len(matches)
+
+ for idx, match in enumerate(matches):
+ # Check stop flag every 50 matches
+ if idx > 0 and idx % 50 == 0:
+ if is_stop_requested():
+ print(f"📑 ❌ English name extraction stopped at {idx}/{total_matches}")
+ return
+
+ if total_matches > 500:
+ progress = (idx / total_matches) * 100
+ print(f"📑 Processing English names: {progress:.1f}% ({idx}/{total_matches})")
+
+ potential_name = match.group(1)
+
+ if is_valid_name(potential_name, 'english'):
+ full_form = potential_name + honorific
+ count = self._find_fuzzy_matches(full_form, all_text, fuzzy_threshold)
+
+ if is_stop_requested():
+ print(f"📑 ❌ Name extraction stopped during fuzzy matching")
+ return
+
+ if count >= min_frequency:
+ names_with_honorifics[full_form] = count
+ standalone_names[potential_name] = count
+
+ def _parse_translation_response(self, response, original_terms):
+ """Parse translation response - handles numbered format"""
+ translations = {}
+
+ # Handle UnifiedResponse object
+ if hasattr(response, 'content'):
+ response_text = response.content
+ else:
+ response_text = str(response)
+
+ lines = response_text.strip().split('\n')
+
+ for line in lines:
+ line = line.strip()
+ if not line or not line[0].isdigit():
+ continue
+
+ try:
+ number_match = re.match(r'^(\d+)\.?\s*(.+)', line)
+ if number_match:
+ num = int(number_match.group(1)) - 1
+ content = number_match.group(2).strip()
+
+ if 0 <= num < len(original_terms):
+ original_term = original_terms[num]
+
+ for separator in ['->', '→', ':', '-', '—', '=']:
+ if separator in content:
+ parts = content.split(separator, 1)
+ if len(parts) == 2:
+ translation = parts[1].strip()
+ translation = translation.strip('"\'()[]')
+ if translation and translation != original_term:
+ translations[original_term] = translation
+ break
+ else:
+ if content != original_term:
+ translations[original_term] = content
+
+ except (ValueError, IndexError):
+ continue
+
+ return translations
+
+# =====================================================
+# UNIFIED UTILITIES
+# =====================================================
+def sanitize_resource_filename(filename):
+ """Sanitize resource filenames for filesystem compatibility"""
+ filename = unicodedata.normalize('NFC', filename)
+
+ replacements = {
+ '/': '_', '\\': '_', ':': '_', '*': '_',
+ '?': '_', '"': '_', '<': '_', '>': '_',
+ '|': '_', '\0': '', '\n': '_', '\r': '_'
+ }
+
+ for old, new in replacements.items():
+ filename = filename.replace(old, new)
+
+ filename = ''.join(char for char in filename if ord(char) >= 32)
+
+ name, ext = os.path.splitext(filename)
+
+ if not name:
+ name = 'resource'
+
+ return name + ext
+
+def should_retain_source_extension():
+ """Read GUI toggle for retaining original extension and no 'response_' prefix.
+ This is stored in config or env by the GUI; we read env as bridge.
+ """
+ return os.getenv('RETAIN_SOURCE_EXTENSION', os.getenv('retain_source_extension', '0')) in ('1', 'true', 'True')
+
+def make_safe_filename(title, actual_num):
+ """Create a safe filename that works across different filesystems"""
+ if not title:
+ return f"chapter_{actual_num:03d}"
+
+ title = unicodedata.normalize('NFC', str(title))
+
+ dangerous_chars = {
+ '/': '_', '\\': '_', ':': '_', '*': '_', '?': '_',
+ '"': '_', '<': '_', '>': '_', '|': '_', '\0': '',
+ '\n': ' ', '\r': ' ', '\t': ' '
+ }
+
+ for old, new in dangerous_chars.items():
+ title = title.replace(old, new)
+
+ title = ''.join(char for char in title if ord(char) >= 32)
+ title = re.sub(r'\s+', '_', title)
+ title = title.strip('_.• \t')
+
+ if not title or title == '_' * len(title):
+ title = f"chapter_{actual_num:03d}"
+
+ return title
+
+def get_content_hash(html_content):
+ """Create a stable hash of content"""
+ return ContentProcessor.get_content_hash(html_content)
+
+def clean_ai_artifacts(text, remove_artifacts=True):
+ """Remove AI response artifacts from text"""
+ return ContentProcessor.clean_ai_artifacts(text, remove_artifacts)
+
+def find_glossary_file(output_dir):
+ """Return path to glossary file preferring CSV over JSON, or None if not found"""
+ candidates = [
+ os.path.join(output_dir, "glossary.csv"),
+ os.path.join(output_dir, "glossary.json"),
+ ]
+ for p in candidates:
+ if os.path.exists(p):
+ return p
+ return None
+
+def clean_memory_artifacts(text):
+ """Remove any memory/summary artifacts"""
+ return ContentProcessor.clean_memory_artifacts(text)
+
+def emergency_restore_paragraphs(text, original_html=None, verbose=True):
+ """Emergency restoration when AI returns wall of text"""
+ return ContentProcessor.emergency_restore_paragraphs(text, original_html, verbose)
+
+def is_meaningful_text_content(html_content):
+ """Check if chapter has meaningful text beyond just structure"""
+ return ContentProcessor.is_meaningful_text_content(html_content)
+
+# =====================================================
+# GLOBAL SETTINGS AND FLAGS
+# =====================================================
+logging.basicConfig(level=logging.DEBUG)
+
+try:
+ if hasattr(sys.stdout, 'reconfigure'):
+ sys.stdout.reconfigure(encoding='utf-8', errors='ignore')
+except AttributeError:
+ if sys.stdout is None:
+ devnull = open(os.devnull, "wb")
+ sys.stdout = io.TextIOWrapper(devnull, encoding='utf-8', errors='ignore')
+ elif hasattr(sys.stdout, 'buffer'):
+ try:
+ sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='ignore')
+ except:
+ pass
+
+_stop_requested = False
+
+def set_stop_flag(value):
+ """Set the global stop flag"""
+ global _stop_requested
+ _stop_requested = value
+
+def is_stop_requested():
+ """Check if stop was requested"""
+ global _stop_requested
+ return _stop_requested
+
+def set_output_redirect(log_callback=None):
+ """Redirect print statements to a callback function for GUI integration"""
+ if log_callback:
+ class CallbackWriter:
+ def __init__(self, callback):
+ self.callback = callback
+
+ def write(self, text):
+ if text.strip():
+ self.callback(text.strip())
+
+ def flush(self):
+ pass
+
+ sys.stdout = CallbackWriter(log_callback)
+
+# =====================================================
+# EPUB AND FILE PROCESSING
+# =====================================================
+def extract_chapter_number_from_filename(filename, opf_spine_position=None, opf_spine_data=None):
+ """Extract chapter number from filename, prioritizing OPF spine order"""
+
+ # Priority 1: Use OPF spine position if available
+ if opf_spine_position is not None:
+ # Handle special non-chapter files (always chapter 0)
+ filename_lower = filename.lower()
+ name_without_ext = os.path.splitext(filename)[0].lower()
+
+ # Check for special keywords OR no numbers present
+ special_keywords = ['title', 'toc', 'cover', 'index', 'copyright', 'preface', 'nav']
+ has_special_keyword = any(name in filename_lower for name in special_keywords)
+ has_no_numbers = not re.search(r'\d', name_without_ext)
+
+ if has_special_keyword or has_no_numbers:
+ return 0, 'opf_special_file'
+
+ # Use spine position for regular chapters (0, 1, 2, 3...)
+ return opf_spine_position, 'opf_spine_order'
+
+ # Priority 2: Check if this looks like a special file (even without OPF)
+ name_without_ext = os.path.splitext(filename)[0].lower()
+ special_keywords = ['title', 'toc', 'cover', 'index', 'copyright', 'preface']
+ has_special_keyword = any(name in name_without_ext for name in special_keywords)
+ has_no_numbers = not re.search(r'\d', name_without_ext)
+
+ if has_special_keyword or has_no_numbers:
+ return 0, 'special_file'
+
+ # Priority 3: Try to extract sequential numbers (000, 001, 002...)
+ name_without_ext = os.path.splitext(filename)[0]
+
+ # Look for simple sequential patterns first
+ # Priority 3: Try to extract sequential numbers and decimals
+ sequential_patterns = [
+ (r'^(\d+)\.(\d+)$', 'decimal_number'), # 1.5, 2.3 (NEW!)
+ (r'^(\d{3,4})$', 'sequential_number'), # 000, 001, 0001
+ (r'^(\d+)$', 'direct_number'), # 0, 1, 2
+ ]
+
+ for pattern, method in sequential_patterns:
+ match = re.search(pattern, name_without_ext)
+ if match:
+ if method == 'decimal_number':
+ # Return as float for decimal chapters
+ return float(f"{match.group(1)}.{match.group(2)}"), method
+ else:
+ return int(match.group(1)), method
+
+ # Priority 4: Fall back to existing filename parsing patterns
+ fallback_patterns = [
+ (r'^response_(\d+)[_\.]', 'response_prefix'),
+ (r'[Cc]hapter[_\s]*(\d+)', 'chapter_word'),
+ (r'[Cc]h[_\s]*(\d+)', 'ch_abbreviation'),
+ (r'No(\d+)', 'no_prefix'),
+ (r'第(\d+)[章话回]', 'chinese_chapter'),
+ (r'-h-(\d+)', 'h_suffix'), # For your -h-16 pattern
+ (r'_(\d+)', 'underscore_suffix'),
+ (r'-(\d+)', 'dash_suffix'),
+ (r'(\d+)', 'trailing_number'),
+ ]
+
+ for pattern, method in fallback_patterns:
+ match = re.search(pattern, name_without_ext, re.IGNORECASE)
+ if match:
+ return int(match.group(1)), method
+
+ return None, None
+
+def process_chapter_images(chapter_html: str, actual_num: int, image_translator: ImageTranslator,
+ check_stop_fn=None) -> Tuple[str, Dict[str, str]]:
+ """Process and translate images in a chapter"""
+ from bs4 import BeautifulSoup
+ images = image_translator.extract_images_from_chapter(chapter_html)
+
+ if not images:
+ return chapter_html, {}
+
+ print(f"🖼️ Found {len(images)} images in chapter {actual_num}")
+
+ soup = BeautifulSoup(chapter_html, 'html.parser')
+
+ image_translations = {}
+ translated_count = 0
+
+ max_images_per_chapter = int(os.getenv('MAX_IMAGES_PER_CHAPTER', '10'))
+ if len(images) > max_images_per_chapter:
+ print(f" ⚠️ Chapter has {len(images)} images - processing first {max_images_per_chapter} only")
+ images = images[:max_images_per_chapter]
+
+ for idx, img_info in enumerate(images, 1):
+ if check_stop_fn and check_stop_fn():
+ print("❌ Image translation stopped by user")
+ break
+
+ img_src = img_info['src']
+
+ if img_src.startswith('../'):
+ img_path = os.path.join(image_translator.output_dir, img_src[3:])
+ elif img_src.startswith('./'):
+ img_path = os.path.join(image_translator.output_dir, img_src[2:])
+ elif img_src.startswith('/'):
+ img_path = os.path.join(image_translator.output_dir, img_src[1:])
+ else:
+ possible_paths = [
+ os.path.join(image_translator.images_dir, os.path.basename(img_src)),
+ os.path.join(image_translator.output_dir, img_src),
+ os.path.join(image_translator.output_dir, 'images', os.path.basename(img_src)),
+ os.path.join(image_translator.output_dir, os.path.basename(img_src)),
+ os.path.join(image_translator.output_dir, os.path.dirname(img_src), os.path.basename(img_src))
+ ]
+
+ img_path = None
+ for path in possible_paths:
+ if os.path.exists(path):
+ img_path = path
+ print(f" ✅ Found image at: {path}")
+ break
+
+ if not img_path:
+ print(f" ❌ Image not found in any location for: {img_src}")
+ print(f" Tried: {possible_paths}")
+ continue
+
+ img_path = os.path.normpath(img_path)
+
+ if not os.path.exists(img_path):
+ print(f" ⚠️ Image not found: {img_path}")
+ print(f" 📁 Images directory: {image_translator.images_dir}")
+ print(f" 📁 Output directory: {image_translator.output_dir}")
+ print(f" 📁 Working directory: {os.getcwd()}")
+
+ if os.path.exists(image_translator.images_dir):
+ files = os.listdir(image_translator.images_dir)
+ print(f" 📁 Files in images dir: {files[:5]}...")
+ continue
+
+ print(f" 🔍 Processing image {idx}/{len(images)}: {os.path.basename(img_path)}")
+
+ context = ""
+ if img_info.get('alt'):
+ context += f", Alt text: {img_info['alt']}"
+
+ if translated_count > 0:
+ delay = float(os.getenv('IMAGE_API_DELAY', '1.0'))
+ time.sleep(delay)
+
+ translation_result = image_translator.translate_image(img_path, context, check_stop_fn)
+
+ print(f"\n🔍 DEBUG: Image {idx}/{len(images)}")
+ print(f" Translation result: {'Success' if translation_result and '[Image Translation Error:' not in translation_result else 'Failed'}")
+ if translation_result and "[Image Translation Error:" in translation_result:
+ print(f" Error message: {translation_result}")
+
+ if translation_result:
+ img_tag = None
+ for img in soup.find_all('img'):
+ if img.get('src') == img_src:
+ img_tag = img
+ break
+
+ if img_tag:
+ hide_label = os.getenv("HIDE_IMAGE_TRANSLATION_LABEL", "0") == "1"
+
+ print(f" 🔍 DEBUG: Integration Phase")
+ print(f" 🏷️ Hide label mode: {hide_label}")
+ print(f" 📍 Found img tag: {img_tag.get('src')}")
+
+ # Store the translation result in the dictionary FIRST
+ image_translations[img_path] = translation_result
+
+ # Parse the translation result to integrate into the chapter HTML
+ if '' in translation_result:
+ trans_soup = BeautifulSoup(translation_result, 'html.parser')
+
+ # Try to get the full container first
+ full_container = trans_soup.find('div', class_=['translated-text-only', 'image-with-translation'])
+
+ if full_container:
+ # Clone the container to avoid issues
+ new_container = BeautifulSoup(str(full_container), 'html.parser').find('div')
+ img_tag.replace_with(new_container)
+ print(f" ✅ Replaced image with full translation container")
+ else:
+ # Fallback: manually build the structure
+ trans_div = trans_soup.find('div', class_='image-translation')
+ if trans_div:
+ container = soup.new_tag('div', **{'class': 'translated-text-only' if hide_label else 'image-with-translation'})
+ img_tag.replace_with(container)
+
+ if not hide_label:
+ new_img = soup.new_tag('img', src=img_src)
+ if img_info.get('alt'):
+ new_img['alt'] = img_info.get('alt')
+ container.append(new_img)
+
+ # Clone the translation div content
+ new_trans_div = soup.new_tag('div', **{'class': 'image-translation'})
+ # Copy all children from trans_div to new_trans_div
+ for child in trans_div.children:
+ if hasattr(child, 'name'):
+ new_trans_div.append(BeautifulSoup(str(child), 'html.parser'))
+ else:
+ new_trans_div.append(str(child))
+
+ container.append(new_trans_div)
+ print(f" ✅ Built container with translation div")
+ else:
+ print(f" ⚠️ No translation div found in result")
+ continue
+ else:
+ # Plain text translation - build structure manually
+ container = soup.new_tag('div', **{'class': 'translated-text-only' if hide_label else 'image-with-translation'})
+ img_tag.replace_with(container)
+
+ if not hide_label:
+ new_img = soup.new_tag('img', src=img_src)
+ if img_info.get('alt'):
+ new_img['alt'] = img_info.get('alt')
+ container.append(new_img)
+
+ # Create translation div with content
+ translation_div = soup.new_tag('div', **{'class': 'image-translation'})
+ if not hide_label:
+ label_p = soup.new_tag('p')
+ label_em = soup.new_tag('em')
+ #label_em.string = "[Image text translation:]"
+ label_p.append(label_em)
+ translation_div.append(label_p)
+
+ trans_p = soup.new_tag('p')
+ trans_p.string = translation_result
+ translation_div.append(trans_p)
+ container.append(translation_div)
+ print(f" ✅ Created plain text translation structure")
+
+ translated_count += 1
+
+ # Save to translated_images folder
+ trans_filename = f"ch{actual_num:03d}_img{idx:02d}_translation.html"
+ trans_filepath = os.path.join(image_translator.translated_images_dir, trans_filename)
+
+ # Extract just the translation content for saving
+ save_soup = BeautifulSoup(translation_result, 'html.parser')
+ save_div = save_soup.find('div', class_='image-translation')
+ if not save_div:
+ # Create a simple div for plain text
+ save_div = f'{translation_result}
'
+
+ with open(trans_filepath, 'w', encoding='utf-8') as f:
+ f.write(f"""
+
+
+
+ Chapter {actual_num} - Image {idx} Translation
+
+
+ Chapter {actual_num} - Image {idx}
+ Original: {os.path.basename(img_path)}
+
+ {save_div}
+
+""")
+
+ print(f" ✅ Saved translation to: {trans_filename}")
+ else:
+ print(f" ⚠️ Could not find image tag in HTML for: {img_src}")
+
+ if translated_count > 0:
+ print(f" 🖼️ Successfully translated {translated_count} images")
+
+ # Debug output
+ final_html = str(soup)
+ trans_count = final_html.count('')
+ print(f" 📊 Final HTML has {trans_count} translation divs")
+ print(f" 📊 image_translations dict has {len(image_translations)} entries")
+
+ prog = image_translator.load_progress()
+ if "image_chunks" in prog:
+ completed_images = []
+ for img_key, img_data in prog["image_chunks"].items():
+ if len(img_data["completed"]) == img_data["total"]:
+ completed_images.append(img_key)
+
+ for img_key in completed_images:
+ del prog["image_chunks"][img_key]
+
+ if completed_images:
+ image_translator.save_progress(prog)
+ print(f" 🧹 Cleaned up progress for {len(completed_images)} completed images")
+
+ image_translator.save_translation_log(actual_num, image_translations)
+
+ return str(soup), image_translations
+ else:
+ print(f" ℹ️ No images were successfully translated")
+
+ return chapter_html, {}
+
+def detect_novel_numbering(chapters):
+ """Detect if the novel uses 0-based or 1-based chapter numbering with improved accuracy"""
+ print("[DEBUG] Detecting novel numbering system...")
+
+ if not chapters:
+ return False
+
+ if isinstance(chapters[0], str):
+ print("[DEBUG] Text file detected, skipping numbering detection")
+ return False
+
+ patterns = PatternManager.FILENAME_EXTRACT_PATTERNS
+
+ # Special check for prefix_suffix pattern like "0000_1.xhtml"
+ prefix_suffix_pattern = r'^(\d+)_(\d+)[_\.]'
+
+ # Track chapter numbers from different sources
+ filename_numbers = []
+ content_numbers = []
+ has_prefix_suffix = False
+ prefix_suffix_numbers = []
+
+ for idx, chapter in enumerate(chapters):
+ extracted_num = None
+
+ # Check filename patterns
+ if 'original_basename' in chapter and chapter['original_basename']:
+ filename = chapter['original_basename']
+ elif 'filename' in chapter:
+ filename = os.path.basename(chapter['filename'])
+ else:
+ continue
+
+ # First check for prefix_suffix pattern
+ prefix_match = re.search(prefix_suffix_pattern, filename, re.IGNORECASE)
+ if prefix_match:
+ has_prefix_suffix = True
+ # Use the SECOND number (after underscore)
+ suffix_num = int(prefix_match.group(2))
+ prefix_suffix_numbers.append(suffix_num)
+ extracted_num = suffix_num
+ print(f"[DEBUG] Prefix_suffix pattern matched: {filename} -> Chapter {suffix_num}")
+ else:
+ # Try other patterns
+ for pattern in patterns:
+ match = re.search(pattern, filename)
+ if match:
+ extracted_num = int(match.group(1))
+ #print(f"[DEBUG] Pattern '{pattern}' matched: {filename} -> Chapter {extracted_num}")
+ break
+
+ if extracted_num is not None:
+ filename_numbers.append(extracted_num)
+
+ # Also check chapter content for chapter declarations
+ if 'body' in chapter:
+ # Look for "Chapter N" in the first 1000 characters
+ content_preview = chapter['body'][:1000]
+ content_match = re.search(r'Chapter\s+(\d+)', content_preview, re.IGNORECASE)
+ if content_match:
+ content_num = int(content_match.group(1))
+ content_numbers.append(content_num)
+ print(f"[DEBUG] Found 'Chapter {content_num}' in content")
+
+ # Decision logic with improved heuristics
+
+ # 1. If using prefix_suffix pattern, trust those numbers exclusively
+ if has_prefix_suffix and prefix_suffix_numbers:
+ min_suffix = min(prefix_suffix_numbers)
+ if min_suffix >= 1:
+ print(f"[DEBUG] ✅ 1-based novel detected (prefix_suffix pattern starts at {min_suffix})")
+ return False
+ else:
+ print(f"[DEBUG] ✅ 0-based novel detected (prefix_suffix pattern starts at {min_suffix})")
+ return True
+
+ # 2. If we have content numbers, prefer those over filename numbers
+ if content_numbers:
+ min_content = min(content_numbers)
+ # Check if we have a good sequence starting from 0 or 1
+ if 0 in content_numbers and 1 in content_numbers:
+ print(f"[DEBUG] ✅ 0-based novel detected (found both Chapter 0 and Chapter 1 in content)")
+ return True
+ elif min_content == 1:
+ print(f"[DEBUG] ✅ 1-based novel detected (content chapters start at 1)")
+ return False
+
+ # 3. Fall back to filename numbers
+ if filename_numbers:
+ min_filename = min(filename_numbers)
+ max_filename = max(filename_numbers)
+
+ # Check for a proper sequence
+ # If we have 0,1,2,3... it's likely 0-based
+ # If we have 1,2,3,4... it's likely 1-based
+
+ # Count how many chapters we have in sequence starting from 0
+ zero_sequence_count = 0
+ for i in range(len(chapters)):
+ if i in filename_numbers:
+ zero_sequence_count += 1
+ else:
+ break
+
+ # Count how many chapters we have in sequence starting from 1
+ one_sequence_count = 0
+ for i in range(1, len(chapters) + 1):
+ if i in filename_numbers:
+ one_sequence_count += 1
+ else:
+ break
+
+ print(f"[DEBUG] Zero-based sequence length: {zero_sequence_count}")
+ print(f"[DEBUG] One-based sequence length: {one_sequence_count}")
+
+ # If we have a better sequence starting from 1, it's 1-based
+ if one_sequence_count > zero_sequence_count and min_filename >= 1:
+ print(f"[DEBUG] ✅ 1-based novel detected (better sequence match starting from 1)")
+ return False
+
+ # If we have any 0 in filenames and it's part of a sequence
+ if 0 in filename_numbers and zero_sequence_count >= 3:
+ print(f"[DEBUG] ✅ 0-based novel detected (found 0 in sequence)")
+ return True
+
+ # 4. Default to 1-based if uncertain
+ print(f"[DEBUG] ✅ Defaulting to 1-based novel (insufficient evidence for 0-based)")
+ return False
+
+def validate_chapter_continuity(chapters):
+ """Validate chapter continuity and warn about issues"""
+ if not chapters:
+ print("No chapters to translate")
+ return
+
+ issues = []
+
+ # Get all chapter numbers
+ chapter_nums = [c['num'] for c in chapters]
+ actual_nums = [c.get('actual_chapter_num', c['num']) for c in chapters]
+
+ # Check for duplicates
+ duplicates = [num for num in chapter_nums if chapter_nums.count(num) > 1]
+ if duplicates:
+ issues.append(f"Duplicate chapter numbers found: {set(duplicates)}")
+
+ # Check for gaps in sequence
+ min_num = min(chapter_nums)
+ max_num = max(chapter_nums)
+ expected = set(range(min_num, max_num + 1))
+ actual = set(chapter_nums)
+ missing = expected - actual
+
+ if missing:
+ issues.append(f"Missing chapter numbers: {sorted(missing)}")
+ # Show gaps more clearly
+ gaps = []
+ sorted_missing = sorted(missing)
+ if sorted_missing:
+ start = sorted_missing[0]
+ end = sorted_missing[0]
+ for num in sorted_missing[1:]:
+ if num == end + 1:
+ end = num
+ else:
+ gaps.append(f"{start}-{end}" if start != end else str(start))
+ start = end = num
+ gaps.append(f"{start}-{end}" if start != end else str(start))
+ issues.append(f"Gap ranges: {', '.join(gaps)}")
+
+ # Check for duplicate titles
+ title_map = {}
+ for c in chapters:
+ title_lower = c['title'].lower().strip()
+ if title_lower in title_map:
+ title_map[title_lower].append(c['num'])
+ else:
+ title_map[title_lower] = [c['num']]
+
+ for title, nums in title_map.items():
+ if len(nums) > 1:
+ issues.append(f"Duplicate title '{title}' in chapters: {nums}")
+
+ # Print summary
+ print("\n" + "="*60)
+ print("📚 CHAPTER VALIDATION SUMMARY")
+ print("="*60)
+ print(f"Total chapters: {len(chapters)}")
+ print(f"Chapter range: {min_num} to {max_num}")
+ print(f"Expected count: {max_num - min_num + 1}")
+ print(f"Actual count: {len(chapters)}")
+
+ if len(chapters) != (max_num - min_num + 1):
+ print(f"⚠️ Chapter count mismatch - missing {(max_num - min_num + 1) - len(chapters)} chapters")
+
+ if issues:
+ print("\n⚠️ Issues found:")
+ for issue in issues:
+ print(f" - {issue}")
+ else:
+ print("✅ No continuity issues detected")
+
+ print("="*60 + "\n")
+
+def validate_epub_structure(output_dir):
+ """Validate that all necessary EPUB structure files are present"""
+ print("🔍 Validating EPUB structure...")
+
+ required_files = {
+ 'container.xml': 'META-INF container file (critical)',
+ '*.opf': 'OPF package file (critical)',
+ '*.ncx': 'Navigation file (recommended)'
+ }
+
+ found_files = {}
+ missing_files = []
+
+ container_path = os.path.join(output_dir, 'container.xml')
+ if os.path.exists(container_path):
+ found_files['container.xml'] = 'Found'
+ print(" ✅ container.xml - Found")
+ else:
+ missing_files.append('container.xml')
+ print(" ❌ container.xml - Missing (CRITICAL)")
+
+ opf_files = []
+ ncx_files = []
+
+ for file in os.listdir(output_dir):
+ if file.lower().endswith('.opf'):
+ opf_files.append(file)
+ elif file.lower().endswith('.ncx'):
+ ncx_files.append(file)
+
+ if opf_files:
+ found_files['opf'] = opf_files
+ print(f" ✅ OPF file(s) - Found: {', '.join(opf_files)}")
+ else:
+ missing_files.append('*.opf')
+ print(" ❌ OPF file - Missing (CRITICAL)")
+
+ if ncx_files:
+ found_files['ncx'] = ncx_files
+ print(f" ✅ NCX file(s) - Found: {', '.join(ncx_files)}")
+ else:
+ missing_files.append('*.ncx')
+ print(" ⚠️ NCX file - Missing (navigation may not work)")
+
+ html_files = [f for f in os.listdir(output_dir) if f.lower().endswith('.html') and f.startswith('response_')]
+ if html_files:
+ print(f" ✅ Translated chapters - Found: {len(html_files)} files")
+ else:
+ print(" ⚠️ No translated chapter files found")
+
+ critical_missing = [f for f in missing_files if f in ['container.xml', '*.opf']]
+
+ if not critical_missing:
+ print("✅ EPUB structure validation PASSED")
+ print(" All critical files present for EPUB reconstruction")
+ return True
+ else:
+ print("❌ EPUB structure validation FAILED")
+ print(f" Missing critical files: {', '.join(critical_missing)}")
+ print(" EPUB reconstruction may fail without these files")
+ return False
+
+def check_epub_readiness(output_dir):
+ """Check if the output directory is ready for EPUB compilation"""
+ print("📋 Checking EPUB compilation readiness...")
+
+ issues = []
+
+ if not validate_epub_structure(output_dir):
+ issues.append("Missing critical EPUB structure files")
+
+ html_files = [f for f in os.listdir(output_dir) if f.lower().endswith('.html') and f.startswith('response_')]
+ if not html_files:
+ issues.append("No translated chapter files found")
+ else:
+ print(f" ✅ Found {len(html_files)} translated chapters")
+
+ metadata_path = os.path.join(output_dir, 'metadata.json')
+ if os.path.exists(metadata_path):
+ print(" ✅ Metadata file present")
+ try:
+ with open(metadata_path, 'r', encoding='utf-8') as f:
+ metadata = json.load(f)
+ if 'title' not in metadata:
+ issues.append("Metadata missing title")
+ except Exception as e:
+ issues.append(f"Metadata file corrupted: {e}")
+ else:
+ issues.append("Missing metadata.json file")
+
+ resource_dirs = ['css', 'fonts', 'images']
+ found_resources = 0
+ for res_dir in resource_dirs:
+ res_path = os.path.join(output_dir, res_dir)
+ if os.path.exists(res_path):
+ files = [f for f in os.listdir(res_path) if os.path.isfile(os.path.join(res_path, f))]
+ if files:
+ found_resources += len(files)
+ print(f" ✅ Found {len(files)} {res_dir} files")
+
+ if found_resources > 0:
+ print(f" ✅ Total resources: {found_resources} files")
+ else:
+ print(" ⚠️ No resource files found (this may be normal)")
+
+ if not issues:
+ print("🎉 EPUB compilation readiness: READY")
+ print(" All necessary files present for EPUB creation")
+ return True
+ else:
+ print("⚠️ EPUB compilation readiness: ISSUES FOUND")
+ for issue in issues:
+ print(f" • {issue}")
+ return False
+
+def cleanup_previous_extraction(output_dir):
+ """Clean up any files from previous extraction runs (preserves CSS files)"""
+ # Remove 'css' from cleanup_items to preserve CSS files
+ cleanup_items = [
+ 'images', # Removed 'css' from this list
+ '.resources_extracted'
+ ]
+
+ epub_structure_files = [
+ 'container.xml', 'content.opf', 'toc.ncx'
+ ]
+
+ cleaned_count = 0
+
+ # Clean up directories (except CSS)
+ for item in cleanup_items:
+ if item.startswith('.'):
+ continue
+ item_path = os.path.join(output_dir, item)
+ try:
+ if os.path.isdir(item_path):
+ shutil.rmtree(item_path)
+ print(f"🧹 Removed directory: {item}")
+ cleaned_count += 1
+ except Exception as e:
+ print(f"⚠️ Could not remove directory {item}: {e}")
+
+ # Clean up EPUB structure files
+ for epub_file in epub_structure_files:
+ file_path = os.path.join(output_dir, epub_file)
+ try:
+ if os.path.isfile(file_path):
+ os.remove(file_path)
+ print(f"🧹 Removed EPUB file: {epub_file}")
+ cleaned_count += 1
+ except Exception as e:
+ print(f"⚠️ Could not remove {epub_file}: {e}")
+
+ # Clean up any loose .opf and .ncx files
+ try:
+ for file in os.listdir(output_dir):
+ if file.lower().endswith(('.opf', '.ncx')):
+ file_path = os.path.join(output_dir, file)
+ if os.path.isfile(file_path):
+ os.remove(file_path)
+ print(f"🧹 Removed EPUB file: {file}")
+ cleaned_count += 1
+ except Exception as e:
+ print(f"⚠️ Error scanning for EPUB files: {e}")
+
+ # Remove extraction marker
+ marker_path = os.path.join(output_dir, '.resources_extracted')
+ try:
+ if os.path.isfile(marker_path):
+ os.remove(marker_path)
+ print(f"🧹 Removed extraction marker")
+ cleaned_count += 1
+ except Exception as e:
+ print(f"⚠️ Could not remove extraction marker: {e}")
+
+ # Check if CSS files exist and inform user they're being preserved
+ css_path = os.path.join(output_dir, 'css')
+ if os.path.exists(css_path):
+ try:
+ css_files = [f for f in os.listdir(css_path) if os.path.isfile(os.path.join(css_path, f))]
+ if css_files:
+ print(f"📚 Preserving {len(css_files)} CSS files")
+ except Exception:
+ pass
+
+ if cleaned_count > 0:
+ print(f"🧹 Cleaned up {cleaned_count} items from previous runs (CSS files preserved)")
+
+ return cleaned_count
+
+# =====================================================
+# API AND TRANSLATION UTILITIES
+# =====================================================
+def send_with_interrupt(messages, client, temperature, max_tokens, stop_check_fn, chunk_timeout=None, request_id=None, context=None):
+ """Send API request with interrupt capability and optional timeout retry.
+ Optional context parameter is passed through to the client to improve payload labeling.
+ """
+ # Import UnifiedClientError at function level to avoid scoping issues
+ from unified_api_client import UnifiedClientError
+
+ # The client.send() call will handle multi-key rotation automatically
+
+ # Generate request_id if not provided
+ #if request_id is None:
+ # request_id = str(uuid.uuid4())[:8]
+
+ result_queue = queue.Queue()
+
+ def api_call():
+ try:
+ start_time = time.time()
+
+ # Check if client.send accepts request_id parameter
+ send_params = {
+ 'messages': messages,
+ 'temperature': temperature,
+ 'max_tokens': max_tokens
+ }
+ # Add context if supported
+ sig = inspect.signature(client.send)
+ if 'context' in sig.parameters and context is not None:
+ send_params['context'] = context
+
+ # Add request_id if the client supports it
+ sig = inspect.signature(client.send)
+ #if 'request_id' in sig.parameters:
+ # send_params['request_id'] = request_id
+
+ result = client.send(**send_params)
+ elapsed = time.time() - start_time
+ result_queue.put((result, elapsed))
+ except Exception as e:
+ result_queue.put(e)
+
+ api_thread = threading.Thread(target=api_call)
+ api_thread.daemon = True
+ api_thread.start()
+
+ timeout = chunk_timeout if chunk_timeout is not None else 86400
+ check_interval = 0.5
+ elapsed = 0
+
+ while elapsed < timeout:
+ try:
+ result = result_queue.get(timeout=check_interval)
+ if isinstance(result, Exception):
+ # For expected errors like rate limits, preserve the error type without extra traceback
+ if hasattr(result, 'error_type') and result.error_type == "rate_limit":
+ raise result
+ elif "429" in str(result) or "rate limit" in str(result).lower():
+ # Convert generic exceptions to UnifiedClientError for rate limits
+ raise UnifiedClientError(str(result), error_type="rate_limit")
+ else:
+ raise result
+ if isinstance(result, tuple):
+ api_result, api_time = result
+ if chunk_timeout and api_time > chunk_timeout:
+ # Set cleanup flag when chunk timeout occurs
+ if hasattr(client, '_in_cleanup'):
+ client._in_cleanup = True
+ if hasattr(client, 'cancel_current_operation'):
+ client.cancel_current_operation()
+ raise UnifiedClientError(f"API call took {api_time:.1f}s (timeout: {chunk_timeout}s)")
+ return api_result
+ return result
+ except queue.Empty:
+ if stop_check_fn():
+ # Set cleanup flag when user stops
+ if hasattr(client, '_in_cleanup'):
+ client._in_cleanup = True
+ if hasattr(client, 'cancel_current_operation'):
+ client.cancel_current_operation()
+ raise UnifiedClientError("Translation stopped by user")
+ elapsed += check_interval
+
+ # Set cleanup flag when timeout occurs
+ if hasattr(client, '_in_cleanup'):
+ client._in_cleanup = True
+ if hasattr(client, 'cancel_current_operation'):
+ client.cancel_current_operation()
+ raise UnifiedClientError(f"API call timed out after {timeout} seconds")
+
+def handle_api_error(processor, error, chunk_info=""):
+ """Handle API errors with multi-key support"""
+ error_str = str(error)
+
+ # Check for rate limit
+ if "429" in error_str or "rate limit" in error_str.lower():
+ if processor.config.use_multi_api_keys:
+ print(f"⚠️ Rate limit hit {chunk_info}, client should rotate to next key")
+ stats = processor.client.get_stats()
+ print(f"📊 API Stats - Active keys: {stats.get('active_keys', 0)}/{stats.get('total_keys', 0)}")
+
+ if stats.get('active_keys', 0) == 0:
+ print("⏳ All API keys are cooling down - will wait and retry")
+ print(f"🔄 Multi-key error handling: Rate limit processed, preparing for key rotation...")
+ time.sleep(0.1) # Brief pause after rate limit detection for stability
+ return True # Always retry
+ else:
+ print(f"⚠️ Rate limit hit {chunk_info}, waiting before retry...")
+ time.sleep(60)
+ print(f"🔄 Single-key error handling: Rate limit wait completed, ready for retry...")
+ time.sleep(0.1) # Brief pause after rate limit wait for stability
+ return True # Always retry
+
+ # Other errors
+ print(f"❌ API Error {chunk_info}: {error_str}")
+ return False
+
+def parse_token_limit(env_value):
+ """Parse token limit from environment variable"""
+ if not env_value or env_value.strip() == "":
+ return None, "unlimited"
+
+ env_value = env_value.strip()
+ if env_value.lower() == "unlimited":
+ return None, "unlimited"
+
+ if env_value.isdigit() and int(env_value) > 0:
+ limit = int(env_value)
+ return limit, str(limit)
+
+ return 1000000, "1000000 (default)"
+
+def build_system_prompt(user_prompt, glossary_path=None):
+ """Build the system prompt with glossary - TRUE BRUTE FORCE VERSION"""
+ append_glossary = os.getenv("APPEND_GLOSSARY", "1") == "1"
+ actual_glossary_path = glossary_path
+
+
+ system = user_prompt if user_prompt else ""
+
+ if append_glossary and actual_glossary_path and os.path.exists(actual_glossary_path):
+ try:
+ print(f"[DEBUG] ✅ Loading glossary from: {os.path.abspath(actual_glossary_path)}")
+
+ # Try to load as JSON first
+ try:
+ with open(actual_glossary_path, "r", encoding="utf-8") as gf:
+ glossary_data = json.load(gf)
+ glossary_text = json.dumps(glossary_data, ensure_ascii=False, indent=2)
+ print(f"[DEBUG] Loaded as JSON")
+ except json.JSONDecodeError:
+ # If JSON fails, just read as raw text
+ #print(f"[DEBUG] JSON parse failed, reading as raw text")
+ with open(actual_glossary_path, "r", encoding="utf-8") as gf:
+ glossary_text = gf.read()
+
+ if system:
+ system += "\n\n"
+
+ custom_prompt = os.getenv("APPEND_GLOSSARY_PROMPT", "Character/Term Glossary (use these translations consistently):").strip()
+ if not custom_prompt:
+ custom_prompt = "Character/Term Glossary (use these translations consistently):"
+
+ system += f"{custom_prompt}\n{glossary_text}"
+
+ print(f"[DEBUG] ✅ Entire glossary appended!")
+ print(f"[DEBUG] Glossary text length: {len(glossary_text)} characters")
+
+ except Exception as e:
+ print(f"[ERROR] Could not load glossary: {e}")
+ import traceback
+ print(f"[ERROR] Full traceback: {traceback.format_exc()}")
+ else:
+ if not append_glossary:
+ #print(f"[DEBUG] ❌ Glossary append disabled")
+ pass
+ elif not actual_glossary_path:
+ print(f"[DEBUG] ❌ No glossary path provided")
+ elif not os.path.exists(actual_glossary_path):
+ print(f"[DEBUG] ❌ Glossary file does not exist: {actual_glossary_path}")
+
+ print(f"🎯 Final system prompt length: {len(system)} characters")
+
+ return system
+
+def translate_title(title, client, system_prompt, user_prompt, temperature=0.3):
+ """Translate the book title using the configured settings"""
+ if not title or not title.strip():
+ return title
+
+ print(f"📚 Processing book title: {title}")
+
+ try:
+ if os.getenv("TRANSLATE_BOOK_TITLE", "1") == "0":
+ print(f"📚 Book title translation disabled - keeping original")
+ return title
+
+ # Check if we're using a translation service (not AI)
+ client_type = getattr(client, 'client_type', '')
+ is_translation_service = client_type in ['deepl', 'google_translate']
+
+ if is_translation_service:
+ # For translation services, send only the text without AI prompts
+ print(f"📚 Using translation service ({client_type}) - sending text directly")
+ messages = [
+ {"role": "user", "content": title}
+ ]
+ max_tokens = int(os.getenv("MAX_OUTPUT_TOKENS", "8192"))
+ translated_title, _ = client.send(messages, temperature=temperature, max_tokens=max_tokens)
+ else:
+ # For AI services, use prompts as before
+ book_title_prompt = os.getenv("BOOK_TITLE_PROMPT",
+ "Translate this book title to English while retaining any acronyms:")
+
+ # Get the system prompt for book titles, with fallback to default
+ book_title_system_prompt = os.getenv("BOOK_TITLE_SYSTEM_PROMPT",
+ "You are a translator. Respond with only the translated text, nothing else. Do not add any explanation or additional content.")
+
+ messages = [
+ {"role": "system", "content": book_title_system_prompt},
+ {"role": "user", "content": f"{book_title_prompt}\n\n{title}"}
+ ]
+ max_tokens = int(os.getenv("MAX_OUTPUT_TOKENS", "8192"))
+ translated_title, _ = client.send(messages, temperature=temperature, max_tokens=max_tokens)
+
+ print(f"[DEBUG] Raw API response: '{translated_title}'")
+ print(f"[DEBUG] Response length: {len(translated_title)} (original: {len(title)})")
+ newline = '\n'
+ print(f"[DEBUG] Has newlines: {repr(translated_title) if newline in translated_title else 'No'}")
+
+ translated_title = translated_title.strip()
+
+ if ((translated_title.startswith('"') and translated_title.endswith('"')) or
+ (translated_title.startswith("'") and translated_title.endswith("'"))):
+ translated_title = translated_title[1:-1].strip()
+
+ if '\n' in translated_title:
+ print(f"⚠️ API returned multi-line content, keeping original title")
+ return title
+
+ # Check for JSON-like structured content, but allow simple brackets like [END]
+ if (any(char in translated_title for char in ['{', '}']) or
+ '"role":' in translated_title or
+ '"content":' in translated_title or
+ ('[[' in translated_title and ']]' in translated_title)): # Only flag double brackets
+ print(f"⚠️ API returned structured content, keeping original title")
+ return title
+
+ if any(tag in translated_title.lower() for tag in ['', '
', '', '
', '= 2:
+ return True
+
+ # Single strong error indicator in very short response
+ if len(content_str) < 50 and error_count >= 1:
+ return True
+
+ return False
+
+
+# Additional helper function for debugging
+def get_failure_reason(content):
+ """
+ Returns the specific reason why content was marked as qa_failed
+ Useful for debugging and logging
+ """
+ if not content:
+ return "Empty content"
+
+ content_str = str(content).strip()
+ content_lower = content_str.lower()
+
+ # Check each category and return the first match
+ failure_categories = {
+ "Explicit Failure Marker": [
+ "[TRANSLATION FAILED - ORIGINAL TEXT PRESERVED]",
+ "[IMAGE TRANSLATION FAILED]",
+ "API response unavailable",
+ "[]"
+ ],
+ "HTTP Error": [
+ "authentication_error", "rate_limit_error", "api_error"
+ ],
+ "Content Filter": [
+ "content_filter", "safety filter", "blocked by safety"
+ ],
+ "Timeout": [
+ "timeout", "timed out", "apitimeouterror"
+ ],
+ "Rate Limit": [
+ "rate limit exceeded", "quota exceeded", "too many requests"
+ ],
+ "Refusal Pattern": [
+ "i cannot", "i can't", "unable to process"
+ ],
+ "Empty Response": [
+ '"text": ""', "choices: [ { text: ''"
+ ]
+ }
+
+ for category, markers in failure_categories.items():
+ for marker in markers:
+ if marker in content_str or marker in content_lower:
+ return f"{category}: {marker}"
+
+ if len(content_str) < 50:
+ return f"Short response with error indicators: {content_str[:30]}..."
+
+ return "Unknown failure pattern"
+
+def convert_enhanced_text_to_html(plain_text, chapter_info=None):
+ """Convert markdown/plain text back to HTML after translation (for enhanced mode)
+
+ This function handles the conversion of translated markdown back to HTML.
+ The input is the TRANSLATED text that was originally extracted using html2text.
+ """
+ import re
+
+ preserve_structure = chapter_info.get('preserve_structure', False) if chapter_info else False
+
+ # First, try to use markdown2 for proper markdown conversion
+ try:
+ import markdown2
+
+ # Check if the text contains markdown patterns
+ has_markdown = any([
+ '##' in plain_text, # Headers
+ '**' in plain_text, # Bold
+ '*' in plain_text and not '**' in plain_text, # Italic
+ '[' in plain_text and '](' in plain_text, # Links
+ '```' in plain_text, # Code blocks
+ '> ' in plain_text, # Blockquotes
+ '- ' in plain_text or '* ' in plain_text or '1. ' in plain_text # Lists
+ ])
+
+ if has_markdown or preserve_structure:
+ # Use markdown2 for proper conversion
+ html = markdown2.markdown(plain_text, extras=[
+ 'cuddled-lists', # Lists without blank lines
+ 'fenced-code-blocks', # Code blocks with ```
+ 'break-on-newline', # Treat single newlines as
+ 'smarty-pants', # Smart quotes and dashes
+ 'tables', # Markdown tables
+ ])
+
+ # Post-process to ensure proper paragraph structure
+ if not '' in html:
+ # If markdown2 didn't create paragraphs, wrap content
+ lines = html.split('\n')
+ processed_lines = []
+ for line in lines:
+ line = line.strip()
+ if line and not line.startswith('<') and not line.endswith('>'):
+ processed_lines.append(f'
{line}
')
+ elif line:
+ processed_lines.append(line)
+ html = '\n'.join(processed_lines)
+
+ return html
+
+ except ImportError:
+ print("⚠️ markdown2 not available, using fallback HTML conversion")
+
+ # Fallback: Manual markdown-to-HTML conversion
+ lines = plain_text.strip().split('\n')
+ html_parts = []
+ in_code_block = False
+ code_block_content = []
+
+ for line in lines:
+ # Handle code blocks
+ if line.strip().startswith('```'):
+ if in_code_block:
+ # End code block
+ html_parts.append('' + '\n'.join(code_block_content) + '
')
+ code_block_content = []
+ in_code_block = False
+ else:
+ # Start code block
+ in_code_block = True
+ continue
+
+ if in_code_block:
+ code_block_content.append(line)
+ continue
+
+ line = line.strip()
+ if not line:
+ # Preserve empty lines as paragraph breaks
+ if html_parts and not html_parts[-1].endswith(''):
+ # Only add break if not already after a closing tag
+ html_parts.append('
')
+ continue
+
+ # Check for markdown headers
+ if line.startswith('#'):
+ match = re.match(r'^(#+)\s*(.+)$', line)
+ if match:
+ level = min(len(match.group(1)), 6)
+ header_text = match.group(2).strip()
+ html_parts.append(f'{header_text} ')
+ continue
+
+ # Check for blockquotes
+ if line.startswith('> '):
+ quote_text = line[2:].strip()
+ html_parts.append(f'{quote_text}
')
+ continue
+
+ # Check for lists
+ if re.match(r'^[*\-+]\s+', line):
+ list_text = re.sub(r'^[*\-+]\s+', '', line)
+ html_parts.append(f'{list_text} ')
+ continue
+
+ if re.match(r'^\d+\.\s+', line):
+ list_text = re.sub(r'^\d+\.\s+', '', line)
+ html_parts.append(f'{list_text} ')
+ continue
+
+ # Convert inline markdown
+ # Bold
+ line = re.sub(r'\*\*(.+?)\*\*', r'\1', line)
+ line = re.sub(r'__(.+?)__', r'\1', line)
+
+ # Italic
+ line = re.sub(r'\*(.+?)\*', r'\1', line)
+ line = re.sub(r'_(.+?)_', r'\1', line)
+
+ # Links
+ line = re.sub(r'\[([^\]]+)\]\(([^)]+)\)', r'\1', line)
+
+ # Code inline
+ line = re.sub(r'`([^`]+)`', r'\1', line)
+
+ # Regular paragraph
+ html_parts.append(f'{line}
')
+
+ # Post-process lists to wrap in ul/ol tags
+ final_html = []
+ in_list = False
+ list_type = None
+
+ for part in html_parts:
+ if part.startswith(''):
+ if not in_list:
+ # Determine list type based on context (simplified)
+ list_type = 'ul' # Default to unordered
+ final_html.append(f'<{list_type}>')
+ in_list = True
+ final_html.append(part)
+ else:
+ if in_list:
+ final_html.append(f'')
+ in_list = False
+ final_html.append(part)
+
+ # Close any open list
+ if in_list:
+ final_html.append(f'')
+
+ return '\n'.join(final_html)
+# =====================================================
+# MAIN TRANSLATION FUNCTION
+# =====================================================
+def main(log_callback=None, stop_callback=None):
+ """Main translation function with enhanced duplicate detection and progress tracking"""
+
+ config = TranslationConfig()
+ builtins._DISABLE_ZERO_DETECTION = config.DISABLE_ZERO_DETECTION
+
+ if config.DISABLE_ZERO_DETECTION:
+ print("=" * 60)
+ print("⚠️ 0-BASED DETECTION DISABLED BY USER")
+ print("⚠️ All chapter numbers will be used exactly as found")
+ print("=" * 60)
+
+ args = None
+ chapters_completed = 0
+ chunks_completed = 0
+
+ args = None
+ chapters_completed = 0
+ chunks_completed = 0
+
+ input_path = config.input_path
+ if not input_path and len(sys.argv) > 1:
+ input_path = sys.argv[1]
+
+ is_text_file = input_path.lower().endswith('.txt')
+
+ if is_text_file:
+ os.environ["IS_TEXT_FILE_TRANSLATION"] = "1"
+
+ import json as _json
+ _original_load = _json.load
+
+ def debug_json_load(fp, *args, **kwargs):
+ result = _original_load(fp, *args, **kwargs)
+ if isinstance(result, list) and len(result) > 0:
+ if isinstance(result[0], dict) and 'original_name' in result[0]:
+ print(f"[DEBUG] Loaded glossary list with {len(result)} items from {fp.name if hasattr(fp, 'name') else 'unknown'}")
+ return result
+
+ _json.load = debug_json_load
+
+ if log_callback:
+ set_output_redirect(log_callback)
+
+ def check_stop():
+ if stop_callback and stop_callback():
+ print("❌ Translation stopped by user request.")
+ return True
+ return is_stop_requested()
+
+ if config.EMERGENCY_RESTORE:
+ print("✅ Emergency paragraph restoration is ENABLED")
+ else:
+ print("⚠️ Emergency paragraph restoration is DISABLED")
+
+ print(f"[DEBUG] REMOVE_AI_ARTIFACTS environment variable: {os.getenv('REMOVE_AI_ARTIFACTS', 'NOT SET')}")
+ print(f"[DEBUG] REMOVE_AI_ARTIFACTS parsed value: {config.REMOVE_AI_ARTIFACTS}")
+ if config.REMOVE_AI_ARTIFACTS:
+ print("⚠️ AI artifact removal is ENABLED - will clean AI response artifacts")
+ else:
+ print("✅ AI artifact removal is DISABLED - preserving all content as-is")
+
+ if '--epub' in sys.argv or (len(sys.argv) > 1 and sys.argv[1].endswith(('.epub', '.txt'))):
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument('epub', help='Input EPUB or text file')
+ args = parser.parse_args()
+ input_path = args.epub
+
+ is_text_file = input_path.lower().endswith('.txt')
+
+ if is_text_file:
+ file_base = os.path.splitext(os.path.basename(input_path))[0]
+ else:
+ epub_base = os.path.splitext(os.path.basename(input_path))[0]
+ file_base = epub_base
+
+ out = file_base
+ os.makedirs(out, exist_ok=True)
+ print(f"[DEBUG] Created output folder → {out}")
+
+ cleanup_previous_extraction(out)
+
+ os.environ["EPUB_OUTPUT_DIR"] = out
+ payloads_dir = out
+
+ # clear history if CONTEXTUAL is disabled
+ if not config.CONTEXTUAL:
+ history_file = os.path.join(payloads_dir, "translation_history.json")
+ if os.path.exists(history_file):
+ os.remove(history_file)
+ print("[DEBUG] CONTEXTUAL disabled - cleared translation history")
+
+ history_manager = HistoryManager(payloads_dir)
+ chapter_splitter = ChapterSplitter(model_name=config.MODEL)
+ chunk_context_manager = ChunkContextManager()
+ progress_manager = ProgressManager(payloads_dir)
+
+ # Create ChapterExtractor with progress callback if available
+ chapter_progress_callback = None
+ if log_callback:
+ # Create a wrapper that formats progress messages for the log
+ def chapter_progress_callback(msg):
+ log_callback(f"📊 {msg}")
+
+ chapter_extractor = ChapterExtractor(progress_callback=chapter_progress_callback)
+ glossary_manager = GlossaryManager()
+
+ history_file = os.path.join(payloads_dir, "translation_history.json")
+ if os.path.exists(history_file):
+ os.remove(history_file)
+ print(f"[DEBUG] Purged translation history → {history_file}")
+
+ print("🔍 Checking for deleted output files...")
+ progress_manager.cleanup_missing_files(out)
+ progress_manager.save()
+
+ if check_stop():
+ return
+
+ if not config.API_KEY:
+ print("❌ Error: Set API_KEY, OPENAI_API_KEY, or OPENAI_OR_Gemini_API_KEY in your environment.")
+ return
+
+ #print(f"[DEBUG] Found API key: {config.API_KEY[:10]}...")
+ print(f"[DEBUG] Using model = {config.MODEL}")
+ print(f"[DEBUG] Max output tokens = {config.MAX_OUTPUT_TOKENS}")
+
+ client = UnifiedClient(model=config.MODEL, api_key=config.API_KEY, output_dir=out)
+ if hasattr(client, 'use_multi_keys') and client.use_multi_keys:
+ stats = client.get_stats()
+ print(f"🔑 Multi-key mode active: {stats.get('total_keys', 0)} keys loaded")
+ print(f" Active keys: {stats.get('active_keys', 0)}")
+ else:
+ print(f"🔑 Single-key mode: Using {config.MODEL}")
+ # Reset cleanup state when starting new translation
+ if hasattr(client, 'reset_cleanup_state'):
+ client.reset_cleanup_state()
+
+ if is_text_file:
+ print("📄 Processing text file...")
+ try:
+ txt_processor = TextFileProcessor(input_path, out)
+ chapters = txt_processor.extract_chapters()
+ txt_processor.save_original_structure()
+
+ metadata = {
+ "title": os.path.splitext(os.path.basename(input_path))[0],
+ "type": "text",
+ "chapter_count": len(chapters)
+ }
+ except ImportError as e:
+ print(f"❌ Error: Text file processor not available: {e}")
+ if log_callback:
+ log_callback(f"❌ Error: Text file processor not available: {e}")
+ return
+ except Exception as e:
+ print(f"❌ Error processing text file: {e}")
+ if log_callback:
+ log_callback(f"❌ Error processing text file: {e}")
+ return
+ else:
+ # Check if we should use async extraction (for GUI mode)
+ use_async_extraction = os.getenv("USE_ASYNC_CHAPTER_EXTRACTION", "0") == "1"
+
+ if use_async_extraction and log_callback:
+ print("🚀 Using async chapter extraction (subprocess mode)...")
+ from chapter_extraction_manager import ChapterExtractionManager
+
+ # Create manager with log callback
+ extraction_manager = ChapterExtractionManager(log_callback=log_callback)
+
+ # Get extraction mode
+ extraction_mode = os.getenv("EXTRACTION_MODE", "smart").lower()
+
+ # Define completion callback
+ extraction_result = {"completed": False, "result": None}
+
+ def on_extraction_complete(result):
+ extraction_result["completed"] = True
+ extraction_result["result"] = result
+
+ # Safety check for None result
+ if result is None:
+ log_callback("❌ Chapter extraction failed: No result returned")
+ return
+
+ if result.get("success"):
+ log_callback(f"✅ Chapter extraction completed: {result.get('chapters', 0)} chapters")
+ else:
+ log_callback(f"❌ Chapter extraction failed: {result.get('error', 'Unknown error')}")
+
+ # Start async extraction
+ extraction_manager.extract_chapters_async(
+ input_path,
+ out,
+ extraction_mode=extraction_mode,
+ progress_callback=lambda msg: log_callback(f"📊 {msg}"),
+ completion_callback=on_extraction_complete
+ )
+
+ # Wait for completion (with timeout)
+ timeout = 300 # 5 minutes timeout
+ start_time = time.time()
+
+ while not extraction_result["completed"]:
+ if check_stop():
+ extraction_manager.stop_extraction()
+ return
+
+ if time.time() - start_time > timeout:
+ log_callback("⚠️ Chapter extraction timeout")
+ extraction_manager.stop_extraction()
+ return
+
+ time.sleep(0.1) # Check every 100ms
+
+ # Check if extraction was successful
+ if not extraction_result["result"] or not extraction_result["result"].get("success"):
+ log_callback("❌ Chapter extraction failed")
+ return
+
+ # Load the extracted data
+ metadata_path = os.path.join(out, "metadata.json")
+ if os.path.exists(metadata_path):
+ with open(metadata_path, 'r', encoding='utf-8') as f:
+ metadata = json.load(f)
+ else:
+ metadata = extraction_result["result"].get("metadata", {})
+
+ # The async extraction should have saved chapters directly, similar to the sync version
+ # We need to reconstruct the chapters list with body content
+
+ # Check if the extraction actually created a chapters.json file with full content
+ chapters_full_path = os.path.join(out, "chapters_full.json")
+ chapters_info_path = os.path.join(out, "chapters_info.json")
+
+ chapters = []
+
+ # First try to load full chapters if saved
+ if os.path.exists(chapters_full_path):
+ log_callback("Loading full chapters data...")
+ with open(chapters_full_path, 'r', encoding='utf-8') as f:
+ chapters = json.load(f)
+ log_callback(f"✅ Loaded {len(chapters)} chapters with content")
+
+ elif os.path.exists(chapters_info_path):
+ # Fall back to loading from individual files
+ log_callback("Loading chapter info and searching for content files...")
+ with open(chapters_info_path, 'r', encoding='utf-8') as f:
+ chapters_info = json.load(f)
+
+ # List all files in the output directory
+ all_files = os.listdir(out)
+ log_callback(f"Found {len(all_files)} files in output directory")
+
+ # Try to match chapter files
+ for info in chapters_info:
+ chapter_num = info['num']
+ found = False
+
+ # Try different naming patterns
+ patterns = [
+ f"chapter_{chapter_num:04d}_", # With leading zeros
+ f"chapter_{chapter_num}_", # Without leading zeros
+ f"ch{chapter_num:04d}_", # Shortened with zeros
+ f"ch{chapter_num}_", # Shortened without zeros
+ f"{chapter_num:04d}_", # Just number with zeros
+ f"{chapter_num}_" # Just number
+ ]
+
+ for pattern in patterns:
+ # Find files matching this pattern (any extension)
+ matching_files = [f for f in all_files if f.startswith(pattern)]
+
+ if matching_files:
+ # Prefer HTML/XHTML files
+ html_files = [f for f in matching_files if f.endswith(('.html', '.xhtml', '.htm'))]
+ if html_files:
+ chapter_file = html_files[0]
+ else:
+ chapter_file = matching_files[0]
+
+ chapter_path = os.path.join(out, chapter_file)
+
+ try:
+ with open(chapter_path, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ chapters.append({
+ "num": chapter_num,
+ "title": info.get("title", f"Chapter {chapter_num}"),
+ "body": content,
+ "filename": info.get("original_filename", ""),
+ "has_images": info.get("has_images", False),
+ "file_size": len(content),
+ "content_hash": info.get("content_hash", "")
+ })
+ found = True
+ break
+ except Exception as e:
+ log_callback(f"⚠️ Error reading {chapter_file}: {e}")
+
+ if not found:
+ log_callback(f"⚠️ No file found for Chapter {chapter_num}")
+ # Log available files for debugging
+ if len(all_files) < 50:
+ similar_files = [f for f in all_files if str(chapter_num) in f]
+ if similar_files:
+ log_callback(f" Similar files: {similar_files[:3]}")
+
+ if not chapters:
+ log_callback("❌ No chapters could be loaded!")
+ log_callback(f"❌ Output directory: {out}")
+ log_callback(f"❌ Files in directory: {len(os.listdir(out))} files")
+ # Show first few files for debugging
+ sample_files = os.listdir(out)[:10]
+ log_callback(f"❌ Sample files: {sample_files}")
+ return
+
+ # Sort chapters by OPF spine order if available
+ opf_path = os.path.join(out, 'content.opf')
+ if os.path.exists(opf_path) and chapters:
+ log_callback("📋 Sorting chapters according to OPF spine order...")
+ # Use the existing chapter_extractor instance to sort
+ chapters = chapter_extractor._sort_by_opf_spine(chapters, opf_path)
+ log_callback("✅ Chapters sorted according to OPF reading order")
+ else:
+ print("🚀 Using comprehensive chapter extraction with resource handling...")
+ with zipfile.ZipFile(input_path, 'r') as zf:
+ metadata = chapter_extractor._extract_epub_metadata(zf)
+ chapters = chapter_extractor.extract_chapters(zf, out)
+
+ print(f"\n📚 Extraction Summary:")
+ print(f" Total chapters extracted: {len(chapters)}")
+ if chapters:
+ nums = [c.get('num', 0) for c in chapters]
+ print(f" Chapter range: {min(nums)} to {max(nums)}")
+
+ # Check for gaps in the sequence
+ expected_count = max(nums) - min(nums) + 1
+ if len(chapters) < expected_count:
+ print(f"\n⚠️ Potential missing chapters detected:")
+ print(f" Expected {expected_count} chapters (from {min(nums)} to {max(nums)})")
+ print(f" Actually found: {len(chapters)} chapters")
+ print(f" Potentially missing: {expected_count - len(chapters)} chapters")
+
+ validate_chapter_continuity(chapters)
+
+ print("\n" + "="*50)
+ validate_epub_structure(out)
+ print("="*50 + "\n")
+
+ progress_manager.migrate_to_content_hash(chapters)
+ progress_manager.save()
+
+ if check_stop():
+ return
+
+ metadata_path = os.path.join(out, "metadata.json")
+ if os.path.exists(metadata_path):
+ with open(metadata_path, 'r', encoding='utf-8') as mf:
+ metadata = json.load(mf)
+
+ metadata["chapter_count"] = len(chapters)
+ metadata["chapter_titles"] = {str(c["num"]): c["title"] for c in chapters}
+
+ print(f"[DEBUG] Initializing client with model = {config.MODEL}")
+ client = UnifiedClient(api_key=config.API_KEY, model=config.MODEL, output_dir=out)
+ if hasattr(client, 'use_multi_keys') and client.use_multi_keys:
+ stats = client.get_stats()
+ print(f"🔑 Multi-key mode active: {stats.get('total_keys', 0)} keys loaded")
+ print(f" Active keys: {stats.get('active_keys', 0)}")
+ else:
+ print(f"🔑 Single-key mode: Using {config.MODEL}")
+
+ # Reset cleanup state when starting new translation
+ if hasattr(client, 'reset_cleanup_state'):
+ client.reset_cleanup_state()
+
+ if "title" in metadata and config.TRANSLATE_BOOK_TITLE and not metadata.get("title_translated", False):
+ original_title = metadata["title"]
+ print(f"📚 Original title: {original_title}")
+
+ if not check_stop():
+ translated_title = translate_title(
+ original_title,
+ client,
+ None,
+ None,
+ config.TEMP
+ )
+
+ metadata["original_title"] = original_title
+ metadata["title"] = translated_title
+ metadata["title_translated"] = True
+
+ print(f"📚 Translated title: {translated_title}")
+ else:
+ print("❌ Title translation skipped due to stop request")
+
+ # Translate other metadata fields if configured
+ translate_metadata_fields_str = os.getenv('TRANSLATE_METADATA_FIELDS', '{}')
+ metadata_translation_mode = os.getenv('METADATA_TRANSLATION_MODE', 'together')
+
+ try:
+ translate_metadata_fields = json.loads(translate_metadata_fields_str)
+
+ if translate_metadata_fields and any(translate_metadata_fields.values()):
+ # Filter out fields that should be translated (excluding already translated fields)
+ fields_to_translate = {}
+ skipped_fields = []
+
+ for field_name, should_translate in translate_metadata_fields.items():
+ if should_translate and field_name != 'title' and field_name in metadata:
+ # Check if already translated
+ if metadata.get(f"{field_name}_translated", False):
+ skipped_fields.append(field_name)
+ print(f"✓ Skipping {field_name} - already translated")
+ else:
+ fields_to_translate[field_name] = should_translate
+
+ if fields_to_translate:
+ print("\n" + "="*50)
+ print("📋 METADATA TRANSLATION PHASE")
+ print("="*50)
+ print(f"🌐 Translating {len(fields_to_translate)} metadata fields...")
+
+ # Get ALL configuration from environment - NO DEFAULTS
+ system_prompt = os.getenv('BOOK_TITLE_SYSTEM_PROMPT', '')
+ if not system_prompt:
+ print("❌ No system prompt configured, skipping metadata translation")
+ else:
+ # Get field-specific prompts
+ field_prompts_str = os.getenv('METADATA_FIELD_PROMPTS', '{}')
+ try:
+ field_prompts = json.loads(field_prompts_str)
+ except:
+ field_prompts = {}
+
+ if not field_prompts and not field_prompts.get('_default'):
+ print("❌ No field prompts configured, skipping metadata translation")
+ else:
+ # Get language configuration
+ lang_behavior = os.getenv('LANG_PROMPT_BEHAVIOR', 'auto')
+ forced_source_lang = os.getenv('FORCED_SOURCE_LANG', 'Korean')
+ output_language = os.getenv('OUTPUT_LANGUAGE', 'English')
+
+ # Determine source language
+ source_lang = metadata.get('language', '').lower()
+ if lang_behavior == 'never':
+ lang_str = ""
+ elif lang_behavior == 'always':
+ lang_str = forced_source_lang
+ else: # auto
+ if 'zh' in source_lang or 'chinese' in source_lang:
+ lang_str = 'Chinese'
+ elif 'ja' in source_lang or 'japanese' in source_lang:
+ lang_str = 'Japanese'
+ elif 'ko' in source_lang or 'korean' in source_lang:
+ lang_str = 'Korean'
+ else:
+ lang_str = ''
+
+ # Check if batch translation is enabled for parallel processing
+ batch_translate_enabled = os.getenv('BATCH_TRANSLATION', '0') == '1'
+ batch_size = int(os.getenv('BATCH_SIZE', '50')) # Default batch size
+
+ if batch_translate_enabled and len(fields_to_translate) > 1:
+ print(f"⚡ Using parallel metadata translation mode ({len(fields_to_translate)} fields, batch size: {batch_size})...")
+
+ # Import ThreadPoolExecutor for parallel processing
+ from concurrent.futures import ThreadPoolExecutor, as_completed
+ import threading
+
+ # Thread-safe results storage
+ translation_results = {}
+ results_lock = threading.Lock()
+
+ def translate_metadata_field(field_name, original_value):
+ """Translate a single metadata field"""
+ try:
+ print(f"\n📋 Translating {field_name}: {original_value[:100]}..."
+ if len(str(original_value)) > 100 else f"\n📋 Translating {field_name}: {original_value}")
+
+ # Get field-specific prompt
+ prompt_template = field_prompts.get(field_name, field_prompts.get('_default', ''))
+
+ if not prompt_template:
+ print(f"⚠️ No prompt configured for field '{field_name}', skipping")
+ return None
+
+ # Replace variables in prompt
+ field_prompt = prompt_template.replace('{source_lang}', lang_str)
+ field_prompt = field_prompt.replace('{output_lang}', output_language)
+ field_prompt = field_prompt.replace('English', output_language)
+ field_prompt = field_prompt.replace('{field_value}', str(original_value))
+
+ # Check if we're using a translation service (not AI)
+ client_type = getattr(client, 'client_type', '')
+ is_translation_service = client_type in ['deepl', 'google_translate']
+
+ if is_translation_service:
+ # For translation services, send only the field value without AI prompts
+ print(f"🌐 Using translation service ({client_type}) - sending field directly")
+ messages = [
+ {"role": "user", "content": str(original_value)}
+ ]
+ else:
+ # For AI services, use prompts as before
+ messages = [
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": f"{field_prompt}\n\n{original_value}"}
+ ]
+
+ # Add delay for rate limiting
+ if config.DELAY > 0:
+ time.sleep(config.DELAY)
+
+ # Make API call
+ content, finish_reason = client.send(
+ messages,
+ temperature=config.TEMP,
+ max_tokens=config.MAX_OUTPUT_TOKENS
+ )
+ translated_value = content.strip()
+
+ # Store result thread-safely
+ with results_lock:
+ translation_results[field_name] = {
+ 'original': original_value,
+ 'translated': translated_value,
+ 'success': True
+ }
+
+ print(f"✅ Translated {field_name}: {translated_value}")
+ return translated_value
+
+ except Exception as e:
+ print(f"❌ Failed to translate {field_name}: {e}")
+ with results_lock:
+ translation_results[field_name] = {
+ 'original': original_value,
+ 'translated': None,
+ 'success': False,
+ 'error': str(e)
+ }
+ return None
+
+ # Execute parallel translations with limited workers
+ max_workers = min(len(fields_to_translate), batch_size)
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ # Submit all translation tasks
+ futures = {}
+ for field_name in fields_to_translate:
+ if field_name in metadata and not check_stop():
+ original_value = metadata[field_name]
+ future = executor.submit(translate_metadata_field, field_name, original_value)
+ futures[future] = field_name
+
+ # Wait for completion
+ for future in as_completed(futures):
+ if check_stop():
+ print("❌ Metadata translation stopped by user")
+ break
+
+ # Apply results to metadata
+ for field_name, result in translation_results.items():
+ if result['success'] and result['translated']:
+ metadata[f"original_{field_name}"] = result['original']
+ metadata[field_name] = result['translated']
+ metadata[f"{field_name}_translated"] = True
+
+ else:
+ # Sequential translation mode (individual translation)
+ mode_desc = "sequential" if not batch_translate_enabled else "sequential (single field)"
+ print(f"📝 Using {mode_desc} translation mode...")
+
+ for field_name in fields_to_translate:
+ if not check_stop() and field_name in metadata:
+ original_value = metadata[field_name]
+ print(f"\n📋 Translating {field_name}: {original_value[:100]}..."
+ if len(str(original_value)) > 100 else f"\n📋 Translating {field_name}: {original_value}")
+
+ # Get field-specific prompt
+ prompt_template = field_prompts.get(field_name, field_prompts.get('_default', ''))
+
+ if not prompt_template:
+ print(f"⚠️ No prompt configured for field '{field_name}', skipping")
+ continue
+
+ # Replace variables in prompt
+ field_prompt = prompt_template.replace('{source_lang}', lang_str)
+ field_prompt = field_prompt.replace('{output_lang}', output_language)
+ field_prompt = field_prompt.replace('English', output_language)
+ field_prompt = field_prompt.replace('{field_value}', str(original_value))
+
+ # Check if we're using a translation service (not AI)
+ client_type = getattr(client, 'client_type', '')
+ is_translation_service = client_type in ['deepl', 'google_translate']
+
+ if is_translation_service:
+ # For translation services, send only the field value without AI prompts
+ print(f"🌐 Using translation service ({client_type}) - sending field directly")
+ messages = [
+ {"role": "user", "content": str(original_value)}
+ ]
+ else:
+ # For AI services, use prompts as before
+ messages = [
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": f"{field_prompt}\n\n{original_value}"}
+ ]
+
+ try:
+ # Add delay using the config instance from main()
+ if config.DELAY > 0: # ✅ FIXED - use config.DELAY instead of config.SEND_INTERVAL
+ time.sleep(config.DELAY)
+
+ # Use the same client instance from main()
+ # ✅ FIXED - Properly unpack tuple response and provide max_tokens
+ content, finish_reason = client.send(
+ messages,
+ temperature=config.TEMP,
+ max_tokens=config.MAX_OUTPUT_TOKENS # ✅ FIXED - provide max_tokens to avoid NoneType error
+ )
+ translated_value = content.strip() # ✅ FIXED - use content from unpacked tuple
+
+ metadata[f"original_{field_name}"] = original_value
+ metadata[field_name] = translated_value
+ metadata[f"{field_name}_translated"] = True
+
+ print(f"✅ Translated {field_name}: {translated_value}")
+
+ except Exception as e:
+ print(f"❌ Failed to translate {field_name}: {e}")
+
+ else:
+ if check_stop():
+ print("❌ Metadata translation stopped by user")
+ break
+ else:
+ print("📋 No additional metadata fields to translate")
+
+ except Exception as e:
+ print(f"⚠️ Error processing metadata translation settings: {e}")
+ import traceback
+ traceback.print_exc()
+
+ with open(metadata_path, 'w', encoding='utf-8') as mf:
+ json.dump(metadata, mf, ensure_ascii=False, indent=2)
+ print(f"💾 Saved metadata with {'translated' if metadata.get('title_translated', False) else 'original'} title")
+
+ print("\n" + "="*50)
+ print("📑 GLOSSARY GENERATION PHASE")
+ print("="*50)
+
+ print(f"📑 DEBUG: ENABLE_AUTO_GLOSSARY = '{os.getenv('ENABLE_AUTO_GLOSSARY', 'NOT SET')}'")
+ print(f"📑 DEBUG: MANUAL_GLOSSARY = '{config.MANUAL_GLOSSARY}'")
+ print(f"📑 DEBUG: Manual glossary exists? {os.path.isfile(config.MANUAL_GLOSSARY) if config.MANUAL_GLOSSARY else False}")
+
+ # Check if glossary.csv already exists in the source folder
+ existing_glossary_csv = os.path.join(out, "glossary.csv")
+ existing_glossary_json = os.path.join(out, "glossary.json")
+ print(f"📑 DEBUG: Existing glossary.csv? {os.path.exists(existing_glossary_csv)}")
+ print(f"📑 DEBUG: Existing glossary.json? {os.path.exists(existing_glossary_json)}")
+
+ if config.MANUAL_GLOSSARY and os.path.isfile(config.MANUAL_GLOSSARY):
+ ext = os.path.splitext(config.MANUAL_GLOSSARY)[1].lower()
+ target_name = "glossary.csv" if ext == ".csv" else "glossary.json"
+ target_path = os.path.join(out, target_name)
+ if os.path.abspath(config.MANUAL_GLOSSARY) != os.path.abspath(target_path):
+ shutil.copy(config.MANUAL_GLOSSARY, target_path)
+ print("📑 Using manual glossary from:", config.MANUAL_GLOSSARY)
+ else:
+ print("📑 Using existing glossary:", config.MANUAL_GLOSSARY)
+ elif os.path.exists(existing_glossary_csv) or os.path.exists(existing_glossary_json):
+ print("📑 Existing glossary file detected in source folder - skipping automatic generation")
+ if os.path.exists(existing_glossary_csv):
+ print(f"📑 Using existing glossary.csv: {existing_glossary_csv}")
+ elif os.path.exists(existing_glossary_json):
+ print(f"📑 Using existing glossary.json: {existing_glossary_json}")
+ elif os.getenv("ENABLE_AUTO_GLOSSARY", "0") == "1":
+ model = os.getenv("MODEL", "gpt-4")
+ if is_traditional_translation_api(model):
+ print("📑 Automatic glossary generation disabled")
+ print(f" {model} does not support glossary extraction")
+ print(" Traditional translation APIs cannot identify character names/terms")
+ else:
+ print("📑 Starting automatic glossary generation...")
+ try:
+ # Use the new process-safe glossary worker
+ from glossary_process_worker import generate_glossary_in_process
+ import concurrent.futures
+ import multiprocessing
+
+ instructions = ""
+
+ # Get extraction workers setting
+ extraction_workers = int(os.getenv("EXTRACTION_WORKERS", "1"))
+ if extraction_workers == 1:
+ # Auto-detect for better performance
+ extraction_workers = min(os.cpu_count() or 4, 4)
+ print(f"📑 Using {extraction_workers} CPU cores for glossary generation")
+
+ # Collect environment variables to pass to subprocess
+ env_vars = {}
+ important_vars = [
+ 'EXTRACTION_WORKERS', 'GLOSSARY_MIN_FREQUENCY', 'GLOSSARY_MAX_NAMES',
+ 'GLOSSARY_MAX_TITLES', 'GLOSSARY_BATCH_SIZE', 'GLOSSARY_STRIP_HONORIFICS',
+ 'GLOSSARY_FUZZY_THRESHOLD', 'GLOSSARY_MAX_TEXT_SIZE', 'AUTO_GLOSSARY_PROMPT',
+ 'GLOSSARY_USE_SMART_FILTER', 'GLOSSARY_USE_LEGACY_CSV', 'GLOSSARY_PARALLEL_ENABLED',
+ 'GLOSSARY_FILTER_MODE', 'GLOSSARY_SKIP_FREQUENCY_CHECK', 'GLOSSARY_SKIP_ALL_VALIDATION',
+ 'MODEL', 'API_KEY', 'OPENAI_API_KEY', 'GEMINI_API_KEY', 'MAX_OUTPUT_TOKENS',
+ 'GLOSSARY_TEMPERATURE', 'MANUAL_GLOSSARY', 'ENABLE_AUTO_GLOSSARY'
+ ]
+
+ for var in important_vars:
+ if var in os.environ:
+ env_vars[var] = os.environ[var]
+
+ # Create a Queue for real-time log streaming
+ manager = multiprocessing.Manager()
+ log_queue = manager.Queue()
+
+ # Use ProcessPoolExecutor for true parallelism (completely bypasses GIL)
+ print("📑 Starting glossary generation in separate process...")
+ with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor:
+ # Submit to separate process WITH log queue
+ future = executor.submit(
+ generate_glossary_in_process,
+ out,
+ chapters,
+ instructions,
+ env_vars,
+ log_queue # Pass the queue for real-time logs
+ )
+
+ # Poll for completion and stream logs in real-time
+ poll_count = 0
+ while not future.done():
+ poll_count += 1
+
+ # Check for logs from subprocess and print them immediately
+ try:
+ while not log_queue.empty():
+ log_line = log_queue.get_nowait()
+ print(log_line) # Print to GUI
+ except:
+ pass
+
+ # Super short sleep to yield to GUI
+ time.sleep(0.001)
+
+ # Check for stop every 100 polls
+ if poll_count % 100 == 0:
+ if check_stop():
+ print("📑 ❌ Glossary generation cancelled")
+ executor.shutdown(wait=False, cancel_futures=True)
+ return
+
+ # Get any remaining logs from queue
+ try:
+ while not log_queue.empty():
+ log_line = log_queue.get_nowait()
+ print(log_line)
+ except:
+ pass
+
+ # Get result
+ if future.done():
+ try:
+ result = future.result(timeout=0.1)
+ if isinstance(result, dict):
+ if result.get('success'):
+ print(f"📑 ✅ Glossary generation completed successfully")
+ else:
+ print(f"📑 ❌ Glossary generation failed: {result.get('error')}")
+ if result.get('traceback'):
+ print(f"📑 Error details:\n{result.get('traceback')}")
+ except Exception as e:
+ print(f"📑 ❌ Error retrieving glossary result: {e}")
+
+ print("✅ Automatic glossary generation COMPLETED")
+
+ # Handle deferred glossary appending
+ if os.getenv('DEFER_GLOSSARY_APPEND') == '1':
+ print("📑 Processing deferred glossary append to system prompt...")
+
+ glossary_path = find_glossary_file(out)
+ if glossary_path and os.path.exists(glossary_path):
+ try:
+ glossary_block = None
+ if glossary_path.lower().endswith('.csv'):
+ with open(glossary_path, 'r', encoding='utf-8') as f:
+ glossary_block = f.read()
+ else:
+ with open(glossary_path, 'r', encoding='utf-8') as f:
+ glossary_data = json.load(f)
+
+ formatted_entries = {}
+ if isinstance(glossary_data, dict) and 'entries' in glossary_data:
+ formatted_entries = glossary_data['entries']
+ elif isinstance(glossary_data, dict):
+ formatted_entries = {k: v for k, v in glossary_data.items() if k != "metadata"}
+
+ if formatted_entries:
+ glossary_block = json.dumps(formatted_entries, ensure_ascii=False, indent=2)
+ else:
+ glossary_block = None
+
+ if glossary_block:
+ glossary_prompt = os.getenv('GLOSSARY_APPEND_PROMPT',
+ "Character/Term Glossary (use these translations consistently):")
+
+ current_prompt = config.PROMPT
+ if current_prompt:
+ current_prompt += "\n\n"
+ current_prompt += f"{glossary_prompt}\n{glossary_block}"
+
+ config.PROMPT = current_prompt
+
+ print(f"✅ Added auto-generated glossary to system prompt ({os.path.basename(glossary_path)})")
+
+ if 'DEFER_GLOSSARY_APPEND' in os.environ:
+ del os.environ['DEFER_GLOSSARY_APPEND']
+ if 'GLOSSARY_APPEND_PROMPT' in os.environ:
+ del os.environ['GLOSSARY_APPEND_PROMPT']
+ else:
+ print("⚠️ Auto-generated glossary has no entries - skipping append")
+ if 'DEFER_GLOSSARY_APPEND' in os.environ:
+ del os.environ['DEFER_GLOSSARY_APPEND']
+ if 'GLOSSARY_APPEND_PROMPT' in os.environ:
+ del os.environ['GLOSSARY_APPEND_PROMPT']
+ except Exception as e:
+ print(f"⚠️ Failed to append auto-generated glossary: {e}")
+ else:
+ print("⚠️ No glossary file found after automatic generation")
+
+ except Exception as e:
+ print(f"❌ Glossary generation failed: {e}")
+ else:
+ print("📑 Automatic glossary generation disabled")
+ # Don't create an empty glossary - let any existing manual glossary remain
+
+ glossary_file = find_glossary_file(out)
+ if glossary_file and os.path.exists(glossary_file):
+ try:
+ if glossary_file.lower().endswith('.csv'):
+ # Quick CSV stats
+ with open(glossary_file, 'r', encoding='utf-8') as f:
+ lines = [ln.strip() for ln in f.readlines() if ln.strip()]
+ entry_count = max(0, len(lines) - 1) if lines and ',' in lines[0] else len(lines)
+ print(f"📑 Glossary ready (CSV) with {entry_count} entries")
+ print("📑 Sample glossary lines:")
+ for ln in lines[1:4]:
+ print(f" • {ln}")
+ else:
+ with open(glossary_file, 'r', encoding='utf-8') as f:
+ glossary_data = json.load(f)
+
+ if isinstance(glossary_data, dict):
+ if 'entries' in glossary_data and isinstance(glossary_data['entries'], dict):
+ entry_count = len(glossary_data['entries'])
+ sample_items = list(glossary_data['entries'].items())[:3]
+ else:
+ entry_count = len(glossary_data)
+ sample_items = list(glossary_data.items())[:3]
+
+ print(f"📑 Glossary ready with {entry_count} entries")
+ print("📑 Sample glossary entries:")
+ for key, value in sample_items:
+ print(f" • {key} → {value}")
+
+ elif isinstance(glossary_data, list):
+ print(f"📑 Glossary ready with {len(glossary_data)} entries")
+ print("📑 Sample glossary entries:")
+ for i, entry in enumerate(glossary_data[:3]):
+ if isinstance(entry, dict):
+ original = entry.get('original_name', '?')
+ translated = entry.get('name', original)
+ print(f" • {original} → {translated}")
+ else:
+ print(f"⚠️ Unexpected glossary format: {type(glossary_data)}")
+
+ except Exception as e:
+ print(f"⚠️ Failed to inspect glossary file: {e}")
+ else:
+ print("📑 No glossary file found")
+
+ print("="*50)
+ print("🚀 STARTING MAIN TRANSLATION PHASE")
+ print("="*50 + "\n")
+
+ glossary_path = find_glossary_file(out)
+ if glossary_path and os.path.exists(glossary_path) and glossary_path.lower().endswith('.json'):
+ try:
+ with open(glossary_path, 'r', encoding='utf-8') as f:
+ g_data = json.load(f)
+
+ print(f"[DEBUG] Glossary type before translation: {type(g_data)}")
+ if isinstance(g_data, list):
+ print(f"[DEBUG] Glossary is a list")
+ except Exception as e:
+ print(f"[DEBUG] Error checking glossary: {e}")
+ glossary_path = find_glossary_file(out)
+ system = build_system_prompt(config.SYSTEM_PROMPT, glossary_path)
+ base_msg = [{"role": "system", "content": system}]
+ # Preserve the original system prompt to avoid in-place mutations
+ original_system_prompt = system
+ last_summary_block_text = None # Will hold the last rolling summary text for the NEXT chapter only
+
+ image_translator = None
+
+ if config.ENABLE_IMAGE_TRANSLATION:
+ print(f"🖼️ Image translation enabled for model: {config.MODEL}")
+ print("🖼️ Image translation will use your custom system prompt and glossary")
+ image_translator = ImageTranslator(
+ client,
+ out,
+ config.PROFILE_NAME,
+ system,
+ config.TEMP,
+ log_callback ,
+ progress_manager,
+ history_manager,
+ chunk_context_manager
+ )
+
+ known_vision_models = [
+ 'gemini-1.5-pro', 'gemini-1.5-flash', 'gemini-2.0-flash', 'gemini-2.5-flash', 'gemini-2.5-pro',
+ 'gpt-4-turbo', 'gpt-4o', 'gpt-4.1-mini', 'gpt-4.1-nano', 'o4-mini', 'gpt-4.1-mini'
+ ]
+
+ if config.MODEL.lower() not in known_vision_models:
+ print(f"⚠️ Note: {config.MODEL} may not have vision capabilities. Image translation will be attempted anyway.")
+ else:
+ print("ℹ️ Image translation disabled by user")
+
+ total_chapters = len(chapters)
+
+ # Only detect numbering if the toggle is not disabled
+ if config.DISABLE_ZERO_DETECTION:
+ print(f"📊 0-based detection disabled by user setting")
+ uses_zero_based = False
+ # Important: Set a flag that can be checked throughout the codebase
+ config._force_disable_zero_detection = True
+ else:
+ if chapters:
+ uses_zero_based = detect_novel_numbering(chapters)
+ print(f"📊 Novel numbering detected: {'0-based' if uses_zero_based else '1-based'}")
+ else:
+ uses_zero_based = False
+ config._force_disable_zero_detection = False
+
+ # Store this for later use
+ config._uses_zero_based = uses_zero_based
+
+
+ rng = os.getenv("CHAPTER_RANGE", "")
+ start = None
+ end = None
+ if rng and re.match(r"^\d+\s*-\s*\d+$", rng):
+ start, end = map(int, rng.split("-", 1))
+
+ if config.DISABLE_ZERO_DETECTION:
+ print(f"📊 0-based detection disabled - using range as specified: {start}-{end}")
+ elif uses_zero_based:
+ print(f"📊 0-based novel detected")
+ print(f"📊 User range {start}-{end} will be used as-is (chapters are already adjusted)")
+ else:
+ print(f"📊 1-based novel detected")
+ print(f"📊 Using range as specified: {start}-{end}")
+
+ print("📊 Calculating total chunks needed...")
+ total_chunks_needed = 0
+ chunks_per_chapter = {}
+ chapters_to_process = 0
+
+ # When setting actual chapter numbers (in the main function)
+ for idx, c in enumerate(chapters):
+ chap_num = c["num"]
+ content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
+
+ # Extract the raw chapter number from the file
+ raw_num = FileUtilities.extract_actual_chapter_number(c, patterns=None, config=config)
+ #print(f"[DEBUG] Extracted raw_num={raw_num} from {c.get('original_basename', 'unknown')}")
+
+
+ # Apply the offset
+ offset = config.CHAPTER_NUMBER_OFFSET if hasattr(config, 'CHAPTER_NUMBER_OFFSET') else 0
+ raw_num += offset
+
+ # When toggle is disabled, use raw numbers without any 0-based adjustment
+ if config.DISABLE_ZERO_DETECTION:
+ c['actual_chapter_num'] = raw_num
+ # Store raw number for consistency
+ c['raw_chapter_num'] = raw_num
+ c['zero_adjusted'] = False
+ else:
+ # Store raw number
+ c['raw_chapter_num'] = raw_num
+ # Apply adjustment only if this is a 0-based novel
+ if uses_zero_based:
+ c['actual_chapter_num'] = raw_num + 1
+ c['zero_adjusted'] = True
+ else:
+ c['actual_chapter_num'] = raw_num
+ c['zero_adjusted'] = False
+
+ # Now we can safely use actual_num
+ actual_num = c['actual_chapter_num']
+
+
+ if start is not None:
+ if not (start <= c['actual_chapter_num'] <= end):
+ #print(f"[SKIP] Chapter {c['actual_chapter_num']} outside range {start}-{end}")
+ continue
+
+ needs_translation, skip_reason, _ = progress_manager.check_chapter_status(
+ idx, actual_num, content_hash, out
+ )
+
+ if not needs_translation:
+ chunks_per_chapter[idx] = 0
+ continue
+
+ chapters_to_process += 1
+
+ chapter_key = str(actual_num)
+ if chapter_key in progress_manager.prog["chapters"] and progress_manager.prog["chapters"][chapter_key].get("status") == "in_progress":
+ pass
+
+ # Calculate based on OUTPUT limit only
+ max_output_tokens = config.MAX_OUTPUT_TOKENS
+ safety_margin_output = 500
+
+ # Korean to English typically compresses to 0.7-0.9x
+ compression_factor = config.COMPRESSION_FACTOR
+ available_tokens = int((max_output_tokens - safety_margin_output) / compression_factor)
+
+ # Ensure minimum
+ available_tokens = max(available_tokens, 1000)
+
+ #print(f"📊 Chunk size: {available_tokens:,} tokens (based on {max_output_tokens:,} output limit, compression: {compression_factor})")
+
+ # For mixed content chapters, calculate on clean text
+ # For mixed content chapters, calculate on clean text
+ if c.get('has_images', False) and ContentProcessor.is_meaningful_text_content(c["body"]):
+ # Don't modify c["body"] at all during chunk calculation
+ # Just pass the body as-is, the chunking will be slightly off but that's OK
+ chunks = chapter_splitter.split_chapter(c["body"], available_tokens)
+ else:
+ chunks = chapter_splitter.split_chapter(c["body"], available_tokens)
+
+ chapter_key_str = content_hash
+ old_key_str = str(idx)
+
+ if chapter_key_str not in progress_manager.prog.get("chapter_chunks", {}) and old_key_str in progress_manager.prog.get("chapter_chunks", {}):
+ progress_manager.prog["chapter_chunks"][chapter_key_str] = progress_manager.prog["chapter_chunks"][old_key_str]
+ del progress_manager.prog["chapter_chunks"][old_key_str]
+ #print(f"[PROGRESS] Migrated chunks for chapter {actual_num} to new tracking system")
+
+ # Always count actual chunks - ignore "completed" tracking
+ chunks_per_chapter[idx] = len(chunks)
+ total_chunks_needed += chunks_per_chapter[idx]
+
+ terminology = "Sections" if is_text_file else "Chapters"
+ print(f"📊 Total chunks to translate: {total_chunks_needed}")
+ print(f"📚 {terminology} to process: {chapters_to_process}")
+
+ multi_chunk_chapters = [(idx, count) for idx, count in chunks_per_chapter.items() if count > 1]
+ if multi_chunk_chapters:
+ # Determine terminology based on file type
+ terminology = "Sections" if is_text_file else "Chapters"
+ print(f"📄 {terminology} requiring multiple chunks:")
+ for idx, chunk_count in multi_chunk_chapters:
+ chap = chapters[idx]
+ section_term = "Section" if is_text_file else "Chapter"
+ print(f" • {section_term} {idx+1} ({chap['title'][:30]}...): {chunk_count} chunks")
+
+ translation_start_time = time.time()
+ chunks_completed = 0
+ chapters_completed = 0
+
+ current_chunk_number = 0
+
+ if config.BATCH_TRANSLATION:
+ print(f"\n📦 PARALLEL TRANSLATION MODE ENABLED")
+ print(f"📦 Processing chapters with up to {config.BATCH_SIZE} concurrent API calls")
+
+ import concurrent.futures
+ from threading import Lock
+
+ progress_lock = Lock()
+
+ chapters_to_translate = []
+
+ # FIX: First pass to set actual chapter numbers for ALL chapters
+ # This ensures batch mode has the same chapter numbering as non-batch mode
+ print("📊 Setting chapter numbers...")
+ for idx, c in enumerate(chapters):
+ raw_num = FileUtilities.extract_actual_chapter_number(c, patterns=None, config=config)
+
+ # Apply offset if configured
+ offset = config.CHAPTER_NUMBER_OFFSET if hasattr(config, 'CHAPTER_NUMBER_OFFSET') else 0
+ raw_num += offset
+
+ if config.DISABLE_ZERO_DETECTION:
+ # Use raw numbers without adjustment
+ c['actual_chapter_num'] = raw_num
+ c['raw_chapter_num'] = raw_num
+ c['zero_adjusted'] = False
+ else:
+ # Store raw number
+ c['raw_chapter_num'] = raw_num
+ # Apply 0-based adjustment if detected
+ if uses_zero_based:
+ c['actual_chapter_num'] = raw_num + 1
+ c['zero_adjusted'] = True
+ else:
+ c['actual_chapter_num'] = raw_num
+ c['zero_adjusted'] = False
+
+ for idx, c in enumerate(chapters):
+ chap_num = c["num"]
+ content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
+
+ # Check if this is a pre-split text chunk with decimal number
+ if (is_text_file and c.get('is_chunk', False) and isinstance(c['num'], float)):
+ actual_num = c['num'] # Preserve the decimal for text files only
+ else:
+ actual_num = c.get('actual_chapter_num', c['num']) # Now this will exist!
+
+ # Skip chapters outside the range
+ if start is not None and not (start <= actual_num <= end):
+ continue
+
+ # Check if chapter needs translation
+ needs_translation, skip_reason, existing_file = progress_manager.check_chapter_status(
+ idx, actual_num, content_hash, out, c # Pass the chapter object
+ )
+ # Add explicit file check for supposedly completed chapters
+ if not needs_translation and existing_file:
+ file_path = os.path.join(out, existing_file)
+ if not os.path.exists(file_path):
+ print(f"⚠️ Output file missing for chapter {actual_num}: {existing_file}")
+ needs_translation = True
+ skip_reason = None
+ # Update status to file_missing
+ progress_manager.update(idx, actual_num, content_hash, None, status="file_missing")
+ progress_manager.save()
+
+ if not needs_translation:
+ # Modify skip_reason to use appropriate terminology
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+
+ # Replace "Chapter" with appropriate terminology in skip_reason
+ skip_reason_modified = skip_reason.replace("Chapter", terminology)
+ print(f"[SKIP] {skip_reason_modified}")
+ chapters_completed += 1
+ continue
+
+ # Check for empty or image-only chapters
+ has_images = c.get('has_images', False)
+ has_meaningful_text = ContentProcessor.is_meaningful_text_content(c["body"])
+ text_size = c.get('file_size', 0)
+
+ is_empty_chapter = (not has_images and text_size < 10)
+ is_image_only_chapter = (has_images and not has_meaningful_text)
+
+ # Handle empty chapters
+ if is_empty_chapter:
+ print(f"📄 Empty chapter {chap_num} - will process individually")
+
+ safe_title = make_safe_filename(c['title'], c['num'])
+
+ if isinstance(c['num'], float):
+ fname = FileUtilities.create_chapter_filename(c, c['num'])
+ else:
+ fname = FileUtilities.create_chapter_filename(c, c['num'])
+ with open(os.path.join(out, fname), 'w', encoding='utf-8') as f:
+ f.write(c["body"])
+ progress_manager.update(idx, actual_num, content_hash, fname, status="completed_empty")
+ progress_manager.save()
+ chapters_completed += 1
+ continue
+
+ # Add to chapters to translate
+ chapters_to_translate.append((idx, c))
+
+ print(f"📊 Found {len(chapters_to_translate)} chapters to translate in parallel")
+
+ # Continue with the rest of the existing batch processing code...
+ batch_processor = BatchTranslationProcessor(
+ config, client, base_msg, out, progress_lock,
+ progress_manager.save,
+ lambda idx, actual_num, content_hash, output_file=None, status="completed", **kwargs: progress_manager.update(idx, actual_num, content_hash, output_file, status, **kwargs),
+ check_stop,
+ image_translator,
+ is_text_file=is_text_file
+ )
+
+ total_to_process = len(chapters_to_translate)
+ processed = 0
+
+ # Apply conservative batching setting
+ batch_multiplier = 3 if os.getenv('CONSERVATIVE_BATCHING', '0') == '1' else 1
+ batch_group_size = config.BATCH_SIZE * batch_multiplier
+
+ if batch_multiplier > 1:
+ print(f"📦 Using conservative batching: {batch_group_size} chapters per group, {config.BATCH_SIZE} parallel")
+ else:
+ print(f"📦 Using direct batching (default): {batch_group_size} chapters per group, {config.BATCH_SIZE} parallel")
+
+ with concurrent.futures.ThreadPoolExecutor(max_workers=config.BATCH_SIZE) as executor:
+ for batch_start in range(0, total_to_process, batch_group_size):
+ if check_stop():
+ print("❌ Translation stopped during parallel processing")
+ executor.shutdown(wait=False)
+ return
+
+ batch_end = min(batch_start + batch_group_size, total_to_process)
+ current_batch = chapters_to_translate[batch_start:batch_end]
+
+ batch_number = (batch_start // batch_group_size) + 1
+ print(f"\n📦 Submitting batch {batch_number}: {len(current_batch)} chapters")
+
+ future_to_chapter = {
+ executor.submit(batch_processor.process_single_chapter, chapter_data): chapter_data
+ for chapter_data in current_batch
+ }
+
+ active_count = 0
+ completed_in_batch = 0
+ failed_in_batch = 0
+
+ for future in concurrent.futures.as_completed(future_to_chapter):
+ if check_stop():
+ print("❌ Translation stopped")
+ executor.shutdown(wait=False)
+ return
+
+ chapter_data = future_to_chapter[future]
+ idx, chapter = chapter_data
+
+ try:
+ success, chap_num = future.result()
+ if success:
+ completed_in_batch += 1
+ print(f"✅ Chapter {chap_num} done ({completed_in_batch + failed_in_batch}/{len(current_batch)} in batch)")
+ else:
+ failed_in_batch += 1
+ print(f"❌ Chapter {chap_num} failed ({completed_in_batch + failed_in_batch}/{len(current_batch)} in batch)")
+ except Exception as e:
+ failed_in_batch += 1
+ print(f"❌ Chapter thread error: {e}")
+
+ processed += 1
+
+ progress_percent = (processed / total_to_process) * 100
+ print(f"📊 Overall Progress: {processed}/{total_to_process} ({progress_percent:.1f}%)")
+
+ print(f"\n📦 Batch Summary:")
+ print(f" ✅ Successful: {completed_in_batch}")
+ print(f" ❌ Failed: {failed_in_batch}")
+
+ if batch_end < total_to_process:
+ print(f"⏳ Waiting {config.DELAY}s before next batch...")
+ time.sleep(config.DELAY)
+
+ chapters_completed = batch_processor.chapters_completed
+ chunks_completed = batch_processor.chunks_completed
+
+ print(f"\n🎉 Parallel translation complete!")
+ print(f" Total chapters processed: {processed}")
+
+ # Count qa_failed chapters correctly
+ qa_failed_count = 0
+ actual_successful = 0
+
+ for idx, c in enumerate(chapters):
+ # Get the chapter's actual number
+ if (is_text_file and c.get('is_chunk', False) and isinstance(c['num'], float)):
+ actual_num = c['num']
+ else:
+ actual_num = c.get('actual_chapter_num', c['num'])
+
+ # Check if this chapter was processed and has qa_failed status
+ content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
+
+ # Check if this chapter exists in progress
+ chapter_info = progress_manager.prog["chapters"].get(content_hash, {})
+ status = chapter_info.get("status")
+
+ if status == "qa_failed":
+ qa_failed_count += 1
+ elif status == "completed":
+ actual_successful += 1
+
+ # Correct the displayed counts
+ print(f" Successful: {actual_successful}")
+ if qa_failed_count > 0:
+ print(f"\n⚠️ {qa_failed_count} chapters failed due to content policy violations:")
+ qa_failed_chapters = []
+ for idx, c in enumerate(chapters):
+ if (is_text_file and c.get('is_chunk', False) and isinstance(c['num'], float)):
+ actual_num = c['num']
+ else:
+ actual_num = c.get('actual_chapter_num', c['num'])
+
+ content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
+ chapter_info = progress_manager.prog["chapters"].get(content_hash, {})
+ if chapter_info.get("status") == "qa_failed":
+ qa_failed_chapters.append(actual_num)
+
+ print(f" Failed chapters: {', '.join(map(str, sorted(qa_failed_chapters)))}")
+
+ # Stop translation completely after batch mode
+ print("\n📌 Batch translation completed.")
+
+ elif not config.BATCH_TRANSLATION:
+ translation_processor = TranslationProcessor(config, client, out, log_callback, check_stop, uses_zero_based, is_text_file)
+
+ if config.DUPLICATE_DETECTION_MODE == 'ai-hunter':
+ # Build the main config from environment variables and config object
+ main_config = {
+ 'duplicate_lookback_chapters': config.DUPLICATE_LOOKBACK_CHAPTERS,
+ 'duplicate_detection_mode': config.DUPLICATE_DETECTION_MODE,
+ }
+
+ # Check if AI Hunter config was passed via environment variable
+ ai_hunter_config_str = os.getenv('AI_HUNTER_CONFIG')
+ if ai_hunter_config_str:
+ try:
+ ai_hunter_config = json.loads(ai_hunter_config_str)
+ main_config['ai_hunter_config'] = ai_hunter_config
+ print("🤖 AI Hunter: Loaded configuration from environment")
+ except json.JSONDecodeError:
+ print("⚠️ AI Hunter: Failed to parse AI_HUNTER_CONFIG from environment")
+
+ # If no AI Hunter config in environment, try to load from file as fallback
+ if 'ai_hunter_config' not in main_config:
+ # Try multiple locations for config.json
+ config_paths = [
+ os.path.join(os.getcwd(), 'config.json'),
+ os.path.join(out, '..', 'config.json'),
+ ]
+
+ if getattr(sys, 'frozen', False):
+ config_paths.append(os.path.join(os.path.dirname(sys.executable), 'config.json'))
+ else:
+ script_dir = os.path.dirname(os.path.abspath(__file__))
+ config_paths.extend([
+ os.path.join(script_dir, 'config.json'),
+ os.path.join(os.path.dirname(script_dir), 'config.json')
+ ])
+
+ for config_path in config_paths:
+ if os.path.exists(config_path):
+ try:
+ with open(config_path, 'r', encoding='utf-8') as f:
+ file_config = json.load(f)
+ if 'ai_hunter_config' in file_config:
+ main_config['ai_hunter_config'] = file_config['ai_hunter_config']
+ print(f"🤖 AI Hunter: Loaded configuration from {config_path}")
+ break
+ except Exception as e:
+ print(f"⚠️ Failed to load config from {config_path}: {e}")
+
+ # Always create and inject the improved AI Hunter when ai-hunter mode is selected
+ ai_hunter = ImprovedAIHunterDetection(main_config)
+
+ # The TranslationProcessor class has a method that checks for duplicates
+ # We need to replace it with our enhanced AI Hunter
+
+ # Create a wrapper to match the expected signature
+ def enhanced_duplicate_check(self, result, idx, prog, out, actual_num=None):
+ # If actual_num is not provided, try to get it from progress
+ if actual_num is None:
+ # Look for the chapter being processed
+ for ch_key, ch_info in prog.get("chapters", {}).items():
+ if ch_info.get("chapter_idx") == idx:
+ actual_num = ch_info.get("actual_num", idx + 1)
+ break
+
+ # Fallback to idx+1 if not found
+ if actual_num is None:
+ actual_num = idx + 1
+
+ return ai_hunter.detect_duplicate_ai_hunter_enhanced(result, idx, prog, out, actual_num)
+
+ # Bind the enhanced method to the processor instance
+ translation_processor.check_duplicate_content = enhanced_duplicate_check.__get__(translation_processor, TranslationProcessor)
+
+ print("🤖 AI Hunter: Using enhanced detection with configurable thresholds")
+
+ # First pass: set actual chapter numbers respecting the config
+ for idx, c in enumerate(chapters):
+ raw_num = FileUtilities.extract_actual_chapter_number(c, patterns=None, config=config)
+ #print(f"[DEBUG] Extracted raw_num={raw_num} from {c.get('original_basename', 'unknown')}")
+
+
+ # Apply offset if configured
+ offset = config.CHAPTER_NUMBER_OFFSET if hasattr(config, 'CHAPTER_NUMBER_OFFSET') else 0
+ raw_num += offset
+
+ if config.DISABLE_ZERO_DETECTION:
+ # Use raw numbers without adjustment
+ c['actual_chapter_num'] = raw_num
+ c['raw_chapter_num'] = raw_num
+ c['zero_adjusted'] = False
+ else:
+ # Store raw number
+ c['raw_chapter_num'] = raw_num
+ # Apply 0-based adjustment if detected
+ if uses_zero_based:
+ c['actual_chapter_num'] = raw_num + 1
+ c['zero_adjusted'] = True
+ else:
+ c['actual_chapter_num'] = raw_num
+ c['zero_adjusted'] = False
+
+ # Second pass: process chapters
+ for idx, c in enumerate(chapters):
+ chap_num = c["num"]
+
+ # Check if this is a pre-split text chunk with decimal number
+ if (is_text_file and c.get('is_chunk', False) and isinstance(c['num'], float)):
+ actual_num = c['num'] # Preserve the decimal for text files only
+ else:
+ actual_num = c.get('actual_chapter_num', c['num'])
+ content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
+
+ if start is not None and not (start <= actual_num <= end):
+ #print(f"[SKIP] Chapter {actual_num} (file: {c.get('original_basename', 'unknown')}) outside range {start}-{end}")
+ continue
+
+ needs_translation, skip_reason, existing_file = progress_manager.check_chapter_status(
+ idx, actual_num, content_hash, out, c # Pass the chapter object
+ )
+ # Add explicit file check for supposedly completed chapters
+ if not needs_translation and existing_file:
+ file_path = os.path.join(out, existing_file)
+ if not os.path.exists(file_path):
+ print(f"⚠️ Output file missing for chapter {actual_num}: {existing_file}")
+ needs_translation = True
+ skip_reason = None
+ # Update status to file_missing
+ progress_manager.update(idx, actual_num, content_hash, None, status="file_missing")
+ progress_manager.save()
+ if not needs_translation:
+ # Modify skip_reason to use appropriate terminology
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+
+ # Replace "Chapter" with appropriate terminology in skip_reason
+ skip_reason_modified = skip_reason.replace("Chapter", terminology)
+ print(f"[SKIP] {skip_reason_modified}")
+ continue
+
+ chapter_position = f"{chapters_completed + 1}/{chapters_to_process}"
+
+ # Determine if this is a text file
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+
+ # Determine file reference based on type
+ if c.get('is_chunk', False):
+ file_ref = f"Section_{c['num']}"
+ else:
+ file_ref = c.get('original_basename', f'{terminology}_{actual_num}')
+
+ print(f"\n🔄 Processing #{idx+1}/{total_chapters} (Actual: {terminology} {actual_num}) ({chapter_position} to translate): {c['title']} [File: {file_ref}]")
+
+ chunk_context_manager.start_chapter(chap_num, c['title'])
+
+ has_images = c.get('has_images', False)
+ has_meaningful_text = ContentProcessor.is_meaningful_text_content(c["body"])
+ text_size = c.get('file_size', 0)
+
+ is_empty_chapter = (not has_images and text_size < 10)
+ is_image_only_chapter = (has_images and not has_meaningful_text)
+ is_mixed_content = (has_images and has_meaningful_text)
+ is_text_only = (not has_images and has_meaningful_text)
+
+ if is_empty_chapter:
+ print(f"📄 Empty chapter {actual_num} detected")
+
+ # Create filename for empty chapter
+ if isinstance(c['num'], float):
+ fname = FileUtilities.create_chapter_filename(c, c['num'])
+ else:
+ fname = FileUtilities.create_chapter_filename(c, actual_num)
+
+ # Save original content
+ with open(os.path.join(out, fname), 'w', encoding='utf-8') as f:
+ f.write(c["body"])
+
+ # Update progress tracking
+ progress_manager.update(idx, actual_num, content_hash, fname, status="completed_empty")
+ progress_manager.save()
+ chapters_completed += 1
+
+ # CRITICAL: Skip translation!
+ continue
+
+ elif is_image_only_chapter:
+ print(f"📸 Image-only chapter: {c.get('image_count', 0)} images")
+
+ translated_html = c["body"]
+ image_translations = {}
+
+ # Step 1: Process images if image translation is enabled
+ if image_translator and config.ENABLE_IMAGE_TRANSLATION:
+ print(f"🖼️ Translating {c.get('image_count', 0)} images...")
+ image_translator.set_current_chapter(chap_num)
+
+ translated_html, image_translations = process_chapter_images(
+ c["body"],
+ actual_num,
+ image_translator,
+ check_stop
+ )
+
+ if image_translations:
+ print(f"✅ Translated {len(image_translations)} images")
+
+ # Step 2: Check for headers/titles that need translation
+ from bs4 import BeautifulSoup
+ soup = BeautifulSoup(c["body"], 'html.parser')
+
+ # Look for headers
+ headers = soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'title'])
+
+ # If we have headers, we should translate them even in "image-only" chapters
+ if headers and any(h.get_text(strip=True) for h in headers):
+ print(f"📝 Found headers to translate in image-only chapter")
+
+ # Create a minimal HTML with just the headers for translation
+ headers_html = ""
+ for header in headers:
+ if header.get_text(strip=True):
+ headers_html += str(header) + "\n"
+
+ if headers_html:
+ print(f"📤 Translating chapter headers...")
+
+ # Send just the headers for translation
+ header_msgs = base_msg + [{"role": "user", "content": headers_html}]
+
+ # Use the standard filename
+ fname = FileUtilities.create_chapter_filename(c, actual_num)
+ client.set_output_filename(fname)
+
+ # Simple API call for headers
+ header_result, _ = client.send(
+ header_msgs,
+ temperature=config.TEMP,
+ max_tokens=config.MAX_OUTPUT_TOKENS
+ )
+
+ if header_result:
+ # Clean the result
+ header_result = re.sub(r"^```(?:html)?\s*\n?", "", header_result, count=1, flags=re.MULTILINE)
+ header_result = re.sub(r"\n?```\s*$", "", header_result, count=1, flags=re.MULTILINE)
+
+ # Parse both the translated headers and the original body
+ soup_headers = BeautifulSoup(header_result, 'html.parser')
+ soup_body = BeautifulSoup(translated_html, 'html.parser')
+
+ # Replace headers in the body with translated versions
+ translated_headers = soup_headers.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'title'])
+ original_headers = soup_body.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'title'])
+
+ # Match and replace headers
+ for orig, trans in zip(original_headers, translated_headers):
+ if trans and trans.get_text(strip=True):
+ orig.string = trans.get_text(strip=True)
+
+ translated_html = str(soup_body)
+ print(f"✅ Headers translated successfully")
+ status = "completed"
+ else:
+ print(f"⚠️ Failed to translate headers")
+ status = "completed_image_only"
+ else:
+ status = "completed_image_only"
+ else:
+ print(f"ℹ️ No headers found to translate")
+ status = "completed_image_only"
+
+ # Step 3: Save with correct filename
+ fname = FileUtilities.create_chapter_filename(c, actual_num)
+
+ with open(os.path.join(out, fname), 'w', encoding='utf-8') as f:
+ f.write(translated_html)
+
+ print(f"[Chapter {idx+1}/{total_chapters}] ✅ Saved image-only chapter")
+ progress_manager.update(idx, actual_num, content_hash, fname, status=status)
+ progress_manager.save()
+ chapters_completed += 1
+ continue
+
+ else:
+ # Set default text to translate
+ text_to_translate = c["body"]
+ image_translations = {}
+ if is_mixed_content and image_translator and config.ENABLE_IMAGE_TRANSLATION:
+ print(f"🖼️ Processing {c.get('image_count', 0)} images first...")
+
+ print(f"[DEBUG] Content before image processing (first 200 chars):")
+ print(c["body"][:200])
+ print(f"[DEBUG] Has h1 tags: {'' in c['body']}")
+ print(f"[DEBUG] Has h2 tags: {'' in c['body']}")
+
+ image_translator.set_current_chapter(chap_num)
+
+ # Store the original body before processing
+ original_body = c["body"]
+
+ # Calculate original chapter tokens before modification
+ original_chapter_tokens = chapter_splitter.count_tokens(original_body)
+
+ # Process images and get body with translations
+ body_with_images, image_translations = process_chapter_images(
+ c["body"],
+ actual_num,
+ image_translator,
+ check_stop
+ )
+
+ if image_translations:
+ print(f"✅ Translated {len(image_translations)} images")
+
+ # Store the body with images for later merging
+ c["body_with_images"] = c["body"]
+
+ # For chapters with only images and title, we still need to translate the title
+ # Extract clean text for translation from ORIGINAL body
+ from bs4 import BeautifulSoup
+ soup_clean = BeautifulSoup(original_body, 'html.parser')
+
+ # Remove images from the original to get pure text
+ for img in soup_clean.find_all('img'):
+ img.decompose()
+
+ # Set clean text for translation - use prettify() or str() on the full document
+ c["body"] = str(soup_clean) if soup_clean.body else original_body
+
+ # If there's no meaningful text content after removing images,
+ # the text translation will just translate the title, which is correct
+ print(f" 📝 Clean text for translation: {len(c['body'])} chars")
+
+ # Update text_size to reflect actual text to translate
+ text_size = len(c["body"])
+
+ # Recalculate the actual token count for clean text
+ actual_text_tokens = chapter_splitter.count_tokens(c["body"])
+ print(f" 📊 Actual text tokens: {actual_text_tokens} (was counting {original_chapter_tokens} with images)")
+ else:
+ print(f"ℹ️ No translatable text found in images")
+ # Keep original body if no image translations
+ c["body"] = original_body
+
+ print(f"📖 Translating text content ({text_size} characters)")
+ progress_manager.update(idx, actual_num, content_hash, output_file=None, status="in_progress")
+ progress_manager.save()
+
+ # Apply ignore filtering to the content before chunk splitting
+ batch_translate_active = os.getenv('BATCH_TRANSLATE_HEADERS', '0') == '1'
+ ignore_title_tag = os.getenv('IGNORE_TITLE', '0') == '1' and batch_translate_active
+ ignore_header_tags = os.getenv('IGNORE_HEADER', '0') == '1' and batch_translate_active
+
+ if (ignore_title_tag or ignore_header_tags) and c["body"]:
+ from bs4 import BeautifulSoup
+ content_soup = BeautifulSoup(c["body"], 'html.parser')
+
+ # Remove title tags if ignored
+ if ignore_title_tag:
+ for title_tag in content_soup.find_all('title'):
+ title_tag.decompose()
+
+ # Remove header tags if ignored
+ if ignore_header_tags:
+ for header_tag in content_soup.find_all(['h1', 'h2', 'h3']):
+ header_tag.decompose()
+
+ c["body"] = str(content_soup) # Update the chapter body
+
+ # Check if this chapter is already a chunk from text file splitting
+ if c.get('is_chunk', False):
+ # This is already a pre-split chunk, but still check if it needs further splitting
+ # Calculate based on OUTPUT limit only
+ max_output_tokens = config.MAX_OUTPUT_TOKENS
+ safety_margin_output = 500
+
+ # CJK to English typically compresses to 0.7-0.9x
+ compression_factor = config.COMPRESSION_FACTOR
+ available_tokens = int((max_output_tokens - safety_margin_output) / compression_factor)
+
+ # Ensure minimum
+ available_tokens = max(available_tokens, 1000)
+
+ print(f"📊 Chunk size: {available_tokens:,} tokens (based on {max_output_tokens:,} output limit, compression: {compression_factor})")
+
+ chapter_tokens = chapter_splitter.count_tokens(c["body"])
+
+ if chapter_tokens > available_tokens:
+ # Even pre-split chunks might need further splitting
+ chunks = chapter_splitter.split_chapter(c["body"], available_tokens)
+ print(f"📄 Section {c['num']} (pre-split from text file) needs further splitting into {len(chunks)} chunks")
+ else:
+ chunks = [(c["body"], 1, 1)]
+ print(f"📄 Section {c['num']} (pre-split from text file)")
+ else:
+ # Normal splitting logic for non-text files
+ # Calculate based on OUTPUT limit only
+ max_output_tokens = config.MAX_OUTPUT_TOKENS
+ safety_margin_output = 500
+
+ # CJK to English typically compresses to 0.7-0.9x
+ compression_factor = config.COMPRESSION_FACTOR
+ available_tokens = int((max_output_tokens - safety_margin_output) / compression_factor)
+
+ # Ensure minimum
+ available_tokens = max(available_tokens, 1000)
+
+ print(f"📊 Chunk size: {available_tokens:,} tokens (based on {max_output_tokens:,} output limit, compression: {compression_factor})")
+
+ chunks = chapter_splitter.split_chapter(c["body"], available_tokens)
+
+ # Use consistent terminology
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+ print(f"📄 {terminology} will be processed in {len(chunks)} chunk(s)")
+
+ # Recalculate tokens on the actual text to be translated
+ actual_chapter_tokens = chapter_splitter.count_tokens(c["body"])
+
+ if len(chunks) > 1:
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+ print(f" ℹ️ {terminology} size: {actual_chapter_tokens:,} tokens (limit: {available_tokens:,} tokens per chunk)")
+ else:
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+ print(f" ℹ️ {terminology} size: {actual_chapter_tokens:,} tokens (within limit of {available_tokens:,} tokens)")
+
+ chapter_key_str = str(idx)
+ if chapter_key_str not in progress_manager.prog["chapter_chunks"]:
+ progress_manager.prog["chapter_chunks"][chapter_key_str] = {
+ "total": len(chunks),
+ "completed": [],
+ "chunks": {}
+ }
+
+ progress_manager.prog["chapter_chunks"][chapter_key_str]["total"] = len(chunks)
+
+ translated_chunks = []
+
+ for chunk_html, chunk_idx, total_chunks in chunks:
+ chapter_key_str = content_hash
+ old_key_str = str(idx)
+
+ if chapter_key_str not in progress_manager.prog.get("chapter_chunks", {}) and old_key_str in progress_manager.prog.get("chapter_chunks", {}):
+ progress_manager.prog["chapter_chunks"][chapter_key_str] = progress_manager.prog["chapter_chunks"][old_key_str]
+ del progress_manager.prog["chapter_chunks"][old_key_str]
+ #print(f"[PROGRESS] Migrated chunks for chapter {chap_num} to new tracking system")
+
+ if chapter_key_str not in progress_manager.prog["chapter_chunks"]:
+ progress_manager.prog["chapter_chunks"][chapter_key_str] = {
+ "total": len(chunks),
+ "completed": [],
+ "chunks": {}
+ }
+
+ progress_manager.prog["chapter_chunks"][chapter_key_str]["total"] = len(chunks)
+
+ # Get chapter status to check for qa_failed
+ chapter_info = progress_manager.prog["chapters"].get(chapter_key_str, {})
+ chapter_status = chapter_info.get("status")
+
+ if chapter_status == "qa_failed":
+ # Force retranslation of qa_failed chapters
+ print(f" [RETRY] Chunk {chunk_idx}/{total_chunks} - retranslating due to QA failure")
+
+ if config.CONTEXTUAL and history_manager.will_reset_on_next_append(config.HIST_LIMIT):
+ print(f" 📌 History will reset after this chunk (current: {len(history_manager.load_history())//2}/{config.HIST_LIMIT} exchanges)")
+
+ if check_stop():
+ print(f"❌ Translation stopped during chapter {actual_num}, chunk {chunk_idx}")
+ return
+
+ current_chunk_number += 1
+
+ progress_percent = (current_chunk_number / total_chunks_needed) * 100 if total_chunks_needed > 0 else 0
+
+ if chunks_completed > 0:
+ elapsed_time = time.time() - translation_start_time
+ avg_time_per_chunk = elapsed_time / chunks_completed
+ remaining_chunks = total_chunks_needed - current_chunk_number + 1
+ eta_seconds = remaining_chunks * avg_time_per_chunk
+
+ eta_hours = int(eta_seconds // 3600)
+ eta_minutes = int((eta_seconds % 3600) // 60)
+ eta_str = f"{eta_hours}h {eta_minutes}m" if eta_hours > 0 else f"{eta_minutes}m"
+ else:
+ eta_str = "calculating..."
+
+ if total_chunks > 1:
+ print(f" 🔄 Translating chunk {chunk_idx}/{total_chunks} for #{idx+1} (Overall: {current_chunk_number}/{total_chunks_needed} - {progress_percent:.1f}% - ETA: {eta_str})")
+ print(f" ⏳ Chunk size: {len(chunk_html):,} characters (~{chapter_splitter.count_tokens(chunk_html):,} tokens)")
+ else:
+ # Determine terminology and file reference
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+
+ # Consistent file reference
+ if c.get('is_chunk', False):
+ file_ref = f"Section_{c['num']}"
+ else:
+ file_ref = c.get('original_basename', f'{terminology}_{actual_num}')
+
+ print(f" 📄 Translating {terminology.lower()} content (Overall: {current_chunk_number}/{total_chunks_needed} - {progress_percent:.1f}% - ETA: {eta_str}) [File: {file_ref}]")
+ print(f" 📊 {terminology} {actual_num} size: {len(chunk_html):,} characters (~{chapter_splitter.count_tokens(chunk_html):,} tokens)")
+
+ print(f" ℹ️ This may take 30-60 seconds. Stop will take effect after completion.")
+
+ if log_callback:
+ if hasattr(log_callback, '__self__') and hasattr(log_callback.__self__, 'append_chunk_progress'):
+ if total_chunks == 1:
+ # Determine terminology based on source type
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+
+ log_callback.__self__.append_chunk_progress(
+ 1, 1, "text",
+ f"{terminology} {actual_num}",
+ overall_current=current_chunk_number,
+ overall_total=total_chunks_needed,
+ extra_info=f"{len(chunk_html):,} chars"
+ )
+ else:
+ log_callback.__self__.append_chunk_progress(
+ chunk_idx,
+ total_chunks,
+ "text",
+ f"{terminology} {actual_num}",
+ overall_current=current_chunk_number,
+ overall_total=total_chunks_needed
+ )
+ else:
+ # Determine terminology based on source type
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+ terminology_lower = "section" if is_text_source else "chapter"
+
+ if total_chunks == 1:
+ log_callback(f"📄 Processing {terminology} {actual_num} ({chapters_completed + 1}/{chapters_to_process}) - {progress_percent:.1f}% complete")
+ else:
+ log_callback(f"📄 processing chunk {chunk_idx}/{total_chunks} for {terminology_lower} {actual_num} - {progress_percent:.1f}% complete")
+
+ # Get custom chunk prompt template from environment
+ chunk_prompt_template = os.getenv("TRANSLATION_CHUNK_PROMPT", "[PART {chunk_idx}/{total_chunks}]\n{chunk_html}")
+
+ if total_chunks > 1:
+ user_prompt = chunk_prompt_template.format(
+ chunk_idx=chunk_idx,
+ total_chunks=total_chunks,
+ chunk_html=chunk_html
+ )
+ else:
+ user_prompt = chunk_html
+
+ if config.CONTEXTUAL:
+ history = history_manager.load_history()
+ trimmed = history[-config.HIST_LIMIT*2:]
+ chunk_context = chunk_context_manager.get_context_messages(limit=2)
+ else:
+ history = [] # Set empty history when not contextual
+ trimmed = []
+ chunk_context = []
+
+ # Build the current system prompt from the original each time, and append the last summary block if present
+ current_system_content = original_system_prompt
+ if config.USE_ROLLING_SUMMARY and last_summary_block_text:
+ current_system_content = (
+ current_system_content
+ + "\n\n[Rolling Summary of Previous Chapter]\n"
+ + "(For AI: Use as context only; do not include in output)\n"
+ + last_summary_block_text
+ + "\n[End of Rolling Summary]"
+ )
+ current_base = [{"role": "system", "content": current_system_content}]
+ # If we have a prepared rolling summary from previous chapter, include it as a separate message (do NOT mutate system prompt)
+ summary_msgs_list = []
+ if config.USE_ROLLING_SUMMARY and last_summary_block_text:
+ summary_msgs_list = [{
+ "role": os.getenv("SUMMARY_ROLE", "user"),
+ "content": (
+ "CONTEXT ONLY - DO NOT INCLUDE IN TRANSLATION:\n"
+ "[MEMORY] Previous context summary:\n\n"
+ f"{last_summary_block_text}\n\n"
+ "[END MEMORY]\n"
+ "END OF CONTEXT - BEGIN ACTUAL CONTENT TO TRANSLATE:"
+ )
+ }]
+ msgs = current_base + summary_msgs_list + chunk_context + trimmed + [{"role": "user", "content": user_prompt}]
+
+ c['__index'] = idx
+ c['__progress'] = progress_manager.prog
+ c['history_manager'] = history_manager
+
+ result, finish_reason = translation_processor.translate_with_retry(
+ msgs, chunk_html, c, chunk_idx, total_chunks
+ )
+
+ if result is None:
+ progress_manager.update(idx, actual_num, content_hash, output_file=None, status="failed")
+ progress_manager.save()
+ continue
+
+ if config.REMOVE_AI_ARTIFACTS:
+ result = ContentProcessor.clean_ai_artifacts(result, True)
+
+ if config.EMERGENCY_RESTORE:
+ result = ContentProcessor.emergency_restore_paragraphs(result, chunk_html)
+
+ if config.REMOVE_AI_ARTIFACTS:
+ lines = result.split('\n')
+
+ json_line_count = 0
+ for i, line in enumerate(lines[:5]):
+ if line.strip() and any(pattern in line for pattern in [
+ '"role":', '"content":', '"messages":',
+ '{"role"', '{"content"', '[{', '}]'
+ ]):
+ json_line_count = i + 1
+ else:
+ break
+
+ if json_line_count > 0 and json_line_count < len(lines):
+ remaining = '\n'.join(lines[json_line_count:])
+ if remaining.strip() and len(remaining) > 100:
+ result = remaining
+ print(f"✂️ Removed {json_line_count} lines of JSON artifacts")
+
+ result = re.sub(r'\[PART \d+/\d+\]\s*', '', result, flags=re.IGNORECASE)
+
+ translated_chunks.append((result, chunk_idx, total_chunks))
+
+ chunk_context_manager.add_chunk(user_prompt, result, chunk_idx, total_chunks)
+
+ progress_manager.prog["chapter_chunks"][chapter_key_str]["completed"].append(chunk_idx)
+ progress_manager.prog["chapter_chunks"][chapter_key_str]["chunks"][str(chunk_idx)] = result
+ progress_manager.save()
+
+ chunks_completed += 1
+
+ will_reset = history_manager.will_reset_on_next_append(
+ config.HIST_LIMIT if config.CONTEXTUAL else 0,
+ config.TRANSLATION_HISTORY_ROLLING
+ )
+
+
+ history = history_manager.append_to_history(
+ user_prompt,
+ result,
+ config.HIST_LIMIT if config.CONTEXTUAL else 0,
+ reset_on_limit=True,
+ rolling_window=config.TRANSLATION_HISTORY_ROLLING
+ )
+
+ if chunk_idx < total_chunks:
+ # Handle float delays while checking for stop
+ full_seconds = int(config.DELAY)
+ fractional_second = config.DELAY - full_seconds
+
+ # Check stop signal every second for full seconds
+ for i in range(full_seconds):
+ if check_stop():
+ print("❌ Translation stopped during delay")
+ return
+ time.sleep(1)
+
+ # Handle the fractional part if any
+ if fractional_second > 0:
+ if check_stop():
+ print("❌ Translation stopped during delay")
+ return
+ time.sleep(fractional_second)
+
+ if check_stop():
+ print(f"❌ Translation stopped before saving chapter {actual_num}")
+ return
+
+ if len(translated_chunks) > 1:
+ print(f" 📎 Merging {len(translated_chunks)} chunks...")
+ translated_chunks.sort(key=lambda x: x[1])
+ merged_result = chapter_splitter.merge_translated_chunks(translated_chunks)
+ else:
+ merged_result = translated_chunks[0][0] if translated_chunks else ""
+
+ if config.CONTEXTUAL and len(translated_chunks) > 1:
+ user_summary, assistant_summary = chunk_context_manager.get_summary_for_history()
+
+ if user_summary and assistant_summary:
+ history_manager.append_to_history(
+ user_summary,
+ assistant_summary,
+ config.HIST_LIMIT,
+ reset_on_limit=False,
+ rolling_window=config.TRANSLATION_HISTORY_ROLLING
+ )
+ print(f" 📝 Added chapter summary to history")
+
+ chunk_context_manager.clear()
+
+ # For text file chunks, ensure we pass the decimal number
+ if is_text_file and c.get('is_chunk', False) and isinstance(c.get('num'), float):
+ fname = FileUtilities.create_chapter_filename(c, c['num']) # Use the decimal num directly
+ else:
+ fname = FileUtilities.create_chapter_filename(c, actual_num)
+
+ client.set_output_filename(fname)
+ cleaned = re.sub(r"^```(?:html)?\s*\n?", "", merged_result, count=1, flags=re.MULTILINE)
+ cleaned = re.sub(r"\n?```\s*$", "", cleaned, count=1, flags=re.MULTILINE)
+
+ cleaned = ContentProcessor.clean_ai_artifacts(cleaned, remove_artifacts=config.REMOVE_AI_ARTIFACTS)
+
+ if is_mixed_content and image_translations:
+ print(f"🔀 Merging {len(image_translations)} image translations with text...")
+ from bs4 import BeautifulSoup
+ # Parse the translated text (which has the translated title/header)
+ soup_translated = BeautifulSoup(cleaned, 'html.parser')
+
+ # For each image translation, insert it into the document
+ for img_path, translation_html in image_translations.items():
+ if translation_html and ' 0:
+ combined.write(f"\n\n{'='*50}\n\n")
+
+ # Write the original chapter title (without Part X/Y suffix)
+ original_title = chapter_data['title']
+ # Remove the (Part X/Y) suffix if present
+ if ' (Part ' in original_title:
+ original_title = original_title.split(' (Part ')[0]
+
+ combined.write(f"{original_title}\n\n")
+
+ # Add the chunk content
+ combined.write(content)
+
+ # Add spacing between chunks of the same chapter
+ if chunk_idx < total_chunks:
+ combined.write("\n\n")
+ else:
+ # This is a standalone chapter
+ current_main_chapter = chapter_data['num']
+
+ # Add separator if not first chapter
+ if i > 0:
+ combined.write(f"\n\n{'='*50}\n\n")
+
+ # Write the chapter title
+ combined.write(f"{chapter_data['title']}\n\n")
+
+ # Add the content
+ combined.write(content)
+
+ print(f" • Combined file with preserved sections: {combined_path}")
+
+ total_time = time.time() - translation_start_time
+ hours = int(total_time // 3600)
+ minutes = int((total_time % 3600) // 60)
+ seconds = int(total_time % 60)
+
+ print(f"\n⏱️ Total translation time: {hours}h {minutes}m {seconds}s")
+ print(f"📊 Chapters completed: {chapters_completed}")
+ print(f"✅ Text file translation complete!")
+
+ if log_callback:
+ log_callback(f"✅ Text file translation complete! Created {combined_path}")
+
+ except Exception as e:
+ print(f"❌ Error creating combined text file: {e}")
+ if log_callback:
+ log_callback(f"❌ Error creating combined text file: {e}")
+ else:
+ print("🔍 Checking for translated chapters...")
+ # Respect retain extension toggle: if enabled, don't look for response_ prefix
+ if should_retain_source_extension():
+ response_files = [f for f in os.listdir(out) if f.endswith('.html') and not f.startswith('chapter_')]
+ else:
+ response_files = [f for f in os.listdir(out) if f.startswith('response_') and f.endswith('.html')]
+ chapter_files = [f for f in os.listdir(out) if f.startswith('chapter_') and f.endswith('.html')]
+
+ if not response_files and chapter_files:
+ if should_retain_source_extension():
+ print(f"⚠️ No translated files found, but {len(chapter_files)} original chapters exist")
+ print("ℹ️ Retain-source-extension mode is ON: skipping placeholder creation and using original files for EPUB compilation.")
+ else:
+ print(f"⚠️ No translated files found, but {len(chapter_files)} original chapters exist")
+ print("📝 Creating placeholder response files for EPUB compilation...")
+
+ for chapter_file in chapter_files:
+ response_file = chapter_file.replace('chapter_', 'response_', 1)
+ src = os.path.join(out, chapter_file)
+ dst = os.path.join(out, response_file)
+
+ try:
+ with open(src, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ soup = BeautifulSoup(content, 'html.parser')
+ notice = soup.new_tag('p')
+ notice.string = "[Note: This chapter could not be translated - showing original content]"
+ notice['style'] = "color: red; font-style: italic;"
+
+ if soup.body:
+ soup.body.insert(0, notice)
+
+ with open(dst, 'w', encoding='utf-8') as f:
+ f.write(str(soup))
+
+ except Exception as e:
+ print(f"⚠️ Error processing {chapter_file}: {e}")
+ try:
+ shutil.copy2(src, dst)
+ except:
+ pass
+
+ print(f"✅ Created {len(chapter_files)} placeholder response files")
+ print("⚠️ Note: The EPUB will contain untranslated content")
+
+ print("📘 Building final EPUB…")
+ try:
+ from epub_converter import fallback_compile_epub
+ fallback_compile_epub(out, log_callback=log_callback)
+ print("✅ All done: your final EPUB is in", out)
+
+ total_time = time.time() - translation_start_time
+ hours = int(total_time // 3600)
+ minutes = int((total_time % 3600) // 60)
+ seconds = int(total_time % 60)
+
+ print(f"\n📊 Translation Statistics:")
+ print(f" • Total chunks processed: {chunks_completed}")
+ print(f" • Total time: {hours}h {minutes}m {seconds}s")
+ if chunks_completed > 0:
+ avg_time = total_time / chunks_completed
+ print(f" • Average time per chunk: {avg_time:.1f} seconds")
+
+ stats = progress_manager.get_stats(out)
+ print(f"\n📊 Progress Tracking Summary:")
+ print(f" • Total chapters tracked: {stats['total_tracked']}")
+ print(f" • Successfully completed: {stats['completed']}")
+ print(f" • Missing files: {stats['missing_files']}")
+ print(f" • In progress: {stats['in_progress']}")
+
+ except Exception as e:
+ print("❌ EPUB build failed:", e)
+
+ print("TRANSLATION_COMPLETE_SIGNAL")
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/ai_hunter_enhanced.py b/ai_hunter_enhanced.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d244296a815022b35c326963af2143fd26c0cc8
--- /dev/null
+++ b/ai_hunter_enhanced.py
@@ -0,0 +1,1385 @@
+# ai_hunter_enhanced.py
+# Combined AI Hunter configuration GUI and detection logic
+
+import tkinter as tk
+from tkinter import ttk
+import ttkbootstrap as tb
+import json
+import os
+import re
+import unicodedata
+from difflib import SequenceMatcher
+from collections import Counter
+
+class AIHunterConfigGUI:
+ """GUI for configuring AI Hunter detection parameters"""
+ def __init__(self, parent, config_dict, callback=None):
+ """
+ Initialize with reference to main config dictionary
+
+ Args:
+ parent: Parent window
+ config_dict: Reference to main translator config dictionary
+ callback: Function to call after saving
+ """
+ self.parent = parent
+ self.config = config_dict # Reference to main config
+ self.callback = callback
+ self.window = None
+
+ # Default AI Hunter settings structure
+ self.default_ai_hunter = {
+ 'enabled': True,
+ 'ai_hunter_max_workers': 1,
+ 'retry_attempts': 6,
+ 'disable_temperature_change': False,
+ 'sample_size': 3000,
+ 'thresholds': {
+ 'exact': 90,
+ 'text': 35,
+ 'semantic': 85,
+ 'structural': 85,
+ 'character': 90,
+ 'pattern': 80
+ },
+ 'weights': {
+ 'exact': 1.5,
+ 'text': 1.2,
+ 'semantic': 1.0,
+ 'structural': 1.0,
+ 'character': 0.8,
+ 'pattern': 0.8
+ },
+ 'detection_mode': 'weighted_average',
+ 'multi_method_requirements': {
+ 'methods_required': 3,
+ 'min_methods': ['semantic', 'structural']
+ },
+ 'preprocessing': {
+ 'remove_html_spacing': True,
+ 'normalize_unicode': True,
+ 'ignore_case': True,
+ 'remove_extra_whitespace': True
+ },
+ 'edge_filters': {
+ 'min_text_length': 500,
+ 'max_length_ratio': 1.3,
+ 'min_length_ratio': 0.7
+ },
+ 'language_detection': {
+ 'enabled': False,
+ 'target_language': 'english',
+ 'threshold_characters': 500,
+ 'languages': {
+ 'english': ['en'],
+ 'japanese': ['ja', 'jp'],
+ 'korean': ['ko', 'kr'],
+ 'chinese': ['zh', 'zh-cn', 'zh-tw'],
+ 'spanish': ['es'],
+ 'french': ['fr'],
+ 'german': ['de'],
+ 'russian': ['ru'],
+ 'arabic': ['ar'],
+ 'hindi': ['hi'],
+ 'portuguese': ['pt'],
+ 'italian': ['it'],
+ 'dutch': ['nl'],
+ 'thai': ['th'],
+ 'vietnamese': ['vi'],
+ 'turkish': ['tr'],
+ 'polish': ['pl'],
+ 'swedish': ['sv'],
+ 'danish': ['da'],
+ 'norwegian': ['no'],
+ 'finnish': ['fi']
+ }
+ }
+ }
+
+ # Initialize AI Hunter config in main config if not present
+ if 'ai_hunter_config' not in self.config:
+ self.config['ai_hunter_config'] = self.default_ai_hunter.copy()
+ else:
+ # Merge with defaults to ensure all keys exist
+ self.config['ai_hunter_config'] = self._merge_configs(
+ self.default_ai_hunter,
+ self.config['ai_hunter_config']
+ )
+
+ def _merge_configs(self, default, existing):
+ """Recursively merge existing config with defaults"""
+ result = default.copy()
+ for key, value in existing.items():
+ if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+ result[key] = self._merge_configs(result[key], value)
+ else:
+ result[key] = value
+ return result
+
+ def get_ai_config(self):
+ """Get AI Hunter configuration from main config"""
+ return self.config.get('ai_hunter_config', self.default_ai_hunter)
+
+ def show_ai_hunter_config(self):
+ """Display the AI Hunter configuration window with scrollbar using WindowManager"""
+ if self.window and self.window.winfo_exists():
+ self.window.lift()
+ return
+
+ # Import WindowManager if not already available
+ if not hasattr(self, 'wm'):
+ from translator_gui import WindowManager
+ import sys
+ import os
+ base_dir = getattr(sys, '_MEIPASS', os.path.dirname(os.path.abspath(__file__)))
+ self.wm = WindowManager(base_dir)
+
+ # Create scrollable dialog using WindowManager
+ dialog, scrollable_frame, canvas = self.wm.setup_scrollable(
+ self.parent,
+ "AI Hunter Configuration",
+ width=820,
+ height=None, # Will use default height
+ max_width_ratio=0.9,
+ max_height_ratio=0.85
+ )
+
+ self.window = dialog
+
+ # Create notebook inside scrollable frame
+ notebook = ttk.Notebook(scrollable_frame)
+ notebook.pack(fill='both', expand=True, padx=10, pady=10)
+
+ # Tab 1: Detection Thresholds
+ self.create_thresholds_tab(notebook)
+
+ # Tab 2: Detection Mode
+ self.create_mode_tab(notebook)
+
+ # Tab 3: Preprocessing
+ self.create_preprocessing_tab(notebook)
+
+ # Tab 4: Advanced Settings
+ self.create_advanced_tab(notebook)
+
+ # Buttons at the bottom (inside scrollable frame)
+ button_frame = tk.Frame(scrollable_frame)
+ button_frame.pack(fill='x', padx=10, pady=(10, 20))
+
+ tb.Button(button_frame, text="Save", command=self.apply_ai_hunter_settings,
+ bootstyle="success").pack(side='right', padx=5)
+ tb.Button(button_frame, text="Cancel", command=self.window.destroy,
+ bootstyle="secondary").pack(side='right')
+ tb.Button(button_frame, text="Reset to Defaults", command=self.reset_defaults,
+ bootstyle="warning").pack(side='left')
+
+ # Auto-resize and show
+ self.wm.auto_resize_dialog(dialog, canvas, max_width_ratio=0.9, max_height_ratio=1.1)
+
+ # Handle window close
+ dialog.protocol("WM_DELETE_WINDOW", lambda: [dialog._cleanup_scrolling(), dialog.destroy()])
+
+ def create_thresholds_tab(self, notebook):
+ """Create the thresholds configuration tab"""
+ frame = ttk.Frame(notebook)
+ notebook.add(frame, text="Detection Thresholds")
+
+ # Title
+ tk.Label(frame, text="Detection Method Thresholds",
+ font=('TkDefaultFont', 12, 'bold')).pack(pady=10)
+
+ tk.Label(frame, text="Higher values = fewer false positives (more strict)\n"
+ "Lower values = more false positives (more sensitive)",
+ font=('TkDefaultFont', 10), fg='gray').pack(pady=(0, 20))
+
+ # Threshold controls
+ self.threshold_vars = {}
+ threshold_frame = tk.Frame(frame)
+ threshold_frame.pack(fill='both', expand=True, padx=20)
+
+ descriptions = {
+ 'exact': 'Exact Text Match - Direct character-by-character comparison',
+ 'text': 'Smart Text Similarity - Intelligent text comparison with sampling',
+ 'semantic': 'Semantic Analysis - Character names, dialogue patterns, numbers',
+ 'structural': 'Structural Patterns - Paragraph structure, dialogue distribution',
+ 'character': 'Character Overlap - Common character names between chapters',
+ 'pattern': 'Pattern Analysis - Narrative flow and structure patterns'
+ }
+
+ ai_config = self.get_ai_config()
+
+ for method, desc in descriptions.items():
+ method_frame = tk.Frame(threshold_frame)
+ method_frame.pack(fill='x', pady=10)
+
+ # Method name and description
+ label_frame = tk.Frame(method_frame)
+ label_frame.pack(fill='x')
+
+ tk.Label(label_frame, text=f"{method.title()}:",
+ font=('TkDefaultFont', 10, 'bold')).pack(side='left')
+ tk.Label(label_frame, text=f" {desc}",
+ font=('TkDefaultFont', 9), fg='gray').pack(side='left', padx=(10, 0))
+
+ # Slider and value
+ slider_frame = tk.Frame(method_frame)
+ slider_frame.pack(fill='x', pady=(5, 0))
+
+ self.threshold_vars[method] = tk.IntVar(value=ai_config['thresholds'][method])
+
+ slider = tb.Scale(slider_frame, from_=10, to=100,
+ variable=self.threshold_vars[method],
+ bootstyle="info", length=400)
+ slider.pack(side='left', padx=(20, 10))
+
+ value_label = tk.Label(slider_frame, text="", width=4)
+ value_label.pack(side='left')
+
+ # Update label when slider changes
+ def update_label(val, label=value_label, var=self.threshold_vars[method]):
+ label.config(text=f"{int(var.get())}%")
+
+ self.threshold_vars[method].trace('w', lambda *args, f=update_label: f(None))
+ update_label(None)
+
+ # Weight configuration
+ tk.Label(frame, text="Method Weights (for weighted average mode)",
+ font=('TkDefaultFont', 11, 'bold')).pack(pady=(30, 10))
+
+ self.weight_vars = {}
+ weight_frame = tk.Frame(frame)
+ weight_frame.pack(fill='x', padx=20)
+
+ for method in descriptions.keys():
+ w_frame = tk.Frame(weight_frame)
+ w_frame.pack(fill='x', pady=5)
+
+ tk.Label(w_frame, text=f"{method.title()} weight:", width=20,
+ anchor='w').pack(side='left')
+
+ self.weight_vars[method] = tk.DoubleVar(value=ai_config['weights'][method])
+
+ tb.Spinbox(w_frame, from_=0.1, to=2.0, increment=0.1,
+ textvariable=self.weight_vars[method],
+ width=10).pack(side='left', padx=10)
+
+ def create_mode_tab(self, notebook):
+ """Create the detection mode configuration tab"""
+ frame = ttk.Frame(notebook)
+ notebook.add(frame, text="Detection Mode")
+
+ tk.Label(frame, text="Detection Mode Configuration",
+ font=('TkDefaultFont', 12, 'bold')).pack(pady=10)
+
+ # Detection mode selection
+ mode_frame = tk.LabelFrame(frame, text="Detection Mode", padx=20, pady=20)
+ mode_frame.pack(fill='x', padx=20, pady=10)
+
+ ai_config = self.get_ai_config()
+ self.mode_var = tk.StringVar(value=ai_config['detection_mode'])
+
+ modes = [
+ ('single_method', 'Single Method',
+ 'Flag as duplicate if ANY method exceeds its threshold\n(Most sensitive, most false positives)'),
+ ('multi_method', 'Multi-Method Agreement',
+ 'Require multiple methods to agree before flagging\n(Balanced approach)'),
+ ('weighted_average', 'Weighted Average',
+ 'Calculate weighted average of all methods\n(Most nuanced, least false positives)')
+ ]
+
+ for value, text, desc in modes:
+ rb_frame = tk.Frame(mode_frame)
+ rb_frame.pack(fill='x', pady=10)
+
+ tb.Radiobutton(rb_frame, text=text, variable=self.mode_var,
+ value=value, bootstyle="primary").pack(anchor='w')
+ tk.Label(rb_frame, text=desc, font=('TkDefaultFont', 9),
+ fg='gray').pack(anchor='w', padx=(25, 0))
+
+ # Multi-method configuration
+ multi_frame = tk.LabelFrame(frame, text="Multi-Method Settings", padx=20, pady=20)
+ multi_frame.pack(fill='x', padx=20, pady=10)
+
+ tk.Label(multi_frame, text="Number of methods required to agree:",
+ font=('TkDefaultFont', 10)).pack(anchor='w')
+
+ self.methods_required_var = tk.IntVar(
+ value=ai_config['multi_method_requirements']['methods_required'])
+
+ tb.Spinbox(multi_frame, from_=1, to=6, textvariable=self.methods_required_var,
+ width=10).pack(anchor='w', pady=5)
+
+ tk.Label(multi_frame, text="Required methods (at least one must be included):",
+ font=('TkDefaultFont', 10)).pack(anchor='w', pady=(10, 5))
+
+ self.required_method_vars = {}
+ for method in ['exact', 'text', 'semantic', 'structural', 'character', 'pattern']:
+ var = tk.BooleanVar(
+ value=method in ai_config['multi_method_requirements']['min_methods'])
+ self.required_method_vars[method] = var
+
+ tb.Checkbutton(multi_frame, text=method.title(), variable=var,
+ bootstyle="round-toggle").pack(anchor='w', padx=20)
+
+ def create_preprocessing_tab(self, notebook):
+ """Create the preprocessing configuration tab"""
+ frame = ttk.Frame(notebook)
+ notebook.add(frame, text="Preprocessing")
+
+ tk.Label(frame, text="Text Preprocessing Options",
+ font=('TkDefaultFont', 12, 'bold')).pack(pady=10)
+
+ tk.Label(frame, text="Configure how text is processed before comparison",
+ font=('TkDefaultFont', 10), fg='gray').pack(pady=(0, 20))
+
+ # Preprocessing options
+ prep_frame = tk.Frame(frame)
+ prep_frame.pack(fill='both', expand=True, padx=20)
+
+ self.prep_vars = {}
+ ai_config = self.get_ai_config()
+
+ options = [
+ ('remove_html_spacing', 'Remove HTML with spacing',
+ 'Replace HTML tags with spaces instead of removing completely'),
+ ('normalize_unicode', 'Normalize Unicode',
+ 'Normalize unicode characters (recommended)'),
+ ('ignore_case', 'Case-insensitive comparison',
+ 'Ignore character case when comparing'),
+ ('remove_extra_whitespace', 'Remove extra whitespace',
+ 'Collapse multiple spaces/newlines into single spaces')
+ ]
+
+ for key, text, desc in options:
+ var = tk.BooleanVar(value=ai_config['preprocessing'][key])
+ self.prep_vars[key] = var
+
+ opt_frame = tk.Frame(prep_frame)
+ opt_frame.pack(fill='x', pady=10)
+
+ tb.Checkbutton(opt_frame, text=text, variable=var,
+ bootstyle="round-toggle").pack(anchor='w')
+ tk.Label(opt_frame, text=desc, font=('TkDefaultFont', 9),
+ fg='gray').pack(anchor='w', padx=(25, 0))
+
+ def create_advanced_tab(self, notebook):
+ """Create the advanced settings tab"""
+ frame = ttk.Frame(notebook)
+ notebook.add(frame, text="Advanced")
+
+ tk.Label(frame, text="Advanced Settings",
+ font=('TkDefaultFont', 12, 'bold')).pack(pady=10)
+
+ # General settings
+ general_frame = tk.LabelFrame(frame, text="General", padx=20, pady=20)
+ general_frame.pack(fill='x', padx=20, pady=10)
+
+ ai_config = self.get_ai_config()
+
+ # Add separator for better organization
+ ttk.Separator(general_frame, orient='horizontal').pack(fill='x', pady=(0, 10))
+
+ # Sample size
+ ss_frame = tk.Frame(general_frame)
+ ss_frame.pack(fill='x', pady=5)
+
+ tk.Label(ss_frame, text="Sample size:", width=20, anchor='w').pack(side='left')
+ self.sample_size_var = tk.IntVar(value=ai_config['sample_size'])
+ tb.Spinbox(ss_frame, from_=1000, to=10000, increment=500,
+ textvariable=self.sample_size_var, width=10).pack(side='left', padx=10)
+ tk.Label(ss_frame, text="characters",
+ font=('TkDefaultFont', 9)).pack(side='left')
+
+ # AI Hunter Behavior Settings
+ tk.Label(general_frame, text="AI Hunter Behavior",
+ font=('TkDefaultFont', 10, 'bold')).pack(anchor='w', pady=(0, 5))
+
+ # Retry Attempts
+ retry_frame = tk.Frame(general_frame)
+ retry_frame.pack(fill='x', pady=5)
+
+ tk.Label(retry_frame, text="Retry attempts:", width=20, anchor='w').pack(side='left')
+ self.retry_attempts_var = tk.IntVar(value=ai_config.get('retry_attempts', 3))
+ tb.Spinbox(retry_frame, from_=1, to=10, textvariable=self.retry_attempts_var, width=10).pack(side='left', padx=10)
+ tk.Label(retry_frame, text="attempts", font=('TkDefaultFont', 9)).pack(side='left')
+
+ # Temperature Change Toggle
+ temp_frame = tk.Frame(general_frame)
+ temp_frame.pack(fill='x', pady=10)
+
+ self.disable_temp_change_var = tk.BooleanVar(value=ai_config.get('disable_temperature_change', False))
+ tb.Checkbutton(temp_frame, text="Disable temperature change behavior",
+ variable=self.disable_temp_change_var, bootstyle="round-toggle").pack(anchor='w')
+ tk.Label(temp_frame, text="Prevents AI Hunter from modifying temperature settings during retries",
+ font=('TkDefaultFont', 9), fg='gray').pack(anchor='w', padx=(25, 0))
+
+ # Edge filters
+ edge_frame = tk.LabelFrame(frame, text="Edge Case Filters", padx=20, pady=20)
+ edge_frame.pack(fill='x', padx=20, pady=10)
+
+ # Min text length
+ min_frame = tk.Frame(edge_frame)
+ min_frame.pack(fill='x', pady=5)
+
+ tk.Label(min_frame, text="Minimum text length:", width=20, anchor='w').pack(side='left')
+ self.min_length_var = tk.IntVar(value=ai_config['edge_filters']['min_text_length'])
+ tb.Spinbox(min_frame, from_=100, to=2000, increment=100,
+ textvariable=self.min_length_var, width=10).pack(side='left', padx=10)
+ tk.Label(min_frame, text="characters",
+ font=('TkDefaultFont', 9)).pack(side='left')
+
+ # Length ratios
+ ratio_frame = tk.Frame(edge_frame)
+ ratio_frame.pack(fill='x', pady=10)
+
+ tk.Label(ratio_frame, text="Length ratio limits:").pack(anchor='w')
+
+ r_frame = tk.Frame(ratio_frame)
+ r_frame.pack(fill='x', pady=5)
+
+ tk.Label(r_frame, text="Min ratio:", width=10, anchor='w').pack(side='left', padx=(20, 5))
+ self.min_ratio_var = tk.DoubleVar(value=ai_config['edge_filters']['min_length_ratio'])
+ tb.Spinbox(r_frame, from_=0.5, to=0.9, increment=0.1,
+ textvariable=self.min_ratio_var, width=8).pack(side='left')
+
+ tk.Label(r_frame, text="Max ratio:", width=10, anchor='w').pack(side='left', padx=(20, 5))
+ self.max_ratio_var = tk.DoubleVar(value=ai_config['edge_filters']['max_length_ratio'])
+ tb.Spinbox(r_frame, from_=1.1, to=2.0, increment=0.1,
+ textvariable=self.max_ratio_var, width=8).pack(side='left')
+
+ tk.Label(edge_frame, text="Chapters with vastly different lengths won't be compared",
+ font=('TkDefaultFont', 9), fg='gray').pack(anchor='w', padx=20)
+
+ # Language Detection
+ lang_frame = tk.LabelFrame(frame, text="Non-Target Language Detection", padx=20, pady=20)
+ lang_frame.pack(fill='x', padx=20, pady=10)
+
+ # Enable toggle
+ enable_frame = tk.Frame(lang_frame)
+ enable_frame.pack(fill='x', pady=5)
+
+ self.lang_enabled_var = tk.BooleanVar(value=ai_config['language_detection']['enabled'])
+ tb.Checkbutton(enable_frame, text="Enable non-target language detection",
+ variable=self.lang_enabled_var, bootstyle="round-toggle").pack(anchor='w')
+ tk.Label(enable_frame, text="Trigger retranslation when too much non-target language is detected",
+ font=('TkDefaultFont', 9), fg='gray').pack(anchor='w', padx=(25, 0))
+
+ # Target language selection
+ target_frame = tk.Frame(lang_frame)
+ target_frame.pack(fill='x', pady=10)
+
+ tk.Label(target_frame, text="Target language:", width=20, anchor='w').pack(side='left')
+ self.target_lang_var = tk.StringVar(value=ai_config['language_detection']['target_language'])
+
+ lang_options = list(ai_config['language_detection']['languages'].keys())
+ target_combo = ttk.Combobox(target_frame, textvariable=self.target_lang_var,
+ values=lang_options, state='readonly', width=15)
+ target_combo.pack(side='left', padx=10)
+
+ tk.Label(target_frame, text="Language that should be in the translation",
+ font=('TkDefaultFont', 9), fg='gray').pack(side='left', padx=(10, 0))
+
+ # Threshold setting
+ thresh_frame = tk.Frame(lang_frame)
+ thresh_frame.pack(fill='x', pady=5)
+
+ tk.Label(thresh_frame, text="Character threshold:", width=20, anchor='w').pack(side='left')
+ self.lang_threshold_var = tk.IntVar(value=ai_config['language_detection']['threshold_characters'])
+ tb.Spinbox(thresh_frame, from_=100, to=2000, increment=50,
+ textvariable=self.lang_threshold_var, width=10).pack(side='left', padx=10)
+ tk.Label(thresh_frame, text="non-target language characters to trigger retranslation",
+ font=('TkDefaultFont', 9), fg='gray').pack(side='left')
+
+ def apply_ai_hunter_settings(self):
+ """Apply AI Hunter settings to the main config"""
+ ai_config = self.get_ai_config()
+
+ # Update from GUI variables
+ for method, var in self.threshold_vars.items():
+ ai_config['thresholds'][method] = var.get()
+
+ for method, var in self.weight_vars.items():
+ ai_config['weights'][method] = var.get()
+
+ ai_config['detection_mode'] = self.mode_var.get()
+ ai_config['multi_method_requirements']['methods_required'] = self.methods_required_var.get()
+
+ min_methods = [method for method, var in self.required_method_vars.items() if var.get()]
+ ai_config['multi_method_requirements']['min_methods'] = min_methods
+
+ for key, var in self.prep_vars.items():
+ ai_config['preprocessing'][key] = var.get()
+
+ ai_config['sample_size'] = self.sample_size_var.get()
+
+ ai_config['edge_filters']['min_text_length'] = self.min_length_var.get()
+ ai_config['edge_filters']['min_length_ratio'] = self.min_ratio_var.get()
+ ai_config['edge_filters']['max_length_ratio'] = self.max_ratio_var.get()
+
+ # Language detection settings
+ ai_config['language_detection']['enabled'] = self.lang_enabled_var.get()
+ ai_config['language_detection']['target_language'] = self.target_lang_var.get()
+ ai_config['language_detection']['threshold_characters'] = self.lang_threshold_var.get()
+
+ # Update retry attempts and temperature change settings
+ ai_config['retry_attempts'] = self.retry_attempts_var.get()
+ ai_config['disable_temperature_change'] = self.disable_temp_change_var.get()
+
+ # Update main config
+ self.config['ai_hunter_config'] = ai_config
+
+ # Call callback if provided (this should trigger main save_configuration)
+ if self.callback:
+ self.callback()
+
+ self.window.destroy()
+
+ def reset_defaults(self):
+ """Reset all values to defaults"""
+ import tkinter.messagebox as messagebox
+ result = messagebox.askyesno("Reset to Defaults",
+ "Are you sure you want to reset all settings to defaults?")
+ if result:
+ self.config['ai_hunter_config'] = self.default_ai_hunter.copy()
+ self.window.destroy()
+ self.show_ai_hunter_config() # Reopen with default values
+
+
+class ImprovedAIHunterDetection:
+ """Improved AI Hunter detection methods for TranslateKRtoEN"""
+
+ def __init__(self, main_config):
+ """
+ Initialize with reference to main config
+
+ Args:
+ main_config: Reference to main translator config dictionary
+ """
+ self.main_config = main_config
+
+ # Default AI Hunter settings
+ self.default_ai_hunter = {
+ 'enabled': True,
+ 'lookback_chapters': 5,
+ 'retry_attempts': 3,
+ 'disable_temperature_change': False,
+ 'sample_size': 3000,
+ 'thresholds': {
+ 'exact': 90,
+ 'text': 85,
+ 'semantic': 85,
+ 'structural': 85,
+ 'character': 80,
+ 'pattern': 80
+ },
+ 'weights': {
+ 'exact': 1.5,
+ 'text': 1.2,
+ 'semantic': 1.0,
+ 'structural': 1.0,
+ 'character': 0.8,
+ 'pattern': 0.8
+ },
+ 'detection_mode': 'multi_method',
+ 'multi_method_requirements': {
+ 'methods_required': 2,
+ 'min_methods': ['semantic', 'structural']
+ },
+ 'preprocessing': {
+ 'remove_html_spacing': True,
+ 'normalize_unicode': True,
+ 'ignore_case': True,
+ 'remove_extra_whitespace': True
+ },
+ 'edge_filters': {
+ 'min_text_length': 500,
+ 'max_length_ratio': 1.3,
+ 'min_length_ratio': 0.7
+ },
+ 'language_detection': {
+ 'enabled': False,
+ 'target_language': 'english',
+ 'threshold_characters': 500,
+ 'languages': {
+ 'english': ['en'],
+ 'japanese': ['ja', 'jp'],
+ 'korean': ['ko', 'kr'],
+ 'chinese': ['zh', 'zh-cn', 'zh-tw'],
+ 'spanish': ['es'],
+ 'french': ['fr'],
+ 'german': ['de'],
+ 'russian': ['ru'],
+ 'arabic': ['ar'],
+ 'hindi': ['hi'],
+ 'portuguese': ['pt'],
+ 'italian': ['it'],
+ 'dutch': ['nl'],
+ 'thai': ['th'],
+ 'vietnamese': ['vi'],
+ 'turkish': ['tr'],
+ 'polish': ['pl'],
+ 'swedish': ['sv'],
+ 'danish': ['da'],
+ 'norwegian': ['no'],
+ 'finnish': ['fi']
+ }
+ }
+ }
+
+ def get_ai_config(self):
+ """Get AI Hunter configuration from main config"""
+ return self.main_config.get('ai_hunter_config', self.default_ai_hunter)
+
+ def detect_duplicate_ai_hunter_enhanced(self, result, idx, prog, out, current_chapter_num=None):
+ """Enhanced AI Hunter duplicate detection with configurable parameters"""
+ try:
+ print(f"\n ========== AI HUNTER DEBUG START ==========")
+ print(f" 📍 Current chapter index: {idx}")
+ if current_chapter_num:
+ print(f" 📖 Current chapter number: {current_chapter_num}")
+
+ # Get configuration
+ config = self.get_ai_config()
+
+ if not config.get('enabled', True):
+ print(f" ⚠️ AI Hunter is disabled")
+ print(f" ========== AI HUNTER DEBUG END ==========\n")
+ return False, 0
+
+ # Preprocess text
+ result_clean = self._preprocess_text(result, config['preprocessing'])
+ print(f" 📄 Text length after preprocessing: {len(result_clean)} chars")
+
+ # Check for non-target language detection
+ if config['language_detection']['enabled']:
+ non_target_detected, non_target_count = self._check_non_target_language(
+ result_clean, config['language_detection']
+ )
+ if non_target_detected:
+ print(f"\n 🌐 NON-TARGET LANGUAGE DETECTED!")
+ print(f" Non-target characters found: {non_target_count}")
+ print(f" Threshold: {config['language_detection']['threshold_characters']}")
+ print(f" Target language: {config['language_detection']['target_language']}")
+ print(f" ========== AI HUNTER DEBUG END ==========\n")
+ return True, 100 # High confidence for language detection
+
+ # Check edge cases
+ if len(result_clean) < config['edge_filters']['min_text_length']:
+ print(f" ⚠️ Text too short ({len(result_clean)} < {config['edge_filters']['min_text_length']})")
+ print(f" ========== AI HUNTER DEBUG END ==========\n")
+ return False, 0
+
+ # Extract features
+ print(f" 🔬 Extracting text features...")
+ result_features = self._extract_text_features(result_clean)
+
+ # Get lookback from main config, then fall back to env var if not found
+ lookback = self.main_config.get('duplicate_lookback_chapters',
+ int(os.getenv('DUPLICATE_LOOKBACK_CHAPTERS', '5')))
+
+ # Log configuration
+ print(f"\n 🔧 Configuration:")
+ print(f" Detection mode: {config['detection_mode']}")
+ print(f" Lookback chapters: {lookback}")
+ print(f" Sample size: {config['sample_size']}")
+
+ # FIX: Get all completed chapters sorted by actual chapter number
+ completed_chapters = []
+ for chapter_key, chapter_info in prog["chapters"].items():
+ if chapter_info.get("status") == "completed" and chapter_info.get("output_file"):
+ # Handle both numeric and hash-based chapter keys
+ try:
+ # Get actual_num from progress (this is the real chapter number)
+ chapter_num = chapter_info.get("actual_num")
+ if chapter_num is None:
+ # Try chapter_num as fallback
+ chapter_num = chapter_info.get("chapter_num")
+ if chapter_num is None:
+ # Skip chapters without valid numbers
+ print(f" ⚠️ No chapter number found for key {chapter_key}, skipping")
+ continue
+
+ completed_chapters.append({
+ 'key': chapter_key,
+ 'num': chapter_num,
+ 'file': chapter_info.get("output_file"),
+ 'ai_features': chapter_info.get("ai_features")
+ })
+ except Exception as e:
+ print(f" ⚠️ Error processing chapter {chapter_key}: {e}")
+ continue
+
+ # Sort by actual chapter number
+ completed_chapters.sort(key=lambda x: x['num'])
+
+ # If no current chapter number provided, try to infer it
+ if current_chapter_num is None:
+ # The current chapter should be passed in, but if not, we need to find it
+ # Since we're using content hash keys, we can't use idx directly
+ print(f" ⚠️ No current chapter number provided")
+ print(f" 📊 Current index: {idx}")
+
+ # The current chapter number should have been passed from the wrapper
+ # If it wasn't, we have a problem
+ print(f" ❌ ERROR: Current chapter number not provided to AI Hunter!")
+ print(f" ❌ This indicates the wrapper function is not passing the chapter number correctly")
+
+ # Emergency: just use a high number so we don't compare against anything
+ current_chapter_num = 999999
+ print(f" ⚠️ Using index-based chapter number: {current_chapter_num}")
+
+ print(f"\n 📚 Found {len(completed_chapters)} completed chapters in progress")
+ if completed_chapters:
+ chapter_nums = [ch['num'] for ch in completed_chapters]
+ print(f" 📊 Chapter numbers in progress: {sorted(chapter_nums)[:10]}{'...' if len(chapter_nums) > 10 else ''}")
+ print(f" 🎯 Current chapter number: {current_chapter_num}")
+ print(f" 🔍 Will check against last {lookback} chapters before chapter {current_chapter_num}")
+
+ # Check previous chapters
+ all_similarities = []
+ highest_similarity = 0.0
+ detected_method = None
+ detected_chapter = None
+
+ # FIX: Look at chapters by actual number, not index
+ chapters_checked = 0
+ for completed_chapter in reversed(completed_chapters):
+ # Only check chapters that come before the current one
+ if completed_chapter['num'] >= current_chapter_num:
+ continue
+
+ # Only check up to lookback number of chapters
+ if chapters_checked >= lookback:
+ break
+
+ chapters_checked += 1
+
+ print(f"\n 📝 Checking against chapter {completed_chapter['num']}...")
+
+ # Get previous chapter features
+ prev_features = completed_chapter.get('ai_features')
+ prev_clean = None
+
+ # Try to get cached features first
+ if prev_features:
+ print(f" ✅ Using cached features")
+ else:
+ # Read and extract features
+ prev_path = os.path.join(out, completed_chapter['file'])
+
+ if os.path.exists(prev_path):
+ try:
+ with open(prev_path, 'r', encoding='utf-8') as f:
+ prev_content = f.read()
+ prev_clean = self._preprocess_text(prev_content, config['preprocessing'])
+
+ # Check length ratio
+ len_ratio = len(result_clean) / max(1, len(prev_clean))
+ if (len_ratio < config['edge_filters']['min_length_ratio'] or
+ len_ratio > config['edge_filters']['max_length_ratio']):
+ print(f" ⚠️ Length ratio out of bounds: {len_ratio:.2f}")
+ continue
+
+ prev_features = self._extract_text_features(prev_clean)
+ print(f" 📄 Extracted features from file")
+ except Exception as e:
+ print(f" ❌ Failed to read file: {e}")
+ continue
+ else:
+ print(f" ❌ File not found: {prev_path}")
+ continue
+
+ # Calculate similarities
+ print(f" 🔍 Calculating similarities...")
+ similarities = self._calculate_all_similarities(
+ result_clean, result_features,
+ prev_clean, prev_features, config
+ )
+
+ # Store for reporting
+ all_similarities.append({
+ 'chapter': completed_chapter['num'],
+ 'similarities': similarities
+ })
+
+ # Log similarity scores
+ for method, score in similarities.items():
+ if score > 0:
+ print(f" {method}: {int(score*100)}%")
+
+ # Check if duplicate based on configured mode
+ is_duplicate, confidence, methods_triggered = self._evaluate_duplicate(
+ similarities, config
+ )
+
+ if is_duplicate:
+ print(f"\n 🚨 DUPLICATE DETECTED!")
+ print(f" Detection mode: {config['detection_mode']}")
+ print(f" Confidence: {int(confidence*100)}%")
+ print(f" Triggered methods: {', '.join(methods_triggered)}")
+ print(f" Match with: Chapter {completed_chapter['num']}")
+ print(f" ========== AI HUNTER DEBUG END ==========\n")
+ return True, int(confidence * 100)
+
+ # Track highest for reporting
+ for method, sim in similarities.items():
+ if sim > highest_similarity:
+ highest_similarity = sim
+ detected_method = method
+ detected_chapter = completed_chapter['num']
+
+ # No duplicate found
+ print(f"\n ✅ No duplicate found")
+ if detected_method:
+ print(f" Highest similarity: {int(highest_similarity*100)}% via {detected_method}")
+ print(f" Closest match: Chapter {detected_chapter}")
+
+ # Show top 3 closest matches
+ if all_similarities:
+ print(f"\n 📊 Top 3 closest matches:")
+ sorted_chapters = sorted(all_similarities,
+ key=lambda x: self._get_chapter_score(x['similarities'], config),
+ reverse=True)[:3]
+ for i, chapter_data in enumerate(sorted_chapters, 1):
+ score = self._get_chapter_score(chapter_data['similarities'], config)
+ print(f" {i}. Chapter {chapter_data['chapter']}: {int(score*100)}%")
+
+ print(f" ========== AI HUNTER DEBUG END ==========\n")
+ return False, 0
+
+ except Exception as e:
+ print(f" ❌ AI Hunter detection failed with error: {e}")
+ import traceback
+ print(f" {traceback.format_exc()}")
+ print(f" ========== AI HUNTER DEBUG END ==========\n")
+ return False, 0
+
+ def _preprocess_text(self, text, prep_config):
+ """Preprocess text according to configuration"""
+ # Remove HTML
+ if prep_config.get('remove_html_spacing', True):
+ text = re.sub(r'<[^>]+>', ' ', text)
+ else:
+ text = re.sub(r'<[^>]+>', '', text)
+
+ # Normalize unicode
+ if prep_config.get('normalize_unicode', True):
+ text = unicodedata.normalize('NFKD', text)
+
+ # Remove extra whitespace
+ if prep_config.get('remove_extra_whitespace', True):
+ text = re.sub(r'\s+', ' ', text)
+ text = re.sub(r'\n\s*\n', '\n\n', text)
+
+ text = text.strip()
+
+ # Convert to lowercase if case-insensitive
+ if prep_config.get('ignore_case', True):
+ text = text.lower()
+
+ return text
+
+ def _calculate_all_similarities(self, result_clean, result_features,
+ prev_clean, prev_features, config):
+ """Calculate all similarity metrics"""
+ similarities = {}
+
+ # Method 1: Exact content match
+ if prev_clean is not None:
+ sample_size = min(config['sample_size'], len(result_clean), len(prev_clean))
+ exact_sim = self._calculate_exact_similarity(
+ result_clean[:sample_size],
+ prev_clean[:sample_size]
+ )
+ similarities['exact'] = exact_sim
+
+ # Method 2: Smart text similarity
+ text_sim = self._calculate_smart_similarity(
+ result_clean, prev_clean, config['sample_size']
+ )
+ similarities['text'] = text_sim
+ else:
+ similarities['exact'] = 0.0
+ similarities['text'] = 0.0
+
+ # Method 3: Semantic fingerprint
+ semantic_sim = self._calculate_semantic_similarity(
+ result_features.get('semantic', {}),
+ prev_features.get('semantic', {})
+ )
+ similarities['semantic'] = semantic_sim
+
+ # Method 4: Structural signature
+ structural_sim = self._calculate_structural_similarity(
+ result_features.get('structural', {}),
+ prev_features.get('structural', {})
+ )
+ similarities['structural'] = structural_sim
+
+ # Method 5: Character analysis
+ char_sim = self._calculate_character_similarity(
+ result_features.get('characters', []),
+ prev_features.get('characters', [])
+ )
+ similarities['character'] = char_sim
+
+ # Method 6: Pattern analysis
+ pattern_sim = self._calculate_pattern_similarity(
+ result_features.get('patterns', {}),
+ prev_features.get('patterns', {})
+ )
+ similarities['pattern'] = pattern_sim
+
+ return similarities
+
+ def _evaluate_duplicate(self, similarities, config):
+ """Evaluate if similarities indicate a duplicate based on detection mode"""
+ mode = config['detection_mode']
+ thresholds = {k: v/100.0 for k, v in config['thresholds'].items()}
+
+ if mode == 'single_method':
+ # Any method exceeding threshold
+ for method, sim in similarities.items():
+ if sim >= thresholds.get(method, 0.85):
+ return True, sim, [method]
+ return False, 0, []
+
+ elif mode == 'multi_method':
+ # Multiple methods must agree
+ triggered_methods = []
+ for method, sim in similarities.items():
+ if sim >= thresholds.get(method, 0.85):
+ triggered_methods.append(method)
+
+ # Check if enough methods triggered
+ required = config.get('multi_method_requirements', {}).get('methods_required', 2)
+ min_methods = config.get('multi_method_requirements', {}).get('min_methods', [])
+
+ if len(triggered_methods) >= required:
+ # Check if at least one required method is included
+ if not min_methods or any(m in triggered_methods for m in min_methods):
+ # Calculate average confidence of triggered methods
+ confidence = sum(similarities[m] for m in triggered_methods) / len(triggered_methods)
+ return True, confidence, triggered_methods
+
+ return False, 0, []
+
+ elif mode == 'weighted_average':
+ # Calculate weighted average
+ weights = config.get('weights', {})
+ total_weight = sum(weights.get(m, 1.0) for m in similarities)
+ weighted_sum = sum(similarities[m] * weights.get(m, 1.0) for m in similarities)
+ weighted_avg = weighted_sum / total_weight if total_weight > 0 else 0
+
+ # Check if weighted average exceeds average threshold
+ avg_threshold = sum(thresholds.values()) / len(thresholds) if thresholds else 0.85
+
+ if weighted_avg >= avg_threshold:
+ # Find which methods contributed most
+ triggered = [m for m, sim in similarities.items()
+ if sim >= thresholds.get(m, 0.85)]
+ return True, weighted_avg, triggered
+
+ return False, 0, []
+
+ return False, 0, []
+
+ def _get_chapter_score(self, similarities, config):
+ """Calculate overall score for a chapter comparison"""
+ if config['detection_mode'] == 'weighted_average':
+ weights = config.get('weights', {})
+ total_weight = sum(weights.get(m, 1.0) for m in similarities)
+ return sum(similarities.get(m, 0) * weights.get(m, 1.0) for m in similarities) / total_weight if total_weight > 0 else 0
+ else:
+ return max(similarities.values()) if similarities else 0
+
+ def _extract_text_features(self, text):
+ """Extract multiple features from text for AI Hunter analysis"""
+ features = {
+ 'semantic': {},
+ 'structural': {},
+ 'characters': [],
+ 'patterns': {}
+ }
+
+ # Semantic fingerprint
+ lines = text.split('\n')
+
+ # Character extraction (names that appear 3+ times)
+ words = re.findall(r'\b[A-Z][a-z]+\b', text)
+ word_freq = Counter(words)
+ features['characters'] = [name for name, count in word_freq.items()
+ if count >= 3 and name not in {
+ 'The', 'A', 'An', 'In', 'On', 'At', 'To',
+ 'From', 'With', 'By', 'For', 'Of', 'As',
+ 'But', 'And', 'Or', 'He', 'She', 'It',
+ 'They', 'We', 'You', 'What', 'When', 'Where',
+ 'Who', 'Why', 'How', 'That', 'This', 'These'
+ }]
+
+ # Dialogue patterns
+ dialogue_patterns = re.findall(r'"([^"]+)"', text)
+ features['semantic']['dialogue_count'] = len(dialogue_patterns)
+ features['semantic']['dialogue_lengths'] = [len(d) for d in dialogue_patterns[:10]]
+
+ # Speaker patterns
+ speaker_patterns = re.findall(r'(\w+)\s+(?:said|asked|replied|shouted|whispered)', text.lower())
+ features['semantic']['speakers'] = list(set(speaker_patterns[:20]))
+
+ # Number extraction
+ numbers = re.findall(r'\b\d+\b', text)
+ features['patterns']['numbers'] = numbers[:20]
+
+ # Structural signature
+ para_lengths = []
+ dialogue_count = 0
+ for para in text.split('\n\n'):
+ if para.strip():
+ para_lengths.append(len(para))
+ if '"' in para:
+ dialogue_count += 1
+
+ features['structural']['para_count'] = len(para_lengths)
+ features['structural']['avg_para_length'] = sum(para_lengths) / max(1, len(para_lengths))
+ features['structural']['dialogue_ratio'] = dialogue_count / max(1, len(para_lengths))
+
+ # Create structural pattern string
+ pattern = []
+ for para in text.split('\n\n')[:20]: # First 20 paragraphs
+ if para.strip():
+ if '"' in para:
+ pattern.append('D') # Dialogue
+ elif len(para) > 300:
+ pattern.append('L') # Long
+ elif len(para) < 100:
+ pattern.append('S') # Short
+ else:
+ pattern.append('M') # Medium
+ features['structural']['pattern'] = ''.join(pattern)
+
+ # Action density
+ action_verbs = len(re.findall(r'\b\w+ed\b', text))
+ features['semantic']['action_density'] = action_verbs / max(1, len(text.split()))
+
+ # Text length
+ features['semantic']['text_length'] = len(text)
+
+ return features
+
+ def _calculate_exact_similarity(self, text1, text2):
+ """Calculate exact text similarity"""
+ return SequenceMatcher(None, text1, text2).ratio()
+
+ def _calculate_smart_similarity(self, text1, text2, sample_size):
+ """Smart similarity with configurable sample size"""
+ if len(text1) > sample_size * 3 and len(text2) > sample_size * 3:
+ # Use multiple samples
+ samples1 = [
+ text1[:sample_size],
+ text1[len(text1)//2 - sample_size//2:len(text1)//2 + sample_size//2],
+ text1[-sample_size:]
+ ]
+ samples2 = [
+ text2[:sample_size],
+ text2[len(text2)//2 - sample_size//2:len(text2)//2 + sample_size//2],
+ text2[-sample_size:]
+ ]
+ similarities = [SequenceMatcher(None, s1, s2).ratio()
+ for s1, s2 in zip(samples1, samples2)]
+ return sum(similarities) / len(similarities)
+ else:
+ # Use full text up to sample size
+ return SequenceMatcher(None, text1[:sample_size], text2[:sample_size]).ratio()
+
+ def _calculate_semantic_similarity(self, sem1, sem2):
+ """Calculate semantic fingerprint similarity"""
+ score = 0.0
+ weights = 0.0
+
+ # Compare dialogue counts
+ if 'dialogue_count' in sem1 and 'dialogue_count' in sem2:
+ weights += 0.3
+ if sem1['dialogue_count'] > 0 or sem2['dialogue_count'] > 0:
+ ratio = min(sem1['dialogue_count'], sem2['dialogue_count']) / \
+ max(1, max(sem1['dialogue_count'], sem2['dialogue_count']))
+ score += ratio * 0.3
+
+ # Compare speakers
+ if 'speakers' in sem1 and 'speakers' in sem2:
+ weights += 0.4
+ if sem1['speakers'] and sem2['speakers']:
+ overlap = len(set(sem1['speakers']) & set(sem2['speakers']))
+ total = len(set(sem1['speakers']) | set(sem2['speakers']))
+ score += (overlap / max(1, total)) * 0.4
+ elif not sem1['speakers'] and not sem2['speakers']:
+ score += 0.4 # Both have no speakers
+
+ # Compare dialogue lengths pattern
+ if 'dialogue_lengths' in sem1 and 'dialogue_lengths' in sem2:
+ weights += 0.2
+ if sem1['dialogue_lengths'] and sem2['dialogue_lengths']:
+ len1 = sem1['dialogue_lengths'][:10]
+ len2 = sem2['dialogue_lengths'][:10]
+ if len1 and len2:
+ avg1 = sum(len1) / len(len1)
+ avg2 = sum(len2) / len(len2)
+ ratio = min(avg1, avg2) / max(1, max(avg1, avg2))
+ score += ratio * 0.2
+ elif not sem1['dialogue_lengths'] and not sem2['dialogue_lengths']:
+ score += 0.2 # Both have no dialogue
+
+ # Action density
+ if 'action_density' in sem1 and 'action_density' in sem2:
+ weights += 0.1
+ act_sim = 1 - abs(sem1['action_density'] - sem2['action_density'])
+ score += act_sim * 0.1
+
+ return score / max(0.1, weights)
+
+ def _calculate_structural_similarity(self, struct1, struct2):
+ """Calculate structural signature similarity"""
+ score = 0.0
+
+ # Compare paragraph patterns
+ if 'pattern' in struct1 and 'pattern' in struct2:
+ pattern_sim = SequenceMatcher(None, struct1['pattern'], struct2['pattern']).ratio()
+ score += pattern_sim * 0.5
+
+ # Compare paragraph statistics
+ if all(k in struct1 for k in ['para_count', 'avg_para_length', 'dialogue_ratio']) and \
+ all(k in struct2 for k in ['para_count', 'avg_para_length', 'dialogue_ratio']):
+
+ # Paragraph count ratio
+ para_ratio = min(struct1['para_count'], struct2['para_count']) / \
+ max(1, max(struct1['para_count'], struct2['para_count']))
+ score += para_ratio * 0.2
+
+ # Average length ratio
+ avg_ratio = min(struct1['avg_para_length'], struct2['avg_para_length']) / \
+ max(1, max(struct1['avg_para_length'], struct2['avg_para_length']))
+ score += avg_ratio * 0.15
+
+ # Dialogue ratio similarity
+ dialogue_diff = abs(struct1['dialogue_ratio'] - struct2['dialogue_ratio'])
+ score += (1 - min(1, dialogue_diff)) * 0.15
+
+ return score
+
+ def _calculate_character_similarity(self, chars1, chars2):
+ """Calculate character overlap similarity"""
+ if not chars1 or not chars2:
+ return 0.0
+
+ # Convert to sets
+ set1 = set(chars1)
+ set2 = set(chars2)
+
+ # If no overlap at all, return 0
+ intersection = set1 & set2
+ if not intersection:
+ return 0.0
+
+ # Calculate Jaccard index (intersection over union)
+ union = set1 | set2
+ jaccard = len(intersection) / len(union)
+
+ # Also consider the proportion of matching characters relative to each set
+ # This prevents small overlaps from scoring too high
+ overlap1 = len(intersection) / len(set1)
+ overlap2 = len(intersection) / len(set2)
+
+ # Take the minimum overlap to be more conservative
+ min_overlap = min(overlap1, overlap2)
+
+ # Combine jaccard and overlap scores
+ # Jaccard penalizes when sets are very different sizes
+ # Min overlap ensures both texts share a significant portion of characters
+ score = (jaccard + min_overlap) / 2
+
+ return score
+
+ def _calculate_pattern_similarity(self, pat1, pat2):
+ """Calculate pattern similarity (numbers, etc.)"""
+ score = 0.0
+
+ # Number overlap
+ if 'numbers' in pat1 and 'numbers' in pat2:
+ nums1 = set(pat1['numbers'])
+ nums2 = set(pat2['numbers'])
+
+ if nums1 or nums2:
+ overlap = len(nums1 & nums2)
+ total = len(nums1 | nums2)
+ score = overlap / max(1, total)
+ else:
+ score = 1.0 # Both have no numbers
+
+ return score
+
+ def _check_non_target_language(self, text, lang_config):
+ """Check if text contains too much non-target language"""
+ target_language = lang_config['target_language'].lower()
+ threshold = lang_config['threshold_characters']
+
+ # Character ranges for different languages
+ language_ranges = {
+ 'english': [ # Latin script + basic symbols
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ (0x2000, 0x206F), # General Punctuation
+ (0x20A0, 0x20CF), # Currency Symbols
+ (0xFF00, 0xFFEF), # Halfwidth and Fullwidth Forms
+ ],
+ 'japanese': [
+ (0x3040, 0x309F), # Hiragana
+ (0x30A0, 0x30FF), # Katakana
+ (0x4E00, 0x9FAF), # CJK Unified Ideographs
+ (0x3400, 0x4DBF), # CJK Extension A
+ (0xFF66, 0xFF9F), # Halfwidth Katakana
+ ],
+ 'korean': [
+ (0xAC00, 0xD7AF), # Hangul Syllables
+ (0x1100, 0x11FF), # Hangul Jamo
+ (0x3130, 0x318F), # Hangul Compatibility Jamo
+ (0xA960, 0xA97F), # Hangul Jamo Extended-A
+ (0xD7B0, 0xD7FF), # Hangul Jamo Extended-B
+ ],
+ 'chinese': [
+ (0x4E00, 0x9FAF), # CJK Unified Ideographs
+ (0x3400, 0x4DBF), # CJK Extension A
+ (0x20000, 0x2A6DF), # CJK Extension B
+ (0x2A700, 0x2B73F), # CJK Extension C
+ (0x2B740, 0x2B81F), # CJK Extension D
+ (0x3000, 0x303F), # CJK Symbols and Punctuation
+ ],
+ 'arabic': [
+ (0x0600, 0x06FF), # Arabic
+ (0x0750, 0x077F), # Arabic Supplement
+ (0x08A0, 0x08FF), # Arabic Extended-A
+ (0xFB50, 0xFDFF), # Arabic Presentation Forms-A
+ (0xFE70, 0xFEFF), # Arabic Presentation Forms-B
+ ],
+ 'russian': [
+ (0x0400, 0x04FF), # Cyrillic
+ (0x0500, 0x052F), # Cyrillic Supplement
+ (0x2DE0, 0x2DFF), # Cyrillic Extended-A
+ (0xA640, 0xA69F), # Cyrillic Extended-B
+ ],
+ 'thai': [
+ (0x0E00, 0x0E7F), # Thai
+ ],
+ 'hindi': [
+ (0x0900, 0x097F), # Devanagari
+ (0xA8E0, 0xA8FF), # Devanagari Extended
+ ],
+ 'spanish': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'french': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'german': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'portuguese': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'italian': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'dutch': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'vietnamese': [
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ (0x1EA0, 0x1EFF), # Latin Extended Additional (Vietnamese)
+ ],
+ 'turkish': [
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'polish': [
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'swedish': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'danish': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'norwegian': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'finnish': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ }
+
+ # Get target language ranges
+ target_ranges = language_ranges.get(target_language, language_ranges['english'])
+
+ # Count characters that are NOT in target language ranges
+ non_target_count = 0
+ total_letters = 0
+
+ for char in text:
+ # Skip whitespace, punctuation, and numbers for counting
+ if char.isspace() or char.isdigit():
+ continue
+
+ # Count as letter character
+ total_letters += 1
+
+ # Check if character is in any target language range
+ char_code = ord(char)
+ is_target_char = any(start <= char_code <= end for start, end in target_ranges)
+
+ if not is_target_char:
+ non_target_count += 1
+
+ # Debug logging
+ if non_target_count > 0:
+ print(f" 🌐 Language detection: {non_target_count}/{total_letters} non-target chars ({target_language})")
+
+ # Return True if non-target character count exceeds threshold
+ return non_target_count >= threshold, non_target_count
diff --git a/api_key_encryption.py b/api_key_encryption.py
new file mode 100644
index 0000000000000000000000000000000000000000..42560e2d917bfbf4b890d1cc400b871f58bbbca8
--- /dev/null
+++ b/api_key_encryption.py
@@ -0,0 +1,244 @@
+"""
+Simple API Key Encryption Module for Glossarion
+Encrypts only specific API key fields including multi-key support
+"""
+
+import os
+import json
+import base64
+from cryptography.fernet import Fernet
+from pathlib import Path
+
+
+class APIKeyEncryption:
+ """Simple encryption handler for API keys"""
+
+ def __init__(self):
+ self.key_file = Path('.glossarion_key')
+ self.cipher = self._get_or_create_cipher()
+
+ # Define which fields to encrypt
+ self.api_key_fields = [
+ 'api_key',
+ 'replicate_api_key',
+ # Add more field names here if needed
+ ]
+
+ def _get_or_create_cipher(self):
+ """Get existing cipher or create new one"""
+ if self.key_file.exists():
+ try:
+ key = self.key_file.read_bytes()
+ return Fernet(key)
+ except:
+ pass
+
+ # Generate new key
+ key = Fernet.generate_key()
+ self.key_file.write_bytes(key)
+
+ # Hide file on Windows
+ if os.name == 'nt':
+ import ctypes
+ ctypes.windll.kernel32.SetFileAttributesW(str(self.key_file), 2)
+ else:
+ # Restrict permissions on Unix
+ os.chmod(self.key_file, 0o600)
+
+ return Fernet(key)
+
+ def encrypt_value(self, value):
+ """Encrypt a single value"""
+ try:
+ encrypted = self.cipher.encrypt(value.encode())
+ return f"ENC:{base64.b64encode(encrypted).decode()}"
+ except:
+ return value
+
+ def decrypt_value(self, value):
+ """Decrypt a single value"""
+ if not isinstance(value, str) or not value.startswith('ENC:'):
+ return value
+
+ try:
+ encrypted_data = base64.b64decode(value[4:])
+ return self.cipher.decrypt(encrypted_data).decode()
+ except:
+ return value
+
+ def encrypt_multi_keys(self, multi_keys):
+ """Encrypt API keys in multi_api_keys array"""
+ if not isinstance(multi_keys, list):
+ return multi_keys
+
+ encrypted_keys = []
+ for key_entry in multi_keys:
+ if isinstance(key_entry, dict):
+ encrypted_entry = key_entry.copy()
+ # Encrypt the api_key field in each entry
+ if 'api_key' in encrypted_entry and encrypted_entry['api_key']:
+ value = encrypted_entry['api_key']
+ if isinstance(value, str) and not value.startswith('ENC:'):
+ encrypted_entry['api_key'] = self.encrypt_value(value)
+ encrypted_keys.append(encrypted_entry)
+ else:
+ encrypted_keys.append(key_entry)
+
+ return encrypted_keys
+
+ def decrypt_multi_keys(self, multi_keys):
+ """Decrypt API keys in multi_api_keys array"""
+ if not isinstance(multi_keys, list):
+ return multi_keys
+
+ decrypted_keys = []
+ for key_entry in multi_keys:
+ if isinstance(key_entry, dict):
+ decrypted_entry = key_entry.copy()
+ # Decrypt the api_key field in each entry
+ if 'api_key' in decrypted_entry and decrypted_entry['api_key']:
+ decrypted_entry['api_key'] = self.decrypt_value(decrypted_entry['api_key'])
+ decrypted_keys.append(decrypted_entry)
+ else:
+ decrypted_keys.append(key_entry)
+
+ return decrypted_keys
+
+ def encrypt_config(self, config):
+ """Encrypt specific API key fields including multi-key support"""
+ encrypted = config.copy()
+
+ # Encrypt regular API key fields
+ for field in self.api_key_fields:
+ if field in encrypted and encrypted[field]:
+ value = encrypted[field]
+ # Only encrypt if not already encrypted
+ if isinstance(value, str) and not value.startswith('ENC:'):
+ encrypted[field] = self.encrypt_value(value)
+
+ # Encrypt multi_api_keys if present
+ if 'multi_api_keys' in encrypted:
+ encrypted['multi_api_keys'] = self.encrypt_multi_keys(encrypted['multi_api_keys'])
+
+ # Encrypt fallback_keys if present
+ if 'fallback_keys' in encrypted:
+ encrypted['fallback_keys'] = self.encrypt_multi_keys(encrypted['fallback_keys'])
+
+ return encrypted
+
+ def decrypt_config(self, config):
+ """Decrypt specific API key fields including multi-key support"""
+ decrypted = config.copy()
+
+ # Decrypt regular API key fields
+ for field in self.api_key_fields:
+ if field in decrypted and decrypted[field]:
+ decrypted[field] = self.decrypt_value(decrypted[field])
+
+ # Decrypt multi_api_keys if present
+ if 'multi_api_keys' in decrypted:
+ decrypted['multi_api_keys'] = self.decrypt_multi_keys(decrypted['multi_api_keys'])
+
+ # Decrypt fallback_keys if present
+ if 'fallback_keys' in decrypted:
+ decrypted['fallback_keys'] = self.decrypt_multi_keys(decrypted['fallback_keys'])
+
+ return decrypted
+
+# Simple interface functions
+_handler = None
+
+def get_handler():
+ global _handler
+ if _handler is None:
+ _handler = APIKeyEncryption()
+ return _handler
+
+def encrypt_config(config):
+ """Encrypt API keys in config"""
+ return get_handler().encrypt_config(config)
+
+def decrypt_config(config):
+ """Decrypt API keys in config"""
+ return get_handler().decrypt_config(config)
+
+def migrate_config_file(config_file='config.json'):
+ """Migrate existing config to encrypted format"""
+ try:
+ # Read config
+ with open(config_file, 'r', encoding='utf-8') as f:
+ config = json.load(f)
+
+ # Check if already encrypted
+ handler = get_handler()
+ needs_encryption = False
+
+ # Check regular API key fields
+ for field in handler.api_key_fields:
+ if field in config and config[field]:
+ if isinstance(config[field], str) and not config[field].startswith('ENC:'):
+ needs_encryption = True
+ break
+
+ # Check multi_api_keys
+ if 'multi_api_keys' in config and isinstance(config['multi_api_keys'], list):
+ for key_entry in config['multi_api_keys']:
+ if isinstance(key_entry, dict) and 'api_key' in key_entry:
+ if key_entry['api_key'] and not key_entry['api_key'].startswith('ENC:'):
+ needs_encryption = True
+ break
+
+ # Check fallback_keys
+ if 'fallback_keys' in config and isinstance(config['fallback_keys'], list):
+ for key_entry in config['fallback_keys']:
+ if isinstance(key_entry, dict) and 'api_key' in key_entry:
+ if key_entry['api_key'] and not key_entry['api_key'].startswith('ENC:'):
+ needs_encryption = True
+ break
+
+ if not needs_encryption:
+ print("Config already encrypted or no API keys found.")
+ return True
+
+ # Backup
+ backup_file = f"{config_file}.backup"
+ with open(backup_file, 'w', encoding='utf-8') as f:
+ json.dump(config, f, ensure_ascii=False, indent=2)
+ print(f"Created backup: {backup_file}")
+
+ # Encrypt
+ encrypted = encrypt_config(config)
+
+ # Save
+ with open(config_file, 'w', encoding='utf-8') as f:
+ json.dump(encrypted, f, ensure_ascii=False, indent=2)
+
+ print("✅ Successfully encrypted API keys!")
+
+ # Show summary
+ if 'multi_api_keys' in config:
+ print(f" - Encrypted {len(config['multi_api_keys'])} multi-key entries")
+
+ if 'fallback_keys' in config:
+ print(f" - Encrypted {len(config['fallback_keys'])} fallback-key entries")
+
+ return True
+
+ except Exception as e:
+ print(f"❌ Error: {e}")
+ return False
+
+
+if __name__ == "__main__":
+ # Simple migration script
+ import sys
+
+ config_file = 'config.json'
+ if len(sys.argv) > 1:
+ config_file = sys.argv[1]
+
+ if os.path.exists(config_file):
+ print(f"Encrypting API keys in {config_file}...")
+ migrate_config_file(config_file)
+ else:
+ print(f"Config file not found: {config_file}")
diff --git a/async_api_processor.py b/async_api_processor.py
new file mode 100644
index 0000000000000000000000000000000000000000..6da743c76e0025765e266118518a3c8f74de3bf2
--- /dev/null
+++ b/async_api_processor.py
@@ -0,0 +1,3431 @@
+# async_api_processor.py
+"""
+Asynchronous API Processing for Glossarion
+Implements batch API processing with 50% discount from supported providers.
+This is SEPARATE from the existing batch processing (parallel API calls).
+
+Supported Providers with Async/Batch APIs (50% discount):
+- Gemini (Batch API)
+- Anthropic (Message Batches API)
+- OpenAI (Batch API)
+- Mistral (Batch API)
+- Amazon Bedrock (Batch Inference)
+- Groq (Batch API)
+
+Providers without Async APIs:
+- DeepSeek (no batch API)
+- Cohere (only batch embeddings, not completions)
+"""
+
+import os
+import sys
+import re
+from bs4 import BeautifulSoup
+import ebooklib
+from ebooklib import epub
+import json
+import time
+import threading
+import logging
+import hashlib
+import traceback
+from datetime import datetime, timedelta
+from typing import Dict, List, Optional, Tuple, Any
+import tkinter as tk
+from tkinter import ttk, messagebox
+import ttkbootstrap as tb
+from dataclasses import dataclass, asdict
+from enum import Enum
+import requests
+import uuid
+from pathlib import Path
+
+try:
+ import tiktoken
+except ImportError:
+ tiktoken = None
+
+# For TXT file processing
+try:
+ from txt_processor import TextFileProcessor
+except ImportError:
+ TextFileProcessor = None
+ print("txt_processor not available - TXT file support disabled")
+# For provider-specific implementations
+try:
+ import google.generativeai as genai
+ HAS_GEMINI = True
+except ImportError:
+ HAS_GEMINI = False
+
+try:
+ import anthropic
+ HAS_ANTHROPIC = True
+except ImportError:
+ HAS_ANTHROPIC = False
+
+try:
+ import openai
+ HAS_OPENAI = True
+except ImportError:
+ HAS_OPENAI = False
+
+logger = logging.getLogger(__name__)
+
+class AsyncAPIStatus(Enum):
+ """Status states for async API jobs"""
+ PENDING = "pending"
+ PROCESSING = "processing"
+ COMPLETED = "completed"
+ FAILED = "failed"
+ CANCELLED = "cancelled"
+ EXPIRED = "expired"
+
+@dataclass
+class AsyncJobInfo:
+ """Information about an async API job"""
+ job_id: str
+ provider: str
+ model: str
+ status: AsyncAPIStatus
+ created_at: datetime
+ updated_at: datetime
+ total_requests: int
+ completed_requests: int = 0
+ failed_requests: int = 0
+ cost_estimate: float = 0.0
+ input_file: Optional[str] = None
+ output_file: Optional[str] = None
+ error_message: Optional[str] = None
+ metadata: Dict[str, Any] = None
+
+ def to_dict(self) -> Dict[str, Any]:
+ """Convert to dictionary for JSON serialization"""
+ data = asdict(self)
+ data['status'] = self.status.value
+ data['created_at'] = self.created_at.isoformat()
+ data['updated_at'] = self.updated_at.isoformat()
+ return data
+
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> 'AsyncJobInfo':
+ """Create from dictionary"""
+ data['status'] = AsyncAPIStatus(data['status'])
+ data['created_at'] = datetime.fromisoformat(data['created_at'])
+ data['updated_at'] = datetime.fromisoformat(data['updated_at'])
+ if data.get('metadata') is None:
+ data['metadata'] = {}
+ return cls(**data)
+
+class AsyncAPIProcessor:
+ """Handles asynchronous batch API processing for supported providers"""
+
+ # Provider configurations
+ PROVIDER_CONFIGS = {
+ 'gemini': {
+ 'batch_endpoint': 'native_sdk', # Uses native SDK instead of REST
+ 'status_endpoint': 'native_sdk',
+ 'max_requests_per_batch': 10000,
+ 'supports_chunking': False,
+ 'discount': 0.5,
+ 'available': True # Now available!
+ },
+ 'anthropic': {
+ 'batch_endpoint': 'https://api.anthropic.com/v1/messages/batches',
+ 'status_endpoint': 'https://api.anthropic.com/v1/messages/batches/{job_id}',
+ 'max_requests_per_batch': 10000,
+ 'supports_chunking': False,
+ 'discount': 0.5
+ },
+ 'openai': {
+ 'batch_endpoint': 'https://api.openai.com/v1/batches',
+ 'status_endpoint': 'https://api.openai.com/v1/batches/{job_id}',
+ 'cancel_endpoint': 'https://api.openai.com/v1/batches/{job_id}/cancel',
+ 'max_requests_per_batch': 50000,
+ 'supports_chunking': False,
+ 'discount': 0.5
+ },
+ 'mistral': {
+ 'batch_endpoint': 'https://api.mistral.ai/v1/batch/jobs',
+ 'status_endpoint': 'https://api.mistral.ai/v1/batch/jobs/{job_id}',
+ 'max_requests_per_batch': 10000,
+ 'supports_chunking': False,
+ 'discount': 0.5
+ },
+ 'bedrock': {
+ 'batch_endpoint': 'batch-inference', # AWS SDK specific
+ 'max_requests_per_batch': 10000,
+ 'supports_chunking': False,
+ 'discount': 0.5
+ },
+ 'groq': {
+ 'batch_endpoint': 'https://api.groq.com/openai/v1/batch',
+ 'status_endpoint': 'https://api.groq.com/openai/v1/batch/{job_id}',
+ 'max_requests_per_batch': 1000,
+ 'supports_chunking': False,
+ 'discount': 0.5
+ }
+ }
+
+ def __init__(self, gui_instance):
+ """Initialize the async processor
+
+ Args:
+ gui_instance: Reference to TranslatorGUI instance
+ """
+ self.gui = gui_instance
+ self.jobs_file = os.path.join(os.path.dirname(__file__), 'async_jobs.json')
+ self.jobs: Dict[str, AsyncJobInfo] = {}
+ self.stop_flag = threading.Event()
+ self.processing_thread = None
+ self._load_jobs()
+
+ def _load_jobs(self):
+ """Load saved async jobs from file"""
+ try:
+ if os.path.exists(self.jobs_file):
+ with open(self.jobs_file, 'r', encoding='utf-8') as f:
+ data = json.load(f)
+ for job_id, job_data in data.items():
+ try:
+ self.jobs[job_id] = AsyncJobInfo.from_dict(job_data)
+ except Exception as e:
+ print(f"Failed to load job {job_id}: {e}")
+ except Exception as e:
+ print(f"Failed to load async jobs: {e}")
+
+ def _save_jobs(self):
+ """Save async jobs to file"""
+ try:
+ data = {job_id: job.to_dict() for job_id, job in self.jobs.items()}
+ with open(self.jobs_file, 'w', encoding='utf-8') as f:
+ json.dump(data, f, indent=2)
+ except Exception as e:
+ print(f"Failed to save async jobs: {e}")
+
+ def get_provider_from_model(self, model: str) -> Optional[str]:
+ """Determine provider from model name"""
+ model_lower = model.lower()
+
+ # Check prefixes
+ if model_lower.startswith(('gpt', 'o1', 'o3', 'o4')):
+ return 'openai'
+ elif model_lower.startswith('gemini'):
+ return 'gemini'
+ elif model_lower.startswith(('claude', 'sonnet', 'opus', 'haiku')):
+ return 'anthropic'
+ elif model_lower.startswith(('mistral', 'mixtral', 'codestral')):
+ return 'mistral'
+ elif model_lower.startswith('groq'):
+ return 'groq'
+ elif model_lower.startswith('bedrock'):
+ return 'bedrock'
+
+ # Check for aggregator prefixes that might support async
+ if model_lower.startswith(('eh/', 'electronhub/', 'electron/')):
+ # Extract actual model after prefix
+ actual_model = model.split('/', 1)[1] if '/' in model else model
+ return self.get_provider_from_model(actual_model)
+
+ return None
+
+ def supports_async(self, model: str) -> bool:
+ """Check if model supports async processing"""
+ provider = self.get_provider_from_model(model)
+ return provider in self.PROVIDER_CONFIGS
+
+ def estimate_cost(self, num_chapters: int, avg_tokens_per_chapter: int, model: str, compression_factor: float = 1.0) -> Tuple[float, float]:
+ """Estimate costs for async vs regular processing
+
+ Returns:
+ Tuple of (async_cost, regular_cost)
+ """
+ provider = self.get_provider_from_model(model)
+ if not provider:
+ return (0.0, 0.0)
+
+ # UPDATED PRICING AS OF JULY 2025
+ # Prices are (input_price, output_price) per 1M tokens
+ token_prices = {
+ 'openai': {
+ # GPT-4.1 Series (Latest - June 2024 knowledge)
+ 'gpt-4.1': (2.0, 8.0),
+ 'gpt-4.1-mini': (0.4, 1.6),
+ 'gpt-4.1-nano': (0.1, 0.4),
+
+ # GPT-4.5 Preview
+ 'gpt-4.5-preview': (75.0, 150.0),
+
+ # GPT-4o Series
+ 'gpt-4o': (2.5, 10.0),
+ 'gpt-4o-mini': (0.15, 0.6),
+ 'gpt-4o-audio': (2.5, 10.0),
+ 'gpt-4o-audio-preview': (2.5, 10.0),
+ 'gpt-4o-realtime': (5.0, 20.0),
+ 'gpt-4o-realtime-preview': (5.0, 20.0),
+ 'gpt-4o-mini-audio': (0.15, 0.6),
+ 'gpt-4o-mini-audio-preview': (0.15, 0.6),
+ 'gpt-4o-mini-realtime': (0.6, 2.4),
+ 'gpt-4o-mini-realtime-preview': (0.6, 2.4),
+
+ # GPT-4 Legacy
+ 'gpt-4': (30.0, 60.0),
+ 'gpt-4-turbo': (10.0, 30.0),
+ 'gpt-4-32k': (60.0, 120.0),
+ 'gpt-4-0613': (30.0, 60.0),
+ 'gpt-4-0314': (30.0, 60.0),
+
+ # GPT-3.5
+ 'gpt-3.5-turbo': (0.5, 1.5),
+ 'gpt-3.5-turbo-instruct': (1.5, 2.0),
+ 'gpt-3.5-turbo-16k': (3.0, 4.0),
+ 'gpt-3.5-turbo-0125': (0.5, 1.5),
+
+ # O-series Reasoning Models (NOT batch compatible usually)
+ 'o1': (15.0, 60.0),
+ 'o1-pro': (150.0, 600.0),
+ 'o1-mini': (1.1, 4.4),
+ 'o3': (1.0, 4.0),
+ 'o3-pro': (20.0, 80.0),
+ 'o3-deep-research': (10.0, 40.0),
+ 'o3-mini': (1.1, 4.4),
+ 'o4-mini': (1.1, 4.4),
+ 'o4-mini-deep-research': (2.0, 8.0),
+
+ # Special models
+ 'chatgpt-4o-latest': (5.0, 15.0),
+ 'computer-use-preview': (3.0, 12.0),
+ 'gpt-4o-search-preview': (2.5, 10.0),
+ 'gpt-4o-mini-search-preview': (0.15, 0.6),
+ 'codex-mini-latest': (1.5, 6.0),
+
+ # Small models
+ 'davinci-002': (2.0, 2.0),
+ 'babbage-002': (0.4, 0.4),
+
+ 'default': (2.5, 10.0)
+ },
+ 'anthropic': {
+ # Claude 4 Series (Latest)
+ 'claude-4-opus': (3.0, 15.0),
+ 'claude-opus-4': (3.0, 15.0),
+ 'claude-4-sonnet': (3.0, 15.0),
+ 'claude-sonnet-4': (3.0, 15.0),
+
+ # Claude 3.5 Series
+ 'claude-3.5-sonnet': (3.0, 15.0),
+ 'claude-3.5-opus': (15.0, 75.0),
+ 'claude-3.5-haiku': (0.25, 1.25),
+
+ # Claude 3 Series
+ 'claude-3-opus': (15.0, 75.0),
+ 'claude-3-sonnet': (3.0, 15.0),
+ 'claude-3-haiku': (0.25, 1.25),
+
+ # Legacy
+ 'claude-2.1': (8.0, 24.0),
+ 'claude-2': (8.0, 24.0),
+ 'claude-instant': (0.8, 2.4),
+
+ 'default': (3.0, 15.0)
+ },
+ 'gemini': {
+ # Gemini 2.5 Series (Latest)
+ 'gemini-2.5-pro': (1.25, 10.0), # ≤200k tokens
+ 'gemini-2.5-flash': (0.3, 2.5),
+ 'gemini-2.5-flash-lite': (0.1, 0.4),
+ 'gemini-2.5-flash-lite-preview': (0.1, 0.4),
+ 'gemini-2.5-flash-lite-preview-06-17': (0.1, 0.4),
+ 'gemini-2.5-flash-native-audio': (0.5, 12.0), # Audio output
+ 'gemini-2.5-flash-preview-native-audio-dialog': (0.5, 12.0),
+ 'gemini-2.5-flash-exp-native-audio-thinking-dialog': (0.5, 12.0),
+ 'gemini-2.5-flash-preview-tts': (0.5, 10.0),
+ 'gemini-2.5-pro-preview-tts': (1.0, 20.0),
+
+ # Gemini 2.0 Series
+ 'gemini-2.0-flash': (0.1, 0.4),
+ 'gemini-2.0-flash-lite': (0.075, 0.3),
+ 'gemini-2.0-flash-live': (0.35, 1.5),
+ 'gemini-2.0-flash-live-001': (0.35, 1.5),
+ 'gemini-live-2.5-flash-preview': (0.35, 1.5),
+
+ # Gemini 1.5 Series
+ 'gemini-1.5-flash': (0.075, 0.3), # ≤128k tokens
+ 'gemini-1.5-flash-8b': (0.0375, 0.15),
+ 'gemini-1.5-pro': (1.25, 5.0),
+
+ # Legacy/Deprecated
+ 'gemini-1.0-pro': (0.5, 1.5),
+ 'gemini-pro': (0.5, 1.5),
+
+ # Experimental
+ 'gemini-exp': (1.25, 5.0),
+
+ 'default': (0.3, 2.5)
+ },
+ 'mistral': {
+ 'mistral-large': (3.0, 9.0),
+ 'mistral-large-2': (3.0, 9.0),
+ 'mistral-medium': (0.4, 2.0),
+ 'mistral-medium-3': (0.4, 2.0),
+ 'mistral-small': (1.0, 3.0),
+ 'mistral-small-v24.09': (1.0, 3.0),
+ 'mistral-nemo': (0.3, 0.3),
+ 'mixtral-8x7b': (0.24, 0.24),
+ 'mixtral-8x22b': (1.0, 3.0),
+ 'codestral': (0.1, 0.3),
+ 'ministral': (0.1, 0.3),
+ 'default': (0.4, 2.0)
+ },
+ 'groq': {
+ 'llama-4-scout': (0.11, 0.34), # Official pricing
+ 'llama-4-maverick': (0.5, 0.77), # Official pricing
+ 'llama-3.1-405b': (2.5, 2.5),
+ 'llama-3.1-70b': (0.59, 0.79),
+ 'llama-3.1-8b': (0.05, 0.1),
+ 'llama-3-70b': (0.59, 0.79),
+ 'llama-3-8b': (0.05, 0.1),
+ 'mixtral-8x7b': (0.24, 0.24),
+ 'gemma-7b': (0.07, 0.07),
+ 'gemma2-9b': (0.1, 0.1),
+ 'default': (0.3, 0.3)
+ },
+ 'deepseek': {
+ 'deepseek-v3': (0.27, 1.09), # Regular price
+ 'deepseek-v3-promo': (0.14, 0.27), # Promo until Feb 8
+ 'deepseek-chat': (0.27, 1.09),
+ 'deepseek-r1': (0.27, 1.09),
+ 'deepseek-reasoner': (0.27, 1.09),
+ 'deepseek-coder': (0.14, 0.14),
+ 'default': (0.27, 1.09)
+ },
+ 'cohere': {
+ 'command-a': (2.5, 10.0),
+ 'command-r-plus': (2.5, 10.0),
+ 'command-r+': (2.5, 10.0),
+ 'command-r': (0.15, 0.6),
+ 'command-r7b': (0.0375, 0.15),
+ 'command': (1.0, 3.0),
+ 'default': (0.5, 2.0)
+ }
+ }
+
+ provider_prices = token_prices.get(provider, {'default': (2.5, 10.0)})
+
+ # Find the right price for this model
+ price_tuple = provider_prices.get('default', (2.5, 10.0))
+ model_lower = model.lower()
+
+ # Try exact match first
+ if model_lower in provider_prices:
+ price_tuple = provider_prices[model_lower]
+ else:
+ # Try prefix matching
+ for model_key, price in provider_prices.items():
+ if model_key == 'default':
+ continue
+ # Remove version numbers for matching
+ model_key_clean = model_key.replace('-', '').replace('.', '')
+ model_lower_clean = model_lower.replace('-', '').replace('.', '')
+
+ if (model_lower.startswith(model_key) or
+ model_lower_clean.startswith(model_key_clean) or
+ model_key in model_lower):
+ price_tuple = price
+ break
+
+ # Calculate weighted average price based on compression_factor
+ input_price, output_price = price_tuple
+ input_ratio = 1 / (1 + compression_factor)
+ output_ratio = compression_factor / (1 + compression_factor)
+ price_per_million = (input_ratio * input_price) + (output_ratio * output_price)
+
+ # Calculate total tokens
+ # For translation: output is typically 1.2-1.5x input length
+ output_multiplier = compression_factor # Conservative estimate
+ total_tokens_per_chapter = avg_tokens_per_chapter * (1 + output_multiplier)
+ total_tokens = num_chapters * total_tokens_per_chapter
+
+ # Convert to cost
+ regular_cost = (total_tokens / 1_000_000) * price_per_million
+
+ # Batch API discount (50% off)
+ discount = self.PROVIDER_CONFIGS.get(provider, {}).get('discount', 0.5)
+ async_cost = regular_cost * discount
+
+ # Log for debugging
+ logger.info(f"Cost calculation for {model}:")
+ logger.info(f" Provider: {provider}")
+ logger.info(f" Input price: ${input_price:.4f}/1M tokens")
+ logger.info(f" Output price: ${output_price:.4f}/1M tokens")
+ logger.info(f" Compression factor: {compression_factor}")
+ logger.info(f" Weighted avg price: ${price_per_million:.4f}/1M tokens")
+ logger.info(f" Chapters: {num_chapters}")
+ logger.info(f" Avg input tokens/chapter: {avg_tokens_per_chapter:,}")
+ logger.info(f" Total tokens (input+output): {total_tokens:,}")
+ logger.info(f" Regular cost: ${regular_cost:.4f}")
+ logger.info(f" Async cost (50% off): ${async_cost:.4f}")
+
+ return (async_cost, regular_cost)
+
+ def prepare_batch_request(self, chapters: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+ """Prepare batch request for provider
+
+ Args:
+ chapters: List of chapter data with prompts
+ model: Model name
+
+ Returns:
+ Provider-specific batch request format
+ """
+ provider = self.get_provider_from_model(model)
+
+ if provider == 'openai':
+ return self._prepare_openai_batch(chapters, model)
+ elif provider == 'anthropic':
+ return self._prepare_anthropic_batch(chapters, model)
+ elif provider == 'gemini':
+ return self._prepare_gemini_batch(chapters, model)
+ elif provider == 'mistral':
+ return self._prepare_mistral_batch(chapters, model)
+ elif provider == 'groq':
+ return self._prepare_groq_batch(chapters, model)
+ else:
+ raise ValueError(f"Unsupported provider for async: {provider}")
+
+ def _prepare_openai_batch(self, chapters: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+ """Prepare OpenAI batch format"""
+
+ # CRITICAL: Map to exact supported model names
+ supported_batch_models = {
+ # Current models (as of July 2025)
+ 'gpt-4o': 'gpt-4o',
+ 'gpt-4o-mini': 'gpt-4o-mini',
+ 'gpt-4-turbo': 'gpt-4-turbo',
+ 'gpt-4-turbo-preview': 'gpt-4-turbo',
+ 'gpt-3.5-turbo': 'gpt-3.5-turbo',
+ 'gpt-3.5': 'gpt-3.5-turbo',
+
+ # New GPT-4.1 models (if available in your region)
+ 'gpt-4.1': 'gpt-4.1',
+ 'gpt-4.1-mini': 'gpt-4.1-mini',
+ 'gpt-4o-nano': 'gpt-4o-nano',
+
+ # Legacy models (may still work)
+ 'gpt-4': 'gpt-4',
+ 'gpt-4-0613': 'gpt-4-0613',
+ 'gpt-4-0314': 'gpt-4-0314',
+ }
+
+ # Check if model is supported
+ model_lower = model.lower()
+ actual_model = None
+
+ for key, value in supported_batch_models.items():
+ if model_lower == key.lower() or model_lower.startswith(key.lower()):
+ actual_model = value
+ break
+
+ if not actual_model:
+ print(f"Model '{model}' is not supported for batch processing!")
+ print(f"Supported models: {list(supported_batch_models.values())}")
+ raise ValueError(f"Model '{model}' is not supported for OpenAI Batch API")
+
+ logger.info(f"Using batch-supported model: '{actual_model}' (from '{model}')")
+
+ requests = []
+
+ for chapter in chapters:
+ # Validate messages
+ messages = chapter.get('messages', [])
+ if not messages:
+ print(f"Chapter {chapter['id']} has no messages!")
+ continue
+
+ # Ensure all messages have required fields
+ valid_messages = []
+ for msg in messages:
+ if not msg.get('role') or not msg.get('content'):
+ print(f"Skipping invalid message: {msg}")
+ continue
+
+ # Ensure content is string and not empty
+ content = str(msg['content']).strip()
+ if not content:
+ print(f"Skipping message with empty content")
+ continue
+
+ valid_messages.append({
+ 'role': msg['role'],
+ 'content': content
+ })
+
+ if not valid_messages:
+ print(f"No valid messages for chapter {chapter['id']}")
+ continue
+
+ request = {
+ "custom_id": chapter['id'],
+ "method": "POST",
+ "url": "/v1/chat/completions",
+ "body": {
+ "model": actual_model,
+ "messages": valid_messages,
+ "temperature": float(chapter.get('temperature', 0.3)),
+ "max_tokens": int(chapter.get('max_tokens', 8192))
+ }
+ }
+ # LOG THE FIRST REQUEST COMPLETELY
+ if len(requests) == 0:
+ print(f"=== FIRST REQUEST ===")
+ print(json.dumps(request, indent=2))
+ print(f"=== END FIRST REQUEST ===")
+
+ requests.append(request)
+
+ return {"requests": requests}
+
+ def _prepare_anthropic_batch(self, chapters: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+ """Prepare Anthropic batch format"""
+ requests = []
+
+ for chapter in chapters:
+ # Extract system message if present
+ system = None
+ messages = []
+
+ for msg in chapter['messages']:
+ if msg['role'] == 'system':
+ system = msg['content']
+ else:
+ messages.append(msg)
+
+ request = {
+ "custom_id": chapter['id'],
+ "params": {
+ "model": model,
+ "messages": messages,
+ "max_tokens": chapter.get('max_tokens', 8192),
+ "temperature": chapter.get('temperature', 0.3)
+ }
+ }
+
+ if system:
+ request["params"]["system"] = system
+
+ requests.append(request)
+
+ return {"requests": requests}
+
+ def _prepare_gemini_batch(self, chapters: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+ """Prepare Gemini batch format"""
+ requests = []
+
+ for chapter in chapters:
+ # Format messages for Gemini
+ prompt = self._format_messages_for_gemini(chapter['messages'])
+
+ request = {
+ "custom_id": chapter['id'],
+ "generateContentRequest": {
+ "model": f"models/{model}",
+ "contents": [{"parts": [{"text": prompt}]}],
+ "generationConfig": {
+ "temperature": chapter.get('temperature', 0.3),
+ "maxOutputTokens": chapter.get('max_tokens', 8192)
+ }
+ }
+ }
+
+ # Add safety settings if disabled
+ if os.getenv("DISABLE_GEMINI_SAFETY", "false").lower() == "true":
+ request["generateContentRequest"]["safetySettings"] = [
+ {"category": cat, "threshold": "BLOCK_NONE"}
+ for cat in ["HARM_CATEGORY_HARASSMENT", "HARM_CATEGORY_HATE_SPEECH",
+ "HARM_CATEGORY_SEXUALLY_EXPLICIT", "HARM_CATEGORY_DANGEROUS_CONTENT",
+ "HARM_CATEGORY_CIVIC_INTEGRITY"]
+ ]
+
+ requests.append(request)
+
+ return {"requests": requests}
+
+ def _prepare_mistral_batch(self, chapters: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+ """Prepare Mistral batch format"""
+ requests = []
+
+ for chapter in chapters:
+ request = {
+ "custom_id": chapter['id'],
+ "model": model,
+ "messages": chapter['messages'],
+ "temperature": chapter.get('temperature', 0.3),
+ "max_tokens": chapter.get('max_tokens', 8192)
+ }
+ requests.append(request)
+
+ return {"requests": requests}
+
+ def _prepare_groq_batch(self, chapters: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+ """Prepare Groq batch format (OpenAI-compatible)"""
+ return self._prepare_openai_batch(chapters, model)
+
+ def _format_messages_for_gemini(self, messages: List[Dict[str, str]]) -> str:
+ """Format messages for Gemini prompt"""
+ formatted_parts = []
+
+ for msg in messages:
+ role = msg.get('role', 'user').upper()
+ content = msg['content']
+
+ if role == 'SYSTEM':
+ formatted_parts.append(f"INSTRUCTIONS: {content}")
+ else:
+ formatted_parts.append(f"{role}: {content}")
+
+ return "\n\n".join(formatted_parts)
+
+ async def submit_batch(self, batch_data: Dict[str, Any], model: str, api_key: str) -> AsyncJobInfo:
+ """Submit batch to provider and create job entry"""
+ provider = self.get_provider_from_model(model)
+
+ if provider == 'openai':
+ return await self._submit_openai_batch(batch_data, model, api_key)
+ elif provider == 'anthropic':
+ return await self._submit_anthropic_batch(batch_data, model, api_key)
+ elif provider == 'gemini':
+ return await self._submit_gemini_batch(batch_data, model, api_key)
+ elif provider == 'mistral':
+ return await self._submit_mistral_batch(batch_data, model, api_key)
+ elif provider == 'groq':
+ return await self._submit_groq_batch(batch_data, model, api_key)
+ else:
+ raise ValueError(f"Unsupported provider: {provider}")
+
+ def _submit_openai_batch_sync(self, batch_data, model, api_key):
+ """Submit OpenAI batch synchronously"""
+ try:
+ # Remove aiofiles import - not needed for sync operations
+ import tempfile
+ import json
+
+ # Create temporary file for batch data
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f:
+ # Write each request as JSONL
+ for request in batch_data['requests']:
+ json.dump(request, f)
+ f.write('\n')
+ temp_path = f.name
+
+ try:
+ # Upload file to OpenAI
+ headers = {'Authorization': f'Bearer {api_key}'}
+
+ with open(temp_path, 'rb') as f:
+ files = {'file': ('batch.jsonl', f, 'application/jsonl')}
+ data = {'purpose': 'batch'}
+
+ response = requests.post(
+ 'https://api.openai.com/v1/files',
+ headers=headers,
+ files=files,
+ data=data
+ )
+
+ if response.status_code != 200:
+ raise Exception(f"File upload failed: {response.text}")
+
+ file_id = response.json()['id']
+
+ # Create batch job
+ batch_request = {
+ 'input_file_id': file_id,
+ 'endpoint': '/v1/chat/completions',
+ 'completion_window': '24h'
+ }
+
+ response = requests.post(
+ 'https://api.openai.com/v1/batches',
+ headers={**headers, 'Content-Type': 'application/json'},
+ json=batch_request
+ )
+
+ if response.status_code != 200:
+ raise Exception(f"Batch creation failed: {response.text}")
+
+ batch_info = response.json()
+
+ # Calculate cost estimate
+ total_tokens = sum(r.get('token_count', 15000) for r in batch_data['requests'])
+ async_cost, _ = self.estimate_cost(
+ len(batch_data['requests']),
+ total_tokens // len(batch_data['requests']),
+ model
+ )
+
+ job = AsyncJobInfo(
+ job_id=batch_info['id'],
+ provider='openai',
+ model=model,
+ status=AsyncAPIStatus.PENDING,
+ created_at=datetime.now(),
+ updated_at=datetime.now(),
+ total_requests=len(batch_data['requests']),
+ cost_estimate=async_cost,
+ metadata={'file_id': file_id, 'batch_info': batch_info}
+ )
+
+ return job
+
+ finally:
+ # Clean up temp file
+ if os.path.exists(temp_path):
+ os.unlink(temp_path)
+
+ except Exception as e:
+ print(f"OpenAI batch submission failed: {e}")
+ raise
+
+ def _submit_anthropic_batch_sync(self, batch_data: Dict[str, Any], model: str, api_key: str) -> AsyncJobInfo:
+ """Submit Anthropic batch (synchronous version)"""
+ try:
+ headers = {
+ 'X-API-Key': api_key,
+ 'Content-Type': 'application/json',
+ 'anthropic-version': '2023-06-01',
+ 'anthropic-beta': 'message-batches-2024-09-24'
+ }
+
+ response = requests.post(
+ 'https://api.anthropic.com/v1/messages/batches',
+ headers=headers,
+ json=batch_data
+ )
+
+ if response.status_code != 200:
+ raise Exception(f"Batch creation failed: {response.text}")
+
+ batch_info = response.json()
+
+ job = AsyncJobInfo(
+ job_id=batch_info['id'],
+ provider='anthropic',
+ model=model,
+ status=AsyncAPIStatus.PENDING,
+ created_at=datetime.now(),
+ updated_at=datetime.now(),
+ total_requests=len(batch_data['requests']),
+ metadata={'batch_info': batch_info}
+ )
+
+ return job
+
+ except Exception as e:
+ print(f"Anthropic batch submission failed: {e}")
+ raise
+
+ def check_job_status(self, job_id: str) -> AsyncJobInfo:
+ """Check the status of a batch job"""
+ job = self.jobs.get(job_id)
+ if not job:
+ raise ValueError(f"Job {job_id} not found")
+
+ try:
+ provider = job.provider
+
+ if provider == 'openai':
+ self._check_openai_status(job)
+ elif provider == 'gemini':
+ self._check_gemini_status(job)
+ elif provider == 'anthropic':
+ self._check_anthropic_status(job)
+ else:
+ print(f"Unknown provider: {provider}")
+
+ # Update timestamp
+ job.updated_at = datetime.now()
+ self._save_jobs()
+
+ except Exception as e:
+ print(f"Error checking job status: {e}")
+ job.metadata['last_error'] = str(e)
+
+ return job
+
+ def _check_gemini_status(self, job: AsyncJobInfo):
+ """Check Gemini batch status"""
+ try:
+ # First try the Python SDK approach
+ try:
+ from google import genai
+
+ api_key = self._get_api_key()
+ client = genai.Client(api_key=api_key)
+
+ # Get batch job status
+ batch_job = client.batches.get(name=job.job_id)
+
+ # Log the actual response for debugging
+ logger.info(f"Gemini batch job state: {batch_job.state.name if hasattr(batch_job, 'state') else 'Unknown'}")
+
+ # Map Gemini states to our status
+ state_map = {
+ 'JOB_STATE_PENDING': AsyncAPIStatus.PENDING,
+ 'JOB_STATE_RUNNING': AsyncAPIStatus.PROCESSING,
+ 'JOB_STATE_SUCCEEDED': AsyncAPIStatus.COMPLETED,
+ 'JOB_STATE_FAILED': AsyncAPIStatus.FAILED,
+ 'JOB_STATE_CANCELLED': AsyncAPIStatus.CANCELLED,
+ 'JOB_STATE_CANCELLING': AsyncAPIStatus.PROCESSING
+ }
+
+ job.status = state_map.get(batch_job.state.name, AsyncAPIStatus.PENDING)
+
+ # Update metadata
+ if not job.metadata:
+ job.metadata = {}
+ if 'batch_info' not in job.metadata:
+ job.metadata['batch_info'] = {}
+
+ job.metadata['batch_info']['state'] = batch_job.state.name
+ job.metadata['raw_state'] = batch_job.state.name
+ job.metadata['last_check'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+ # Try to get progress information
+ if hasattr(batch_job, 'completed_count'):
+ job.completed_requests = batch_job.completed_count
+ elif job.status == AsyncAPIStatus.PROCESSING:
+ # If processing but no progress info, show as 1 to indicate it started
+ job.completed_requests = 1
+ elif job.status == AsyncAPIStatus.COMPLETED:
+ # If completed, all requests are done
+ job.completed_requests = job.total_requests
+
+ # If completed, store the result file info
+ if batch_job.state.name == 'JOB_STATE_SUCCEEDED' and hasattr(batch_job, 'dest'):
+ job.output_file = batch_job.dest.file_name if hasattr(batch_job.dest, 'file_name') else None
+
+ except Exception as sdk_error:
+ # Fallback to REST API if SDK fails
+ print(f"Gemini SDK failed, trying REST API: {sdk_error}")
+
+ api_key = self._get_api_key()
+ headers = {'x-goog-api-key': api_key}
+
+ batch_name = job.job_id if job.job_id.startswith('batches/') else f'batches/{job.job_id}'
+
+ response = requests.get(
+ f'https://generativelanguage.googleapis.com/v1beta/{batch_name}',
+ headers=headers
+ )
+
+ if response.status_code == 200:
+ data = response.json()
+
+ # Update job status
+ state = data.get('metadata', {}).get('state', 'JOB_STATE_PENDING')
+
+ # Map states
+ state_map = {
+ 'JOB_STATE_PENDING': AsyncAPIStatus.PENDING,
+ 'JOB_STATE_RUNNING': AsyncAPIStatus.PROCESSING,
+ 'JOB_STATE_SUCCEEDED': AsyncAPIStatus.COMPLETED,
+ 'JOB_STATE_FAILED': AsyncAPIStatus.FAILED,
+ 'JOB_STATE_CANCELLED': AsyncAPIStatus.CANCELLED,
+ }
+
+ job.status = state_map.get(state, AsyncAPIStatus.PENDING)
+
+ # Extract progress from metadata
+ metadata = data.get('metadata', {})
+
+ # Gemini might provide progress info
+ if 'completedRequestCount' in metadata:
+ job.completed_requests = metadata['completedRequestCount']
+ if 'failedRequestCount' in metadata:
+ job.failed_requests = metadata['failedRequestCount']
+ if 'totalRequestCount' in metadata:
+ job.total_requests = metadata['totalRequestCount']
+
+ # Store raw state
+ if not job.metadata:
+ job.metadata = {}
+ job.metadata['raw_state'] = state
+ job.metadata['last_check'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+ # Check if completed
+ if state == 'JOB_STATE_SUCCEEDED' and 'response' in data:
+ job.status = AsyncAPIStatus.COMPLETED
+ if 'responsesFile' in data.get('response', {}):
+ job.output_file = data['response']['responsesFile']
+ else:
+ print(f"Gemini status check failed: {response.status_code} - {response.text}")
+
+ except Exception as e:
+ print(f"Gemini status check failed: {e}")
+ if not job.metadata:
+ job.metadata = {}
+ job.metadata['last_error'] = str(e)
+
+ def _check_openai_status(self, job: AsyncJobInfo):
+ """Check OpenAI batch status"""
+ try:
+ api_key = self._get_api_key()
+ headers = {'Authorization': f'Bearer {api_key}'}
+
+ response = requests.get(
+ f'https://api.openai.com/v1/batches/{job.job_id}',
+ headers=headers
+ )
+
+ if response.status_code != 200:
+ print(f"Status check failed: {response.text}")
+ return
+
+ data = response.json()
+
+ # Log the full response for debugging
+ logger.debug(f"OpenAI batch status response: {json.dumps(data, indent=2)}")
+ # Check for high failure rate while in progress
+ request_counts = data.get('request_counts', {})
+ total = request_counts.get('total', 0)
+ failed = request_counts.get('failed', 0)
+ completed = request_counts.get('completed', 0)
+
+ # Map OpenAI status to our status
+ status_map = {
+ 'validating': AsyncAPIStatus.PENDING,
+ 'in_progress': AsyncAPIStatus.PROCESSING,
+ 'finalizing': AsyncAPIStatus.PROCESSING,
+ 'completed': AsyncAPIStatus.COMPLETED,
+ 'failed': AsyncAPIStatus.FAILED,
+ 'expired': AsyncAPIStatus.EXPIRED,
+ 'cancelled': AsyncAPIStatus.CANCELLED,
+ 'cancelling': AsyncAPIStatus.CANCELLED,
+ }
+
+ job.status = status_map.get(data['status'], AsyncAPIStatus.PENDING)
+
+ # Update progress
+ request_counts = data.get('request_counts', {})
+ job.completed_requests = request_counts.get('completed', 0)
+ job.failed_requests = request_counts.get('failed', 0)
+ job.total_requests = request_counts.get('total', job.total_requests)
+
+ # Store metadata
+ if not job.metadata:
+ job.metadata = {}
+ job.metadata['raw_state'] = data['status']
+ job.metadata['last_check'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+ # Handle completion
+ if data['status'] == 'completed':
+ # Check if all requests failed
+ if job.failed_requests > 0 and job.completed_requests == 0:
+ print(f"OpenAI job completed but all {job.failed_requests} requests failed")
+ job.status = AsyncAPIStatus.FAILED
+ job.metadata['all_failed'] = True
+
+ # Store error file if available
+ if data.get('error_file_id'):
+ job.metadata['error_file_id'] = data['error_file_id']
+ logger.info(f"Error file available: {data['error_file_id']}")
+ else:
+ # Normal completion with some successes
+ if 'output_file_id' in data and data['output_file_id']:
+ job.output_file = data['output_file_id']
+ logger.info(f"OpenAI job completed with output file: {job.output_file}")
+
+ # If there were also failures, note that
+ if job.failed_requests > 0:
+ job.metadata['partial_failure'] = True
+ print(f"Job completed with {job.failed_requests} failed requests out of {job.total_requests}")
+ else:
+ print(f"OpenAI job marked as completed but no output_file_id found: {data}")
+
+ # Always store error file if present
+ if data.get('error_file_id'):
+ job.metadata['error_file_id'] = data['error_file_id']
+
+ except Exception as e:
+ print(f"OpenAI status check failed: {e}")
+ if not job.metadata:
+ job.metadata = {}
+ job.metadata['last_error'] = str(e)
+
+ def _check_anthropic_status(self, job: AsyncJobInfo):
+ """Check Anthropic batch status"""
+ try:
+ api_key = self._get_api_key()
+ headers = {
+ 'X-API-Key': api_key,
+ 'anthropic-version': '2023-06-01',
+ 'anthropic-beta': 'message-batches-2024-09-24'
+ }
+
+ response = requests.get(
+ f'https://api.anthropic.com/v1/messages/batches/{job.job_id}',
+ headers=headers
+ )
+
+ if response.status_code != 200:
+ print(f"Status check failed: {response.text}")
+ return
+
+ data = response.json()
+
+ # Map Anthropic status
+ status_map = {
+ 'created': AsyncAPIStatus.PENDING,
+ 'processing': AsyncAPIStatus.PROCESSING,
+ 'ended': AsyncAPIStatus.COMPLETED,
+ 'failed': AsyncAPIStatus.FAILED,
+ 'expired': AsyncAPIStatus.EXPIRED,
+ 'canceled': AsyncAPIStatus.CANCELLED,
+ }
+
+ job.status = status_map.get(data['processing_status'], AsyncAPIStatus.PENDING)
+
+ # Update progress
+ results_summary = data.get('results_summary', {})
+ job.completed_requests = results_summary.get('succeeded', 0)
+ job.failed_requests = results_summary.get('failed', 0)
+ job.total_requests = results_summary.get('total', job.total_requests)
+
+ # Store metadata
+ if not job.metadata:
+ job.metadata = {}
+ job.metadata['raw_state'] = data['processing_status']
+ job.metadata['last_check'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+ if data.get('results_url'):
+ job.output_file = data['results_url']
+
+ except Exception as e:
+ print(f"Anthropic status check failed: {e}")
+ if not job.metadata:
+ job.metadata = {}
+ job.metadata['last_error'] = str(e)
+
+ def _get_api_key(self) -> str:
+ """Get API key from GUI settings"""
+ if hasattr(self.gui, 'api_key_entry'):
+ return self.gui.api_key_entry.get().strip()
+ elif hasattr(self.gui, 'api_key_var'):
+ return self.gui.api_key_var.get().strip()
+ else:
+ # Fallback to environment variable
+ return os.getenv('API_KEY', '') or os.getenv('GEMINI_API_KEY', '') or os.getenv('GOOGLE_API_KEY', '')
+
+ def retrieve_results(self, job_id: str) -> List[Dict[str, Any]]:
+ """Retrieve results from a completed batch job"""
+ job = self.jobs.get(job_id)
+ if not job:
+ raise ValueError(f"Job {job_id} not found")
+
+ if job.status != AsyncAPIStatus.COMPLETED:
+ raise ValueError(f"Job is not completed. Current status: {job.status.value}")
+
+ # If output file is missing, try to refresh status first
+ if not job.output_file:
+ print(f"No output file for completed job {job_id}, refreshing status...")
+ self.check_job_status(job_id)
+
+ # Re-check after status update
+ if not job.output_file:
+ # Log the job details for debugging
+ print(f"Job details: {json.dumps(job.to_dict(), indent=2)}")
+ raise ValueError(f"No output file available for job {job_id} even after status refresh")
+
+ provider = job.provider
+
+ if provider == 'openai':
+ return self._retrieve_openai_results(job)
+ elif provider == 'gemini':
+ return self._retrieve_gemini_results(job)
+ elif provider == 'anthropic':
+ return self._retrieve_anthropic_results(job)
+ else:
+ raise ValueError(f"Unknown provider: {provider}")
+
+ def _retrieve_gemini_results(self, job: AsyncJobInfo) -> List[Dict[str, Any]]:
+ """Retrieve Gemini batch results"""
+ try:
+ from google import genai
+
+ api_key = self._get_api_key()
+
+ # Create client with API key
+ client = genai.Client(api_key=api_key)
+
+ # Get the batch job
+ batch_job = client.batches.get(name=job.job_id)
+
+ if batch_job.state != 'JOB_STATE_SUCCEEDED':
+ raise ValueError(f"Batch job not completed: {batch_job.state}")
+
+ # Download results
+ if hasattr(batch_job, 'dest') and batch_job.dest:
+ # Extract the file name from the destination object
+ if hasattr(batch_job.dest, 'output_uri'):
+ # For BigQuery or Cloud Storage destinations
+ file_name = batch_job.dest.output_uri
+ elif hasattr(batch_job.dest, 'file_name'):
+ # For file-based destinations
+ file_name = batch_job.dest.file_name
+ else:
+ # Try to get any file reference from the dest object
+ # Log the object to understand its structure
+ logger.info(f"BatchJobDestination object: {batch_job.dest}")
+ logger.info(f"BatchJobDestination attributes: {dir(batch_job.dest)}")
+ raise ValueError(f"Cannot extract file name from destination: {batch_job.dest}")
+
+ # Download the results file
+ results_content_bytes = client.files.download(file=file_name)
+ results_content = results_content_bytes.decode('utf-8')
+
+ results = []
+ # Parse JSONL results
+ for line in results_content.splitlines():
+ if line.strip():
+ result_data = json.loads(line)
+
+ # Extract the response content
+ text_content = ""
+
+ # Handle different response formats
+ if 'response' in result_data:
+ response = result_data['response']
+
+ # Check for different content structures
+ if isinstance(response, dict):
+ if 'candidates' in response and response['candidates']:
+ candidate = response['candidates'][0]
+ if 'content' in candidate and 'parts' in candidate['content']:
+ for part in candidate['content']['parts']:
+ if 'text' in part:
+ text_content += part['text']
+ elif 'text' in candidate:
+ text_content = candidate['text']
+ elif 'text' in response:
+ text_content = response['text']
+ elif 'content' in response:
+ text_content = response['content']
+ elif isinstance(response, str):
+ text_content = response
+
+ results.append({
+ 'custom_id': result_data.get('key', ''),
+ 'content': text_content,
+ 'finish_reason': 'stop'
+ })
+
+ return results
+ else:
+ raise ValueError("No output file available for completed job")
+
+ except ImportError:
+ raise ImportError(
+ "google-genai package not installed. "
+ "Run: pip install google-genai"
+ )
+ except Exception as e:
+ print(f"Failed to retrieve Gemini results: {e}")
+ raise
+
+ def _retrieve_openai_results(self, job: AsyncJobInfo) -> List[Dict[str, Any]]:
+ """Retrieve OpenAI batch results"""
+ if not job.output_file:
+ # Try one more status check
+ self._check_openai_status(job)
+ if not job.output_file:
+ raise ValueError(f"No output file available for OpenAI job {job.job_id}")
+
+ try:
+ api_key = self._get_api_key()
+ headers = {'Authorization': f'Bearer {api_key}'}
+
+ # Download results file
+ response = requests.get(
+ f'https://api.openai.com/v1/files/{job.output_file}/content',
+ headers=headers
+ )
+
+ if response.status_code != 200:
+ raise Exception(f"Failed to download results: {response.status_code} - {response.text}")
+
+ # Parse JSONL results
+ results = []
+ for line in response.text.strip().split('\n'):
+ if line:
+ try:
+ result = json.loads(line)
+ # Extract the actual response content
+ if 'response' in result and 'body' in result['response']:
+ results.append({
+ 'custom_id': result.get('custom_id', ''),
+ 'content': result['response']['body']['choices'][0]['message']['content'],
+ 'finish_reason': result['response']['body']['choices'][0].get('finish_reason', 'stop')
+ })
+ else:
+ print(f"Unexpected result format: {result}")
+ except json.JSONDecodeError as e:
+ print(f"Failed to parse result line: {line} - {e}")
+
+ return results
+
+ except Exception as e:
+ print(f"Failed to retrieve OpenAI results: {e}")
+ print(f"Job details: {json.dumps(job.to_dict(), indent=2)}")
+ raise
+
+ def _retrieve_anthropic_results(self, job: AsyncJobInfo) -> List[Dict[str, Any]]:
+ """Retrieve Anthropic batch results"""
+ if not job.output_file:
+ raise ValueError("No output file available")
+
+ api_key = self._get_api_key()
+ headers = {
+ 'X-API-Key': api_key,
+ 'anthropic-version': '2023-06-01'
+ }
+
+ # Download results
+ response = requests.get(job.output_file, headers=headers)
+
+ if response.status_code != 200:
+ raise Exception(f"Failed to download results: {response.text}")
+
+ # Parse JSONL results
+ results = []
+ for line in response.text.strip().split('\n'):
+ if line:
+ result = json.loads(line)
+ if result['result']['type'] == 'succeeded':
+ message = result['result']['message']
+ results.append({
+ 'custom_id': result['custom_id'],
+ 'content': message['content'][0]['text'],
+ 'finish_reason': message.get('stop_reason', 'stop')
+ })
+
+ return results
+
+
+class AsyncProcessingDialog:
+ """GUI dialog for async processing"""
+
+ def __init__(self, parent, translator_gui):
+ """Initialize dialog
+
+ Args:
+ parent: Parent window
+ translator_gui: Reference to main TranslatorGUI instance
+ """
+ self.parent = parent
+ self.gui = translator_gui
+
+ # Fix for PyInstaller - ensure processor uses correct directory
+ self.processor = AsyncAPIProcessor(translator_gui)
+
+ # If running as exe, update the jobs file path
+ if getattr(sys, 'frozen', False):
+ # Running as compiled exe
+ application_path = os.path.dirname(sys.executable)
+ self.processor.jobs_file = os.path.join(application_path, 'async_jobs.json')
+ # Reload jobs from the correct location
+ self.processor._load_jobs()
+
+ self.selected_job_id = None
+ self.polling_jobs = set() # Track which jobs are being polled
+
+ # Use the correct attribute name 'wm' instead of 'window_manager'
+ self.window_manager = translator_gui.wm # WindowManager is stored as 'wm'
+
+ self._create_dialog()
+ self._refresh_jobs_list()
+
+ def _create_dialog(self):
+ """Create the async processing dialog"""
+ # Create scrollable dialog (stays hidden)
+ self.dialog, scrollable_frame, canvas = self.window_manager.setup_scrollable(
+ self.parent,
+ "Async Batch Processing (50% Discount)",
+ width=0, # Will be auto-sized
+ height=None,
+ max_width_ratio=0.9,
+ max_height_ratio=1.00
+ )
+
+ # Store references
+ self.scrollable_frame = scrollable_frame
+ self.canvas = canvas
+
+ # Main container in scrollable_frame
+ main_frame = ttk.Frame(scrollable_frame)
+ main_frame.pack(fill=tk.BOTH, expand=True, padx=10, pady=10)
+
+ # Top section - Information and controls
+ self._create_info_section(main_frame)
+
+ # Middle section - Configuration
+ self._create_config_section(main_frame)
+
+ # Bottom section - Active jobs
+ self._create_jobs_section(main_frame)
+
+ # Button frame goes in the DIALOG, not scrollable_frame
+ button_frame = ttk.Frame(self.dialog)
+ button_frame.pack(fill=tk.X, padx=10, pady=10)
+ self._create_button_frame(button_frame)
+
+ # Load active jobs
+ self._refresh_jobs_list()
+
+ # Auto-resize and show - THIS is what applies the height ratio!
+ self.window_manager.auto_resize_dialog(
+ self.dialog,
+ canvas,
+ max_width_ratio=0.9,
+ max_height_ratio=0.92 # Can override to any value like 1.43
+ )
+
+ # Handle window close
+ self.dialog.protocol("WM_DELETE_WINDOW",
+ lambda: [self.dialog._cleanup_scrolling(), self.dialog.destroy()])
+
+ self._start_auto_refresh(30)
+
+ def _create_info_section(self, parent):
+ """Create information section"""
+ info_frame = ttk.LabelFrame(parent, text="Async Processing Information", padding=10)
+ info_frame.pack(fill=tk.X, pady=(0, 10))
+
+ # Model and provider info
+ model_frame = ttk.Frame(info_frame)
+ model_frame.pack(fill=tk.X)
+
+ ttk.Label(model_frame, text="Current Model:").pack(side=tk.LEFT, padx=(0, 5))
+ model_name = self.gui.model_var.get() if hasattr(self.gui, 'model_var') else "Not selected"
+ self.model_label = ttk.Label(model_frame, text=model_name, style="info.TLabel")
+ self.model_label.pack(side=tk.LEFT, padx=(0, 20))
+
+ # Check if model supports async
+ provider = self.processor.get_provider_from_model(model_name)
+ if provider and provider in self.processor.PROVIDER_CONFIGS:
+ status_text = f"✓ Supported ({provider.upper()})"
+ status_style = "success.TLabel"
+ else:
+ status_text = "✗ Not supported for async"
+ status_style = "danger.TLabel"
+
+ ttk.Label(model_frame, text=status_text, style=status_style).pack(side=tk.LEFT)
+
+ # Cost estimation
+ cost_frame = ttk.Frame(info_frame)
+ cost_frame.pack(fill=tk.X, pady=(10, 0))
+
+ ttk.Label(cost_frame, text="Cost Estimation:", font=("", 10, "bold")).pack(anchor=tk.W)
+
+ self.cost_info_label = ttk.Label(cost_frame, text="Select chapters to see cost estimate")
+ self.cost_info_label.pack(anchor=tk.W, pady=(5, 0))
+
+ def _create_config_section(self, parent):
+ """Create configuration section"""
+ config_frame = ttk.LabelFrame(parent, text="Async Processing Configuration", padding=10)
+ config_frame.pack(fill=tk.X, pady=(0, 10))
+
+ # Processing options
+ options_frame = ttk.Frame(config_frame)
+ options_frame.pack(fill=tk.X)
+
+ # Wait for completion
+ self.wait_for_completion_var = tk.BooleanVar(value=False)
+ ttk.Checkbutton(
+ options_frame,
+ text="Wait for completion (blocks GUI)",
+ variable=self.wait_for_completion_var
+ ).pack(anchor=tk.W)
+
+ # Poll interval
+ poll_frame = ttk.Frame(options_frame)
+ poll_frame.pack(fill=tk.X, pady=(5, 0))
+
+ ttk.Label(poll_frame, text="Poll interval (seconds):").pack(side=tk.LEFT, padx=(0, 5))
+ self.poll_interval_var = tk.IntVar(value=60)
+ ttk.Spinbox(
+ poll_frame,
+ from_=10,
+ to=600,
+ textvariable=self.poll_interval_var,
+ width=10
+ ).pack(side=tk.LEFT)
+
+ # Chapter selection info
+ chapter_frame = ttk.Frame(config_frame)
+ chapter_frame.pack(fill=tk.X, pady=(10, 0))
+
+ self.chapter_info_label = ttk.Label(
+ chapter_frame,
+ text="Note: Async processing will skip chapters that require chunking",
+ style="warning.TLabel"
+ )
+ self.chapter_info_label.pack(anchor=tk.W)
+
+ def _create_jobs_section(self, parent):
+ """Create active jobs section"""
+ jobs_frame = ttk.LabelFrame(parent, text="Active Async Jobs", padding=10)
+ jobs_frame.pack(fill=tk.BOTH, expand=True, pady=(0, 10))
+
+ # Jobs treeview
+ tree_frame = ttk.Frame(jobs_frame)
+ tree_frame.pack(fill=tk.BOTH, expand=True)
+
+ # Scrollbars
+ vsb = ttk.Scrollbar(tree_frame, orient="vertical")
+ hsb = ttk.Scrollbar(tree_frame, orient="horizontal")
+
+ # Treeview
+ self.jobs_tree = ttk.Treeview(
+ tree_frame,
+ columns=("Provider", "Model", "Status", "Progress", "Created", "Cost"),
+ show="tree headings",
+ yscrollcommand=vsb.set,
+ xscrollcommand=hsb.set
+ )
+
+ vsb.config(command=self.jobs_tree.yview)
+ hsb.config(command=self.jobs_tree.xview)
+
+ # Add a progress bar for the selected job
+ progress_frame = ttk.Frame(jobs_frame)
+ progress_frame.pack(fill=tk.X, pady=(5, 0))
+
+ ttk.Label(progress_frame, text="Selected Job Progress:").pack(side=tk.LEFT, padx=(0, 5))
+
+ self.job_progress_bar = ttk.Progressbar(
+ progress_frame,
+ mode='determinate',
+ style='success.Horizontal.TProgressbar'
+ )
+ self.job_progress_bar.pack(side=tk.LEFT, fill=tk.X, expand=True)
+
+ self.progress_label = ttk.Label(progress_frame, text="0%")
+ self.progress_label.pack(side=tk.LEFT, padx=(5, 0))
+
+ # Configure columns
+ self.jobs_tree.heading("#0", text="Job ID")
+ self.jobs_tree.heading("Provider", text="Provider")
+ self.jobs_tree.heading("Model", text="Model")
+ self.jobs_tree.heading("Status", text="Status")
+ self.jobs_tree.heading("Progress", text="Progress")
+ self.jobs_tree.heading("Created", text="Created")
+ self.jobs_tree.heading("Cost", text="Est. Cost")
+
+ self.jobs_tree.column("#0", width=200)
+ self.jobs_tree.column("Provider", width=100)
+ self.jobs_tree.column("Model", width=150)
+ self.jobs_tree.column("Status", width=100)
+ self.jobs_tree.column("Progress", width=150)
+ self.jobs_tree.column("Created", width=150)
+ self.jobs_tree.column("Cost", width=100)
+
+ # Add right-click menu
+ self.jobs_context_menu = tk.Menu(self.jobs_tree, tearoff=0)
+ self.jobs_context_menu.add_command(label="Check Status", command=self._check_selected_status)
+ self.jobs_context_menu.add_command(label="Retrieve Results", command=self._retrieve_selected_results)
+ self.jobs_context_menu.add_separator()
+ self.jobs_context_menu.add_command(label="Delete", command=self._delete_selected_job)
+
+ # Context menu binding function - use unique name to avoid conflicts
+ def show_jobs_context_menu(event):
+ # Select the item under cursor
+ item = self.jobs_tree.identify_row(event.y)
+ if item:
+ self.jobs_tree.selection_set(item)
+ self._on_job_select(None) # Update selection
+ self.jobs_context_menu.post(event.x_root, event.y_root)
+
+ # Bind right-click
+ self.jobs_tree.bind("", show_jobs_context_menu) # Right-click on Windows/Linux
+ if sys.platform == "darwin":
+ self.jobs_tree.bind("", show_jobs_context_menu) # Right-click on macOS
+
+ # Pack treeview and scrollbars
+ self.jobs_tree.grid(row=0, column=0, sticky="nsew")
+ vsb.grid(row=0, column=1, sticky="ns")
+ hsb.grid(row=1, column=0, sticky="ew")
+
+ tree_frame.grid_rowconfigure(0, weight=1)
+ tree_frame.grid_columnconfigure(0, weight=1)
+
+ # Bind selection
+ self.jobs_tree.bind('<>', self._on_job_select)
+
+ # Job action buttons
+ action_frame = ttk.Frame(jobs_frame)
+ action_frame.pack(fill=tk.X, pady=(10, 0))
+
+ button_width = 15
+
+ ttk.Button(
+ action_frame,
+ text="Check Status",
+ command=self._check_selected_status,
+ style="info.TButton",
+ width=button_width
+ ).pack(side=tk.LEFT, padx=(0, 5))
+
+ ttk.Button(
+ action_frame,
+ text="Retrieve Results",
+ command=self._retrieve_selected_results,
+ style="success.TButton",
+ width=button_width
+ ).pack(side=tk.LEFT, padx=(0, 5))
+
+ ttk.Button(
+ action_frame,
+ text="Cancel Job",
+ command=self._cancel_selected_job,
+ style="warning.TButton",
+ width=button_width
+ ).pack(side=tk.LEFT, padx=(0, 5))
+
+ # delete buttons
+ ttk.Button(
+ action_frame,
+ text="Delete Selected",
+ command=self._delete_selected_job,
+ style="danger.TButton",
+ width=button_width
+ ).pack(side=tk.LEFT, padx=(30, 5)) # Extra padding to separate
+
+ ttk.Button(
+ action_frame,
+ text="Clear Completed",
+ command=self._clear_completed_jobs,
+ style="secondary.TButton",
+ width=button_width
+ ).pack(side=tk.LEFT)
+
+ def _create_button_frame(self, parent):
+ """Create bottom button frame"""
+ button_frame = ttk.Frame(parent)
+ button_frame.pack(fill=tk.X, pady=(20, 0))
+
+ # Start processing button
+ self.start_button = ttk.Button(
+ button_frame,
+ text="Start Async Processing",
+ command=self._start_processing,
+ style="success.TButton"
+ )
+ self.start_button.pack(side=tk.LEFT, padx=(0, 5))
+
+ # Estimate only button
+ ttk.Button(
+ button_frame,
+ text="Estimate Cost Only",
+ command=self._estimate_cost,
+ style="info.TButton"
+ ).pack(side=tk.LEFT, padx=(0, 5))
+
+ # Close button - need to handle cleanup if using WindowManager
+ if hasattr(self.dialog, '_cleanup_scrolling'):
+ ttk.Button(
+ button_frame,
+ text="Close",
+ command=lambda: [self.dialog._cleanup_scrolling(), self.dialog.destroy()]
+ ).pack(side=tk.RIGHT)
+ else:
+ ttk.Button(
+ button_frame,
+ text="Close",
+ command=self.dialog.destroy
+ ).pack(side=tk.RIGHT)
+
+ def _update_selected_job_progress(self, job):
+ """Update progress display for selected job"""
+ if hasattr(self, 'job_progress_bar'):
+ if job.total_requests > 0:
+ progress = int((job.completed_requests / job.total_requests) * 100)
+ self.job_progress_bar['value'] = progress
+
+ # Update progress label if exists
+ if hasattr(self, 'progress_label'):
+ self.progress_label.config(
+ text=f"{progress}% ({job.completed_requests}/{job.total_requests} chapters)"
+ )
+ else:
+ self.job_progress_bar['value'] = 0
+ if hasattr(self, 'progress_label'):
+ self.progress_label.config(text="0% (Waiting)")
+
+ def _refresh_jobs_list(self):
+ """Refresh the jobs list"""
+ # Clear existing items
+ for item in self.jobs_tree.get_children():
+ self.jobs_tree.delete(item)
+
+ # Add jobs
+ for job_id, job in self.processor.jobs.items():
+ # Calculate progress percentage and format progress text
+ if job.total_requests > 0:
+ progress_pct = int((job.completed_requests / job.total_requests) * 100)
+ progress_text = f"{progress_pct}% ({job.completed_requests}/{job.total_requests})"
+ else:
+ progress_pct = 0
+ progress_text = "0% (0/0)"
+
+ # Override progress text for completed/failed/cancelled statuses
+ if job.status == AsyncAPIStatus.COMPLETED:
+ progress_text = "100% (Complete)"
+ elif job.status == AsyncAPIStatus.FAILED:
+ progress_text = f"{progress_pct}% (Failed)"
+ elif job.status == AsyncAPIStatus.CANCELLED:
+ progress_text = f"{progress_pct}% (Cancelled)"
+ elif job.status == AsyncAPIStatus.PENDING:
+ progress_text = "0% (Waiting)"
+
+ created = job.created_at.strftime("%Y-%m-%d %H:%M")
+ cost = f"${job.cost_estimate:.2f}" if job.cost_estimate else "N/A"
+
+ # Determine status style
+ status_text = job.status.value.capitalize()
+
+ # Shorten job ID for display
+ display_id = job_id[:20] + "..." if len(job_id) > 20 else job_id
+
+ self.jobs_tree.insert(
+ "",
+ "end",
+ text=display_id,
+ values=(
+ job.provider.upper(),
+ job.model[:15] + "..." if len(job.model) > 15 else job.model, # Shorten model name
+ status_text,
+ progress_text, # Now shows percentage and counts
+ created,
+ cost
+ ),
+ tags=(job.status.value,)
+ )
+
+ # Configure tags for status colors
+ self.jobs_tree.tag_configure("pending", foreground="#FFA500") # Orange
+ self.jobs_tree.tag_configure("processing", foreground="#007BFF") # Blue
+ self.jobs_tree.tag_configure("completed", foreground="#28A745") # Green
+ self.jobs_tree.tag_configure("failed", foreground="#DC3545") # Red
+ self.jobs_tree.tag_configure("cancelled", foreground="#6C757D") # Gray
+
+ # Update progress bar if a job is selected
+ if hasattr(self, 'selected_job_id') and self.selected_job_id:
+ job = self.processor.jobs.get(self.selected_job_id)
+ if job:
+ self._update_selected_job_progress(job)
+
+ def _on_job_select(self, event):
+ """Handle job selection"""
+ selection = self.jobs_tree.selection()
+ if selection:
+ item = self.jobs_tree.item(selection[0])
+ # Get full job ID from the item
+ job_id_prefix = item['text'].rstrip('...')
+
+ # Find matching job
+ for job_id in self.processor.jobs:
+ if job_id.startswith(job_id_prefix):
+ self.selected_job_id = job_id
+
+ # Update progress display for selected job
+ job = self.processor.jobs.get(job_id)
+ if job:
+ # Update progress bar if it exists
+ if hasattr(self, 'job_progress_bar'):
+ if job.total_requests > 0:
+ progress = int((job.completed_requests / job.total_requests) * 100)
+ self.job_progress_bar['value'] = progress
+ else:
+ self.job_progress_bar['value'] = 0
+
+ # Update progress label if it exists
+ if hasattr(self, 'progress_label'):
+ if job.total_requests > 0:
+ progress = int((job.completed_requests / job.total_requests) * 100)
+ self.progress_label.config(
+ text=f"{progress}% ({job.completed_requests}/{job.total_requests} chapters)"
+ )
+ else:
+ self.progress_label.config(text="0% (Waiting)")
+
+ # Log selection
+ logger.info(f"Selected job: {job_id[:30]}... - Status: {job.status.value}")
+
+ break
+
+ def _check_selected_status(self):
+ """Check status of selected job"""
+ if not self.selected_job_id:
+ messagebox.showwarning("No Selection", "Please select a job to check status")
+ return
+
+ try:
+ job = self.processor.check_job_status(self.selected_job_id)
+ self._refresh_jobs_list()
+
+ # Build detailed status message
+ status_text = f"Job ID: {job.job_id}\n"
+ status_text += f"Provider: {job.provider.upper()}\n"
+ status_text += f"Status: {job.status.value}\n"
+ status_text += f"State: {job.metadata.get('raw_state', 'Unknown')}\n\n"
+
+ # Progress information
+ if job.completed_requests > 0 or job.status == AsyncAPIStatus.PROCESSING:
+ status_text += f"Progress: {job.completed_requests}/{job.total_requests}\n"
+ else:
+ status_text += f"Progress: Waiting to start (0/{job.total_requests})\n"
+
+ status_text += f"Failed: {job.failed_requests}\n\n"
+
+ # Time information
+ status_text += f"Created: {job.created_at.strftime('%Y-%m-%d %H:%M:%S')}\n"
+ status_text += f"Last Updated: {job.updated_at.strftime('%Y-%m-%d %H:%M:%S')}\n"
+
+ if 'last_check' in job.metadata:
+ status_text += f"Last Checked: {job.metadata['last_check']}\n"
+
+ # Show output file if available
+ if job.output_file:
+ status_text += f"\nOutput Ready: {job.output_file}\n"
+
+ messagebox.showinfo("Job Status", status_text)
+
+ except Exception as e:
+ messagebox.showerror("Error", f"Failed to check status: {str(e)}")
+
+ def _start_auto_refresh(self, interval_seconds=30):
+ """Start automatic status refresh"""
+ def refresh():
+ if hasattr(self, 'dialog') and self.dialog.winfo_exists():
+ # Refresh all jobs
+ for job_id in list(self.processor.jobs.keys()):
+ try:
+ job = self.processor.jobs[job_id]
+ if job.status in [AsyncAPIStatus.PENDING, AsyncAPIStatus.PROCESSING]:
+ self.processor.check_job_status(job_id)
+ except:
+ pass
+
+ self._refresh_jobs_list()
+
+ # Schedule next refresh
+ self.dialog.after(interval_seconds * 1000, refresh)
+
+ # Start the refresh cycle
+ refresh()
+
+ def _retrieve_selected_results(self):
+ """Retrieve results from selected job"""
+ if not self.selected_job_id:
+ messagebox.showwarning("No Selection", "Please select a job to retrieve results")
+ return
+
+ # Check job status first
+ job = self.processor.jobs.get(self.selected_job_id)
+ if not job:
+ messagebox.showerror("Error", "Selected job not found")
+ return
+
+ if job.status != AsyncAPIStatus.COMPLETED:
+ messagebox.showwarning(
+ "Job Not Complete",
+ f"This job is currently {job.status.value}.\n"
+ "Only completed jobs can have results retrieved."
+ )
+ return
+
+ try:
+ # Set cursor to busy (with safety check)
+ if hasattr(self, 'dialog') and self.dialog.winfo_exists():
+ self.dialog.config(cursor="wait")
+ self.dialog.update()
+
+ # Retrieve results
+ self._handle_completed_job(self.selected_job_id)
+
+ except Exception as e:
+ self._log(f"❌ Error retrieving results: {e}")
+ messagebox.showerror("Error", f"Failed to retrieve results: {str(e)}")
+ finally:
+ # Reset cursor (with safety check)
+ if hasattr(self, 'dialog') and self.dialog.winfo_exists():
+ try:
+ self.dialog.config(cursor="")
+ except tk.TclError:
+ # Dialog was closed, ignore
+ pass
+
+ def _cancel_selected_job(self):
+ """Cancel selected job"""
+ if not self.selected_job_id:
+ messagebox.showwarning("No Selection", "Please select a job to cancel")
+ return
+
+ job = self.processor.jobs.get(self.selected_job_id)
+ if not job:
+ messagebox.showerror("Error", "Selected job not found")
+ return
+
+ if job.status in [AsyncAPIStatus.COMPLETED, AsyncAPIStatus.FAILED, AsyncAPIStatus.CANCELLED]:
+ messagebox.showwarning(
+ "Cannot Cancel",
+ f"This job is already {job.status.value}"
+ )
+ return
+
+ # Confirm cancellation
+ if not messagebox.askyesno(
+ "Cancel Job",
+ f"Are you sure you want to cancel this job?\n\n"
+ f"Job ID: {job.job_id}\n"
+ f"Status: {job.status.value}"
+ ):
+ return
+
+ try:
+ api_key = self.gui.api_key_entry.get().strip()
+
+ if job.provider == 'openai':
+ headers = {'Authorization': f'Bearer {api_key}'}
+
+ response = requests.post(
+ f'https://api.openai.com/v1/batches/{job.job_id}/cancel',
+ headers=headers
+ )
+
+ if response.status_code == 200:
+ job.status = AsyncAPIStatus.CANCELLED
+ job.updated_at = datetime.now()
+ self.processor._save_jobs()
+ self._refresh_jobs_list()
+ messagebox.showinfo("Job Cancelled", "Job has been cancelled successfully")
+ else:
+ messagebox.showerror("Error", f"Failed to cancel job: {response.text}")
+
+ elif job.provider == 'gemini':
+ # Gemini batch cancellation using REST API
+ headers = {'x-goog-api-key': api_key}
+
+ # Format: batches/123456 -> https://generativelanguage.googleapis.com/v1beta/batches/123456:cancel
+ batch_name = job.job_id if job.job_id.startswith('batches/') else f'batches/{job.job_id}'
+
+ response = requests.post(
+ f'https://generativelanguage.googleapis.com/v1beta/{batch_name}:cancel',
+ headers=headers
+ )
+
+ if response.status_code == 200:
+ job.status = AsyncAPIStatus.CANCELLED
+ job.updated_at = datetime.now()
+ self.processor._save_jobs()
+ self._refresh_jobs_list()
+ messagebox.showinfo("Job Cancelled", "Gemini batch job has been cancelled successfully")
+ else:
+ messagebox.showerror("Error", f"Failed to cancel Gemini job: {response.text}")
+
+ elif job.provider == 'anthropic':
+ # Anthropic doesn't support cancellation via API yet
+ messagebox.showinfo(
+ "Not Supported",
+ "Anthropic doesn't support job cancellation via API.\n"
+ "The job will be marked as cancelled locally only."
+ )
+ job.status = AsyncAPIStatus.CANCELLED
+ job.updated_at = datetime.now()
+ self.processor._save_jobs()
+ self._refresh_jobs_list()
+
+ else:
+ # For other providers, just mark as cancelled locally
+ messagebox.showinfo(
+ "Local Cancellation",
+ f"{job.provider.title()} cancellation not implemented.\n"
+ "The job will be marked as cancelled locally only."
+ )
+ job.status = AsyncAPIStatus.CANCELLED
+ job.updated_at = datetime.now()
+ self.processor._save_jobs()
+ self._refresh_jobs_list()
+
+ except Exception as e:
+ messagebox.showerror("Error", f"Failed to cancel job: {str(e)}")
+
+ def _cancel_openai_job(self, job, api_key):
+ """Cancel OpenAI batch job"""
+ headers = {
+ 'Authorization': f'Bearer {api_key}',
+ 'Content-Type': 'application/json'
+ }
+
+ # OpenAI has a specific cancel endpoint
+ cancel_url = f"https://api.openai.com/v1/batches/{job.job_id}/cancel"
+
+ response = requests.post(cancel_url, headers=headers)
+
+ if response.status_code not in [200, 204]:
+ raise Exception(f"OpenAI cancellation failed: {response.text}")
+
+ logger.info(f"OpenAI job {job.job_id} cancelled successfully")
+
+ def _cancel_anthropic_job(self, job, api_key):
+ """Cancel Anthropic batch job"""
+ headers = {
+ 'X-API-Key': api_key,
+ 'anthropic-version': '2023-06-01',
+ 'anthropic-beta': 'message-batches-2024-09-24'
+ }
+
+ # Anthropic uses DELETE method for cancellation
+ cancel_url = f"https://api.anthropic.com/v1/messages/batches/{job.job_id}"
+
+ response = requests.delete(cancel_url, headers=headers)
+
+ if response.status_code not in [200, 204]:
+ raise Exception(f"Anthropic cancellation failed: {response.text}")
+
+ logger.info(f"Anthropic job {job.job_id} cancelled successfully")
+
+ def _cancel_gemini_job(self, job, api_key):
+ """Cancel Gemini batch job"""
+ try:
+ from google import genai
+
+ # Create client
+ client = genai.Client(api_key=api_key)
+
+ # Try to cancel using the SDK
+ # Note: The SDK might not have a cancel method yet
+ if hasattr(client.batches, 'cancel'):
+ client.batches.cancel(name=job.job_id)
+ logger.info(f"Gemini job {job.job_id} cancelled successfully")
+ else:
+ # If SDK doesn't support cancellation, inform the user
+ raise Exception(
+ "Gemini batch cancellation is not supported yet.\n"
+ "The job will continue to run and complete within 24 hours.\n"
+ "You can check the status later to retrieve results."
+ )
+
+ except AttributeError:
+ # SDK doesn't have cancel method
+ raise Exception(
+ "Gemini batch cancellation is not available in the current SDK.\n"
+ "The job will continue to run and complete within 24 hours."
+ )
+ except Exception as e:
+ # Check if it's a permission error
+ if "403" in str(e) or "PERMISSION_DENIED" in str(e):
+ raise Exception(
+ "Gemini batch jobs cannot be cancelled once submitted.\n"
+ "The job will complete within 24 hours and you can retrieve results then."
+ )
+ else:
+ # Re-raise other errors
+ raise
+
+ def _cancel_mistral_job(self, job, api_key):
+ """Cancel Mistral batch job"""
+ headers = {
+ 'Authorization': f'Bearer {api_key}',
+ 'Content-Type': 'application/json'
+ }
+
+ # Mistral batch cancellation endpoint
+ cancel_url = f"https://api.mistral.ai/v1/batch/jobs/{job.job_id}/cancel"
+
+ response = requests.post(cancel_url, headers=headers)
+
+ if response.status_code not in [200, 204]:
+ raise Exception(f"Mistral cancellation failed: {response.text}")
+
+ logger.info(f"Mistral job {job.job_id} cancelled successfully")
+
+ def _cancel_groq_job(self, job, api_key):
+ """Cancel Groq batch job"""
+ # Groq uses OpenAI-compatible endpoints
+ headers = {
+ 'Authorization': f'Bearer {api_key}',
+ 'Content-Type': 'application/json'
+ }
+
+ cancel_url = f"https://api.groq.com/openai/v1/batch/{job.job_id}/cancel"
+
+ response = requests.post(cancel_url, headers=headers)
+
+ if response.status_code not in [200, 204]:
+ raise Exception(f"Groq cancellation failed: {response.text}")
+
+ logger.info(f"Groq job {job.job_id} cancelled successfully")
+
+ def _estimate_cost(self):
+ """Estimate cost for current file"""
+ # Get current file info
+ if not hasattr(self.gui, 'file_path') or not self.gui.file_path:
+ messagebox.showwarning("No File", "Please select a file first")
+ return
+
+ try:
+ # Show analyzing message
+ self.cost_info_label.config(text="Analyzing file...")
+ self.dialog.update()
+
+ file_path = self.gui.file_path
+ model = self.gui.model_var.get()
+
+ # Calculate overhead tokens (system prompt + glossary)
+ overhead_tokens = 0
+
+ # Count system prompt tokens
+ system_prompt = self.gui.prompt_text.get("1.0", "end").strip()
+ if system_prompt:
+ overhead_tokens += self.count_tokens(system_prompt, model)
+ logger.info(f"System prompt tokens: {overhead_tokens}")
+
+ # Count glossary tokens if enabled
+ glossary_tokens = 0
+
+ # Check if glossary should be appended - match the logic from _prepare_environment_variables
+ if (hasattr(self.gui, 'manual_glossary_path') and
+ self.gui.manual_glossary_path and
+ hasattr(self.gui, 'append_glossary_var') and
+ self.gui.append_glossary_var.get()): # This is the key check!
+
+ try:
+ glossary_path = self.gui.manual_glossary_path
+ logger.info(f"Loading glossary from: {glossary_path}")
+
+ if os.path.exists(glossary_path):
+ with open(glossary_path, 'r', encoding='utf-8') as f:
+ glossary_data = json.load(f)
+
+ # Format glossary same way as in translation
+ #glossary_text = self._format_glossary_for_prompt(glossary_data)
+
+ # Add append prompt if available
+ append_prompt = self.gui.append_glossary_prompt if hasattr(self.gui, 'append_glossary_prompt') else ''
+
+ if append_prompt:
+ if '{glossary}' in append_prompt:
+ glossary_text = append_prompt.replace('{glossary}', glossary_text)
+ else:
+ glossary_text = f"{append_prompt}\n{glossary_text}"
+ else:
+ glossary_text = f"Glossary:\n{glossary_text}"
+
+ glossary_tokens = self.count_tokens(glossary_text, model)
+ overhead_tokens += glossary_tokens
+ logger.info(f"Loaded glossary with {glossary_tokens} tokens")
+ else:
+ print(f"Glossary file not found: {glossary_path}")
+
+ except Exception as e:
+ print(f"Failed to load glossary: {e}")
+
+ logger.info(f"Total overhead per chapter: {overhead_tokens} tokens")
+
+ # Actually extract chapters and count tokens
+ num_chapters = 0
+ total_content_tokens = 0 # Just the chapter content
+ chapters_needing_chunking = 0
+
+ if file_path.lower().endswith('.epub'):
+ # Import and use EPUB extraction
+ try:
+ import ebooklib
+ from ebooklib import epub
+ from bs4 import BeautifulSoup
+
+ book = epub.read_epub(file_path)
+ chapters = []
+
+ # Extract text chapters
+ for item in book.get_items():
+ if item.get_type() == ebooklib.ITEM_DOCUMENT:
+ soup = BeautifulSoup(item.get_content(), 'html.parser')
+ text = soup.get_text(separator='\n').strip()
+ if len(text) > 500: # Minimum chapter length
+ chapters.append(text)
+
+ num_chapters = len(chapters)
+
+ # Count tokens for each chapter (sample more for better accuracy)
+ sample_size = min(20, num_chapters) # Sample up to 20 chapters for better accuracy
+ sampled_content_tokens = 0
+
+ for i, chapter_text in enumerate(chapters[:sample_size]):
+ # Count just the content tokens
+ content_tokens = self.count_tokens(chapter_text, model)
+ sampled_content_tokens += content_tokens
+
+ # Check if needs chunking (including overhead)
+ total_chapter_tokens = content_tokens + overhead_tokens
+ token_limit = int(self.gui.token_limit_entry.get() or 200000)
+ if total_chapter_tokens > token_limit * 0.8:
+ chapters_needing_chunking += 1
+
+ # Update progress
+ if i % 5 == 0:
+ self.cost_info_label.config(text=f"Analyzing chapters... {i+1}/{sample_size}")
+ self.dialog.update()
+
+ # Calculate average based on actual sample
+ if sample_size > 0:
+ avg_content_tokens_per_chapter = sampled_content_tokens // sample_size
+ # Extrapolate chunking needs if we didn't sample all
+ if num_chapters > sample_size:
+ chapters_needing_chunking = int(chapters_needing_chunking * (num_chapters / sample_size))
+ else:
+ avg_content_tokens_per_chapter = 15000 # Default
+
+ except Exception as e:
+ print(f"Failed to analyze EPUB: {e}")
+ # Fall back to estimates
+ num_chapters = 50
+ avg_content_tokens_per_chapter = 15000
+
+ elif file_path.lower().endswith('.txt'):
+ # Import and use TXT extraction
+ try:
+ from txt_processor import TextFileProcessor
+
+ processor = TextFileProcessor(file_path, '')
+ chapters = processor.extract_chapters()
+ num_chapters = len(chapters)
+
+ # Count tokens
+ sample_size = min(20, num_chapters) # Sample up to 20 chapters
+ sampled_content_tokens = 0
+
+ for i, chapter_text in enumerate(chapters[:sample_size]):
+ # Count just the content tokens
+ content_tokens = self.count_tokens(chapter_text, model)
+ sampled_content_tokens += content_tokens
+
+ # Check if needs chunking (including overhead)
+ total_chapter_tokens = content_tokens + overhead_tokens
+ token_limit = int(self.gui.token_limit_entry.get() or 200000)
+ if total_chapter_tokens > token_limit * 0.8:
+ chapters_needing_chunking += 1
+
+ # Update progress
+ if i % 5 == 0:
+ self.cost_info_label.config(text=f"Analyzing chapters... {i+1}/{sample_size}")
+ self.dialog.update()
+
+ # Calculate average based on actual sample
+ if sample_size > 0:
+ avg_content_tokens_per_chapter = sampled_content_tokens // sample_size
+ # Extrapolate chunking needs
+ if num_chapters > sample_size:
+ chapters_needing_chunking = int(chapters_needing_chunking * (num_chapters / sample_size))
+ else:
+ avg_content_tokens_per_chapter = 15000 # Default
+
+ except Exception as e:
+ print(f"Failed to analyze TXT: {e}")
+ # Fall back to estimates
+ num_chapters = 50
+ avg_content_tokens_per_chapter = 15000
+ else:
+ # Unsupported format
+ self.cost_info_label.config(
+ text="Unsupported file format. Only EPUB and TXT are supported."
+ )
+ return
+
+ # Calculate costs
+ processable_chapters = num_chapters - chapters_needing_chunking
+
+ if processable_chapters <= 0:
+ self.cost_info_label.config(
+ text=f"Warning: All {num_chapters} chapters require chunking.\n"
+ f"Async APIs do not support chunked chapters.\n"
+ f"Consider using regular batch translation instead."
+ )
+ return
+
+ # Add overhead to get total average tokens per chapter
+ avg_total_tokens_per_chapter = avg_content_tokens_per_chapter + overhead_tokens
+
+ # Get the translation compression factor from GUI
+ compression_factor = float(self.gui.compression_factor_var.get() or 1.0)
+
+ # Get accurate cost estimate
+ async_cost, regular_cost = self.processor.estimate_cost(
+ processable_chapters,
+ avg_total_tokens_per_chapter, # Now includes content + system prompt + glossary
+ model,
+ compression_factor
+ )
+
+ # Update any existing jobs for this file with the accurate estimate
+ current_file = self.gui.file_path
+ for job_id, job in self.processor.jobs.items():
+ # Check if this job is for the current file and model
+ if (job.metadata and
+ job.metadata.get('source_file') == current_file and
+ job.model == model and
+ job.status in [AsyncAPIStatus.PENDING, AsyncAPIStatus.PROCESSING]):
+ # Update the cost estimate
+ job.cost_estimate = async_cost
+ job.updated_at = datetime.now()
+
+ # Save updated jobs
+ self.processor._save_jobs()
+
+ # Refresh the display
+ self._refresh_jobs_list()
+
+ # Build detailed message
+ cost_text = f"File analysis complete!\n\n"
+ cost_text += f"Total chapters: {num_chapters}\n"
+ cost_text += f"Average content tokens per chapter: {avg_content_tokens_per_chapter:,}\n"
+ cost_text += f"Overhead per chapter: {overhead_tokens:,} tokens"
+ if glossary_tokens > 0:
+ cost_text += f" (system: {overhead_tokens - glossary_tokens:,}, glossary: {glossary_tokens:,})"
+ cost_text += f"\nTotal input tokens per chapter: {avg_total_tokens_per_chapter:,}\n"
+
+ if chapters_needing_chunking > 0:
+ cost_text += f"\nChapters requiring chunking: {chapters_needing_chunking} (will be skipped)\n"
+ cost_text += f"Processable chapters: {processable_chapters}\n"
+
+ cost_text += f"\nEstimated cost for {processable_chapters} chapters:\n"
+ cost_text += f"Regular processing: ${regular_cost:.2f}\n"
+ cost_text += f"Async processing: ${async_cost:.2f} (50% savings: ${regular_cost - async_cost:.2f})"
+
+ # Add note about token calculation
+ cost_text += f"\n\nNote: Costs include input (~{avg_total_tokens_per_chapter:,}) and "
+ cost_text += f"output (~{int(avg_content_tokens_per_chapter * compression_factor):,}) tokens per chapter."
+
+
+ self.cost_info_label.config(text=cost_text)
+
+ except Exception as e:
+ self.cost_info_label.config(
+ text=f"Error estimating cost: {str(e)}"
+ )
+ print(f"Cost estimation error: {traceback.format_exc()}")
+
+ def count_tokens(self, text, model):
+ """Count tokens in text (content only - system prompt and glossary are counted separately)"""
+ try:
+ import tiktoken
+
+ # Get base encoding for model
+ if model.startswith(('gpt-4', 'gpt-3')):
+ try:
+ encoding = tiktoken.encoding_for_model(model)
+ except KeyError:
+ encoding = tiktoken.get_encoding("cl100k_base")
+ elif model.startswith('claude'):
+ encoding = tiktoken.get_encoding("cl100k_base")
+ else:
+ encoding = tiktoken.get_encoding("cl100k_base")
+
+ # Just count the text tokens - don't include system/glossary here
+ # They are counted separately in _estimate_cost to avoid confusion
+ text_tokens = len(encoding.encode(text))
+
+ return text_tokens
+
+ except Exception as e:
+ # Fallback: estimate ~4 characters per token
+ return len(text) // 4
+
+ def _start_processing(self):
+ """Start async processing"""
+ model = self.gui.model_var.get()
+
+ if not self.processor.supports_async(model):
+ messagebox.showerror(
+ "Not Supported",
+ f"Model '{model}' does not support async processing.\n"
+ "Supported providers: Gemini, Anthropic, OpenAI, Mistral, Groq"
+ )
+ return
+
+ # Add special check for Gemini
+ if model.lower().startswith('gemini'):
+ response = messagebox.askyesno(
+ "Gemini Batch API",
+ "Note: Gemini's batch API may not be publicly available yet.\n"
+ "This feature is experimental for Gemini models.\n\n"
+ "Would you like to try anyway?"
+ )
+ if not response:
+ return
+
+ if not self.processor.supports_async(model):
+ messagebox.showerror(
+ "Not Supported",
+ f"Model '{model}' does not support async processing.\n"
+ "Supported providers: Gemini, Anthropic, OpenAI, Mistral, Groq"
+ )
+ return
+
+ if not hasattr(self.gui, 'file_path') or not self.gui.file_path:
+ messagebox.showwarning("No File", "Please select a file to translate first")
+ return
+
+ # Confirm start
+ if not messagebox.askyesno(
+ "Start Async Processing",
+ "Start async batch processing?\n\n"
+ "This will submit all chapters for processing at 50% discount.\n"
+ "Processing may take up to 24 hours."
+ ):
+ return
+
+ # Disable buttons during processing
+ self.start_button.config(state='disabled')
+
+ # Start processing in background thread
+ self.processing_thread = threading.Thread(
+ target=self._async_processing_worker,
+ daemon=True
+ )
+ self.processing_thread.start()
+
+ def _async_processing_worker(self):
+ """Worker thread for async processing"""
+ try:
+ self._log("Starting async processing preparation...")
+
+ # Get all settings from GUI
+ file_path = self.gui.file_path
+ model = self.gui.model_var.get()
+ api_key = self.gui.api_key_entry.get().strip()
+
+ if not api_key:
+ self._show_error("API key is required")
+ return
+
+ # Prepare environment variables like the main translation
+ env_vars = self._prepare_environment_variables()
+
+ # Extract chapters
+ self._log("Extracting chapters from file...")
+ chapters, chapter_mapping = self._extract_chapters_for_async(file_path, env_vars) # CHANGED: Now unpacking both values
+
+ if not chapters:
+ self._show_error("No chapters found in file")
+ return
+
+ self._log(f"Found {len(chapters)} chapters to process")
+
+ # Check for chapters that need chunking
+ chapters_to_process = []
+ skipped_count = 0
+
+ for chapter in chapters:
+ if chapter.get('needs_chunking', False):
+ skipped_count += 1
+ self._log(f"Skipping chapter {chapter['number']} - requires chunking")
+ else:
+ chapters_to_process.append(chapter)
+
+ if skipped_count > 0:
+ self._log(f"⚠️ Skipped {skipped_count} chapters that require chunking")
+
+ if not chapters_to_process:
+ self._show_error("All chapters require chunking. Async APIs don't support chunked chapters.")
+ return
+
+ # Prepare batch request
+ self._log("Preparing batch request...")
+ batch_data = self.processor.prepare_batch_request(chapters_to_process, model)
+
+ # Submit batch
+ self._log("Submitting batch to API...")
+ job = self._submit_batch_sync(batch_data, model, api_key)
+
+ # Save job with chapter mapping in metadata
+ job.metadata = job.metadata or {}
+ job.metadata['chapter_mapping'] = chapter_mapping # ADDED: Store mapping for later use
+
+ # Save job
+ self.processor.jobs[job.job_id] = job
+ self.processor._save_jobs()
+
+ # Update UI
+ self.dialog.after(0, self._refresh_jobs_list)
+
+ self._log(f"✅ Batch submitted successfully! Job ID: {job.job_id}")
+
+ # Show success message
+ self._show_info(
+ "Batch Submitted",
+ f"Successfully submitted {len(chapters_to_process)} chapters for async processing.\n\n"
+ f"Job ID: {job.job_id}\n\n"
+ "You can close this dialog and check back later for results.\n\n"
+ "Tip: Use the 'Estimate Cost Only' button to get accurate cost estimates before submitting."
+ )
+
+ # Start polling if requested
+ if self.wait_for_completion_var.get():
+ self._start_polling(job.job_id)
+
+ except Exception as e:
+ self._log(f"❌ Error: {str(e)}")
+ print(f"Async processing error: {traceback.format_exc()}")
+ self._show_error(f"Failed to start async processing: {str(e)}")
+ finally:
+ # Re-enable button
+ self.dialog.after(0, lambda: self.start_button.config(state='normal'))
+
+ def _prepare_environment_variables(self):
+ """Prepare environment variables from GUI settings"""
+ env_vars = {}
+
+ # Core settings
+ env_vars['MODEL'] = self.gui.model_var.get()
+ env_vars['API_KEY'] = self.gui.api_key_entry.get().strip()
+ env_vars['OPENAI_API_KEY'] = env_vars['API_KEY']
+ env_vars['OPENAI_OR_Gemini_API_KEY'] = env_vars['API_KEY']
+ env_vars['GEMINI_API_KEY'] = env_vars['API_KEY']
+ env_vars['PROFILE_NAME'] = self.gui.lang_var.get().lower()
+ env_vars['CONTEXTUAL'] = '1' if self.gui.contextual_var.get() else '0'
+ env_vars['MAX_OUTPUT_TOKENS'] = str(self.gui.max_output_tokens)
+ env_vars['SYSTEM_PROMPT'] = self.gui.prompt_text.get("1.0", "end").strip()
+ env_vars['TRANSLATION_TEMPERATURE'] = str(self.gui.trans_temp.get())
+ env_vars['TRANSLATION_HISTORY_LIMIT'] = str(self.gui.trans_history.get())
+
+ # API settings
+ env_vars['SEND_INTERVAL_SECONDS'] = str(self.gui.delay_entry.get())
+ env_vars['TOKEN_LIMIT'] = self.gui.token_limit_entry.get() if hasattr(self.gui, 'token_limit_entry') else '200000'
+
+ # Book title translation
+ env_vars['TRANSLATE_BOOK_TITLE'] = "1" if self.gui.translate_book_title_var.get() else "0"
+ env_vars['BOOK_TITLE_PROMPT'] = self.gui.book_title_prompt if hasattr(self.gui, 'book_title_prompt') else ''
+ env_vars['BOOK_TITLE_SYSTEM_PROMPT'] = self.gui.config.get('book_title_system_prompt',
+ "You are a translator. Respond with only the translated text, nothing else. Do not add any explanation or additional content.")
+
+ # Processing options
+ env_vars['CHAPTER_RANGE'] = self.gui.chapter_range_entry.get().strip() if hasattr(self.gui, 'chapter_range_entry') else ''
+ env_vars['REMOVE_AI_ARTIFACTS'] = "1" if self.gui.REMOVE_AI_ARTIFACTS_var.get() else "0"
+ env_vars['BATCH_TRANSLATION'] = "1" if self.gui.batch_translation_var.get() else "0"
+ env_vars['BATCH_SIZE'] = self.gui.batch_size_var.get()
+ env_vars['CONSERVATIVE_BATCHING'] = "1" if self.gui.conservative_batching_var.get() else "0"
+
+ # Anti-duplicate parameters
+ env_vars['ENABLE_ANTI_DUPLICATE'] = '1' if hasattr(self.gui, 'enable_anti_duplicate_var') and self.gui.enable_anti_duplicate_var.get() else '0'
+ env_vars['TOP_P'] = str(self.gui.top_p_var.get()) if hasattr(self.gui, 'top_p_var') else '1.0'
+ env_vars['TOP_K'] = str(self.gui.top_k_var.get()) if hasattr(self.gui, 'top_k_var') else '0'
+ env_vars['FREQUENCY_PENALTY'] = str(self.gui.frequency_penalty_var.get()) if hasattr(self.gui, 'frequency_penalty_var') else '0.0'
+ env_vars['PRESENCE_PENALTY'] = str(self.gui.presence_penalty_var.get()) if hasattr(self.gui, 'presence_penalty_var') else '0.0'
+ env_vars['REPETITION_PENALTY'] = str(self.gui.repetition_penalty_var.get()) if hasattr(self.gui, 'repetition_penalty_var') else '1.0'
+ env_vars['CANDIDATE_COUNT'] = str(self.gui.candidate_count_var.get()) if hasattr(self.gui, 'candidate_count_var') else '1'
+ env_vars['CUSTOM_STOP_SEQUENCES'] = self.gui.custom_stop_sequences_var.get() if hasattr(self.gui, 'custom_stop_sequences_var') else ''
+ env_vars['LOGIT_BIAS_ENABLED'] = '1' if hasattr(self.gui, 'logit_bias_enabled_var') and self.gui.logit_bias_enabled_var.get() else '0'
+ env_vars['LOGIT_BIAS_STRENGTH'] = str(self.gui.logit_bias_strength_var.get()) if hasattr(self.gui, 'logit_bias_strength_var') else '-0.5'
+ env_vars['BIAS_COMMON_WORDS'] = '1' if hasattr(self.gui, 'bias_common_words_var') and self.gui.bias_common_words_var.get() else '0'
+ env_vars['BIAS_REPETITIVE_PHRASES'] = '1' if hasattr(self.gui, 'bias_repetitive_phrases_var') and self.gui.bias_repetitive_phrases_var.get() else '0'
+
+ # Glossary settings
+ env_vars['MANUAL_GLOSSARY'] = self.gui.manual_glossary_path if hasattr(self.gui, 'manual_glossary_path') and self.gui.manual_glossary_path else ''
+ env_vars['DISABLE_AUTO_GLOSSARY'] = "0" if self.gui.enable_auto_glossary_var.get() else "1"
+ env_vars['DISABLE_GLOSSARY_TRANSLATION'] = "0" if self.gui.enable_auto_glossary_var.get() else "1"
+ env_vars['APPEND_GLOSSARY'] = "1" if self.gui.append_glossary_var.get() else "0"
+ env_vars['APPEND_GLOSSARY_PROMPT'] = self.gui.append_glossary_prompt if hasattr(self.gui, 'append_glossary_prompt') else ''
+ env_vars['GLOSSARY_MIN_FREQUENCY'] = self.gui.glossary_min_frequency_var.get()
+ env_vars['GLOSSARY_MAX_NAMES'] = self.gui.glossary_max_names_var.get()
+ env_vars['GLOSSARY_MAX_TITLES'] = self.gui.glossary_max_titles_var.get()
+ env_vars['GLOSSARY_BATCH_SIZE'] = self.gui.glossary_batch_size_var.get()
+ env_vars['GLOSSARY_DUPLICATE_KEY_MODE'] = self.gui.config.get('glossary_duplicate_key_mode', 'auto')
+ env_vars['GLOSSARY_DUPLICATE_CUSTOM_FIELD'] = self.gui.config.get('glossary_duplicate_custom_field', '')
+
+ # History and summary settings
+ env_vars['TRANSLATION_HISTORY_ROLLING'] = "1" if self.gui.translation_history_rolling_var.get() else "0"
+ env_vars['USE_ROLLING_SUMMARY'] = "1" if self.gui.config.get('use_rolling_summary') else "0"
+ env_vars['SUMMARY_ROLE'] = self.gui.config.get('summary_role', 'user')
+ env_vars['ROLLING_SUMMARY_EXCHANGES'] = self.gui.rolling_summary_exchanges_var.get()
+ env_vars['ROLLING_SUMMARY_MODE'] = self.gui.rolling_summary_mode_var.get()
+ env_vars['ROLLING_SUMMARY_SYSTEM_PROMPT'] = self.gui.rolling_summary_system_prompt if hasattr(self.gui, 'rolling_summary_system_prompt') else ''
+ env_vars['ROLLING_SUMMARY_USER_PROMPT'] = self.gui.rolling_summary_user_prompt if hasattr(self.gui, 'rolling_summary_user_prompt') else ''
+ env_vars['ROLLING_SUMMARY_MAX_ENTRIES'] = self.gui.rolling_summary_max_entries_var.get() if hasattr(self.gui, 'rolling_summary_max_entries_var') else '10'
+
+ # Retry and error handling settings
+ env_vars['EMERGENCY_PARAGRAPH_RESTORE'] = "1" if self.gui.emergency_restore_var.get() else "0"
+ env_vars['RETRY_TRUNCATED'] = "1" if self.gui.retry_truncated_var.get() else "0"
+ env_vars['MAX_RETRY_TOKENS'] = self.gui.max_retry_tokens_var.get()
+ env_vars['RETRY_DUPLICATE_BODIES'] = "1" if self.gui.retry_duplicate_var.get() else "0"
+ env_vars['RETRY_TIMEOUT'] = "1" if self.gui.retry_timeout_var.get() else "0"
+ env_vars['CHUNK_TIMEOUT'] = self.gui.chunk_timeout_var.get()
+
+ # Image processing
+ env_vars['ENABLE_IMAGE_TRANSLATION'] = "1" if self.gui.enable_image_translation_var.get() else "0"
+ env_vars['PROCESS_WEBNOVEL_IMAGES'] = "1" if self.gui.process_webnovel_images_var.get() else "0"
+ env_vars['WEBNOVEL_MIN_HEIGHT'] = self.gui.webnovel_min_height_var.get()
+ env_vars['MAX_IMAGES_PER_CHAPTER'] = self.gui.max_images_per_chapter_var.get()
+ env_vars['IMAGE_API_DELAY'] = '1.0'
+ env_vars['SAVE_IMAGE_TRANSLATIONS'] = '1'
+ env_vars['IMAGE_CHUNK_HEIGHT'] = self.gui.image_chunk_height_var.get()
+ env_vars['HIDE_IMAGE_TRANSLATION_LABEL'] = "1" if self.gui.hide_image_translation_label_var.get() else "0"
+
+ # Advanced settings
+ env_vars['REINFORCEMENT_FREQUENCY'] = self.gui.reinforcement_freq_var.get()
+ env_vars['RESET_FAILED_CHAPTERS'] = "1" if self.gui.reset_failed_chapters_var.get() else "0"
+ env_vars['DUPLICATE_LOOKBACK_CHAPTERS'] = self.gui.duplicate_lookback_var.get()
+ env_vars['DUPLICATE_DETECTION_MODE'] = self.gui.duplicate_detection_mode_var.get()
+ env_vars['CHAPTER_NUMBER_OFFSET'] = str(self.gui.chapter_number_offset_var.get())
+ env_vars['COMPRESSION_FACTOR'] = self.gui.compression_factor_var.get()
+ extraction_mode = self.gui.extraction_mode_var.get() if hasattr(self.gui, 'extraction_mode_var') else 'smart'
+ env_vars['COMPREHENSIVE_EXTRACTION'] = "1" if extraction_mode in ['comprehensive', 'full'] else "0"
+ env_vars['EXTRACTION_MODE'] = extraction_mode
+ env_vars['DISABLE_ZERO_DETECTION'] = "1" if self.gui.disable_zero_detection_var.get() else "0"
+ env_vars['USE_HEADER_AS_OUTPUT'] = "1" if self.gui.use_header_as_output_var.get() else "0"
+ env_vars['ENABLE_DECIMAL_CHAPTERS'] = "1" if self.gui.enable_decimal_chapters_var.get() else "0"
+ env_vars['ENABLE_WATERMARK_REMOVAL'] = "1" if self.gui.enable_watermark_removal_var.get() else "0"
+ env_vars['ADVANCED_WATERMARK_REMOVAL'] = "1" if self.gui.advanced_watermark_removal_var.get() else "0"
+ env_vars['SAVE_CLEANED_IMAGES'] = "1" if self.gui.save_cleaned_images_var.get() else "0"
+
+ # EPUB specific settings
+ env_vars['DISABLE_EPUB_GALLERY'] = "1" if self.gui.disable_epub_gallery_var.get() else "0"
+ env_vars['FORCE_NCX_ONLY'] = '1' if self.gui.force_ncx_only_var.get() else '0'
+
+ # Special handling for Gemini safety filters
+ env_vars['DISABLE_GEMINI_SAFETY'] = str(self.gui.config.get('disable_gemini_safety', False)).lower()
+
+ # AI Hunter settings (if enabled)
+ if 'ai_hunter_config' in self.gui.config:
+ env_vars['AI_HUNTER_CONFIG'] = json.dumps(self.gui.config['ai_hunter_config'])
+
+ # Output settings
+ env_vars['EPUB_OUTPUT_DIR'] = os.getcwd()
+ output_path = self.gui.output_entry.get().strip() if hasattr(self.gui, 'output_entry') else ''
+ if output_path:
+ env_vars['OUTPUT_DIR'] = output_path
+
+ # File path (needed by some modules)
+ env_vars['EPUB_PATH'] = self.gui.file_path
+
+ return env_vars
+
+ def _extract_chapters_for_async(self, file_path, env_vars):
+ """Extract chapters and prepare them for async processing"""
+ chapters = []
+ original_basename = None
+ chapter_mapping = {} # Map custom_id to chapter info
+
+ try:
+ if file_path.lower().endswith('.epub'):
+ # Use direct ZIP reading to avoid ebooklib's manifest validation
+ import zipfile
+ from bs4 import BeautifulSoup
+
+ raw_chapters = []
+
+ try:
+ with zipfile.ZipFile(file_path, 'r') as zf:
+ # Get all HTML/XHTML files
+ html_files = [f for f in zf.namelist() if f.endswith(('.html', '.xhtml', '.htm')) and not f.startswith('__MACOSX')]
+ html_files.sort() # Sort to maintain order
+
+ for idx, html_file in enumerate(html_files):
+ try:
+ content = zf.read(html_file)
+ soup = BeautifulSoup(content, 'html.parser')
+
+ # Remove all image tags
+ for img in soup.find_all('img'):
+ img.decompose()
+
+ # Remove all link tags that might reference CSS or other files
+ for link in soup.find_all('link'):
+ link.decompose()
+
+ chapter_text = soup.get_text(separator='\n').strip()
+
+ if len(chapter_text) > 500: # Minimum chapter length
+ chapter_num = idx + 1
+
+ # Try to extract chapter number from content
+ for element in soup.find_all(['h1', 'h2', 'h3', 'title']):
+ text = element.get_text().strip()
+ match = re.search(r'chapter\s*(\d+)', text, re.IGNORECASE)
+ if match:
+ chapter_num = int(match.group(1))
+ break
+
+ raw_chapters.append((chapter_num, chapter_text, html_file))
+
+ except Exception as e:
+ print(f"Error reading {html_file}: {e}")
+ continue
+
+ except Exception as e:
+ print(f"Failed to read EPUB as ZIP: {e}")
+ raise ValueError(f"Cannot read EPUB file: {str(e)}")
+
+ elif file_path.lower().endswith('.txt'):
+ # Import TXT processing
+ from txt_processor import TextFileProcessor
+
+ processor = TextFileProcessor(file_path, '')
+ txt_chapters = processor.extract_chapters()
+ raw_chapters = [(i+1, text, f"section_{i+1:04d}.txt") for i, text in enumerate(txt_chapters)]
+
+ else:
+ raise ValueError(f"Unsupported file type: {file_path}")
+
+ if not raw_chapters:
+ raise ValueError("No valid chapters found in file")
+
+ # Process each chapter to prepare for API
+ for idx, (chapter_num, content, original_filename) in enumerate(raw_chapters):
+ # Count tokens
+ token_count = self.count_tokens(content, env_vars['MODEL'])
+
+ # Check if needs chunking
+ token_limit = int(env_vars.get('TOKEN_LIMIT', '200000'))
+ needs_chunking = token_count > token_limit * 0.8 # 80% threshold
+
+ # Prepare messages format
+ messages = self._prepare_chapter_messages(content, env_vars)
+
+ custom_id = f"chapter_{chapter_num}"
+
+ chapter_data = {
+ 'id': custom_id,
+ 'number': chapter_num,
+ 'content': content,
+ 'messages': messages,
+ 'temperature': float(env_vars.get('TRANSLATION_TEMPERATURE', '0.3')),
+ 'max_tokens': int(env_vars['MAX_OUTPUT_TOKENS']),
+ 'needs_chunking': needs_chunking,
+ 'token_count': token_count,
+ 'original_basename': original_filename # Use original_filename instead of undefined original_basename
+ }
+
+ chapters.append(chapter_data)
+
+ # Store mapping
+ chapter_mapping[custom_id] = {
+ 'original_filename': original_filename,
+ 'chapter_num': chapter_num
+ }
+
+ except Exception as e:
+ print(f"Failed to extract chapters: {e}")
+ raise
+
+ # Return both chapters and mapping
+ return chapters, chapter_mapping
+
+ def _delete_selected_job(self):
+ """Delete selected job from the list"""
+ if not self.selected_job_id:
+ messagebox.showwarning("No Selection", "Please select a job to delete")
+ return
+
+ # Get job details for confirmation
+ job = self.processor.jobs.get(self.selected_job_id)
+ if not job:
+ messagebox.showerror("Error", "Selected job not found")
+ return
+
+ # Confirm deletion
+ response = messagebox.askyesno(
+ "Confirm Delete",
+ f"Are you sure you want to delete this job?\n\n"
+ f"Job ID: {job.job_id}\n"
+ f"Status: {job.status.value}\n"
+ f"Created: {job.created_at.strftime('%Y-%m-%d %H:%M:%S')}\n\n"
+ "Note: This only removes the job from your local list.\n"
+ "The job may still be running on the server."
+ )
+
+ if response:
+ # Remove from jobs dictionary
+ del self.processor.jobs[self.selected_job_id]
+
+ # Save updated jobs
+ self.processor._save_jobs()
+
+ # Clear selection
+ self.selected_job_id = None
+
+ # Refresh the display
+ self._refresh_jobs_list()
+
+ messagebox.showinfo("Job Deleted", "Job removed from local list.")
+
+ def _clear_completed_jobs(self):
+ """Clear all completed/failed/cancelled jobs"""
+ # Get list of jobs to remove
+ jobs_to_remove = []
+ for job_id, job in self.processor.jobs.items():
+ if job.status in [AsyncAPIStatus.COMPLETED, AsyncAPIStatus.FAILED,
+ AsyncAPIStatus.CANCELLED, AsyncAPIStatus.EXPIRED]:
+ jobs_to_remove.append(job_id)
+
+ if not jobs_to_remove:
+ messagebox.showinfo("No Jobs to Clear", "No completed/failed/cancelled jobs to clear.")
+ return
+
+ # Confirm
+ response = messagebox.askyesno(
+ "Clear Completed Jobs",
+ f"Remove {len(jobs_to_remove)} completed/failed/cancelled jobs from the list?\n\n"
+ "This will not affect any running jobs."
+ )
+
+ if response:
+ # Remove jobs
+ for job_id in jobs_to_remove:
+ del self.processor.jobs[job_id]
+
+ # Save
+ self.processor._save_jobs()
+
+ # Refresh
+ self._refresh_jobs_list()
+
+ messagebox.showinfo("Jobs Cleared", f"Removed {len(jobs_to_remove)} jobs from the list.")
+
+ def _prepare_chapter_messages(self, content, env_vars):
+ """Prepare messages array for a chapter"""
+ messages = []
+
+ # System prompt
+ system_prompt = env_vars.get('SYSTEM_PROMPT', '')
+
+ # DEBUG: Log what we're sending
+ logger.info(f"Model: {env_vars.get('MODEL')}")
+ logger.info(f"System prompt length: {len(system_prompt)}")
+ logger.info(f"Content length: {len(content)}")
+
+ # Log the system prompt (first 200 chars)
+ logger.info(f"Using system prompt: {system_prompt[:200]}...")
+
+ # Add glossary if enabled
+ if (env_vars.get('MANUAL_GLOSSARY') and
+ env_vars.get('APPEND_GLOSSARY') == '1' and
+ env_vars.get('DISABLE_GLOSSARY_TRANSLATION') != '1'):
+ try:
+ glossary_path = env_vars['MANUAL_GLOSSARY']
+ with open(glossary_path, 'r', encoding='utf-8') as f:
+ glossary_data = json.load(f)
+
+ # TRUE BRUTE FORCE: Just dump the entire JSON
+ glossary_text = json.dumps(glossary_data, ensure_ascii=False, indent=2)
+
+ # Use the append prompt format if provided
+ append_prompt = env_vars.get('APPEND_GLOSSARY_PROMPT', '')
+ if append_prompt:
+ # Replace placeholder with actual glossary
+ if '{glossary}' in append_prompt:
+ glossary_section = append_prompt.replace('{glossary}', glossary_text)
+ else:
+ glossary_section = f"{append_prompt}\n{glossary_text}"
+ system_prompt = f"{system_prompt}\n\n{glossary_section}"
+ else:
+ # Default format
+ system_prompt = f"{system_prompt}\n\nGlossary:\n{glossary_text}"
+
+ logger.info(f"Glossary appended to system prompt ({len(glossary_text)} chars)")
+
+ # Log preview for debugging
+ if len(glossary_text) > 200:
+ logger.info(f"Glossary preview: {glossary_text[:200]}...")
+ else:
+ logger.info(f"Glossary: {glossary_text}")
+
+ except FileNotFoundError:
+ print(f"Glossary file not found: {env_vars.get('MANUAL_GLOSSARY')}")
+ except json.JSONDecodeError:
+ print(f"Invalid JSON in glossary file")
+ except Exception as e:
+ print(f"Failed to load glossary: {e}")
+ else:
+ # Log why glossary wasn't added
+ if not env_vars.get('MANUAL_GLOSSARY'):
+ logger.info("No glossary path specified")
+ elif env_vars.get('APPEND_GLOSSARY') != '1':
+ logger.info("Glossary append is disabled")
+ elif env_vars.get('DISABLE_GLOSSARY_TRANSLATION') == '1':
+ logger.info("Glossary translation is disabled")
+
+ messages.append({
+ 'role': 'system',
+ 'content': system_prompt
+ })
+
+ # Add context if enabled
+ if env_vars.get('CONTEXTUAL') == '1':
+ # This would need to load context from history
+ # For async, we might need to pre-generate context
+ logger.info("Note: Contextual mode enabled but not implemented for async yet")
+
+ # User message with chapter content
+ messages.append({
+ 'role': 'user',
+ 'content': content
+ })
+
+ return messages
+
+ def _submit_batch_sync(self, batch_data, model, api_key):
+ """Submit batch synchronously (wrapper for async method)"""
+ provider = self.processor.get_provider_from_model(model)
+
+ if provider == 'openai':
+ return self.processor._submit_openai_batch_sync(batch_data, model, api_key)
+ elif provider == 'anthropic':
+ return self.processor._submit_anthropic_batch_sync(batch_data, model, api_key)
+ elif provider == 'gemini':
+ return self._submit_gemini_batch_sync(batch_data, model, api_key)
+ elif provider == 'mistral':
+ return self._submit_mistral_batch_sync(batch_data, model, api_key)
+ elif provider == 'groq':
+ return self._submit_groq_batch_sync(batch_data, model, api_key)
+ else:
+ raise ValueError(f"Unsupported provider: {provider}")
+
+ def _submit_gemini_batch_sync(self, batch_data, model, api_key):
+ """Submit Gemini batch using the official Batch Mode API"""
+ try:
+ # Use the new Google Gen AI SDK
+ from google import genai
+ from google.genai import types
+
+ # Configure client with API key
+ client = genai.Client(api_key=api_key)
+
+ # Log for debugging
+ logger.info(f"Submitting Gemini batch with model: {model}")
+ logger.info(f"Number of requests: {len(batch_data['requests'])}")
+
+ # Create JSONL file for batch requests
+ import tempfile
+
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False, encoding='utf-8') as f:
+ for request in batch_data['requests']:
+ # Format for Gemini batch API
+ batch_line = {
+ "key": request['custom_id'],
+ "request": {
+ "contents": request['generateContentRequest']['contents'],
+ "generation_config": request['generateContentRequest'].get('generationConfig', {})
+ }
+ }
+
+ # Add safety settings if present
+ if 'safetySettings' in request['generateContentRequest']:
+ batch_line['request']['safety_settings'] = request['generateContentRequest']['safetySettings']
+
+ f.write(json.dumps(batch_line) + '\n')
+
+ batch_file_path = f.name
+
+ # Upload the batch file with explicit mime type
+ logger.info("Uploading batch file...")
+
+ # Use the upload config to specify mime type
+ upload_config = types.UploadFileConfig(
+ mime_type='application/jsonl', # Explicit JSONL mime type
+ display_name=f"batch_requests_{datetime.now().strftime('%Y%m%d_%H%M%S')}.jsonl"
+ )
+
+ uploaded_file = client.files.upload(
+ file=batch_file_path,
+ config=upload_config
+ )
+
+ logger.info(f"File uploaded: {uploaded_file.name}")
+
+ # Create batch job
+ batch_job = client.batches.create(
+ model=model,
+ src=uploaded_file.name,
+ config={
+ 'display_name': f"glossarion_batch_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+ }
+ )
+
+ logger.info(f"Gemini batch job created: {batch_job.name}")
+
+ # Clean up temp file
+ os.unlink(batch_file_path)
+
+ # Calculate cost estimate
+ total_tokens = sum(r.get('token_count', 15000) for r in batch_data['requests'])
+ async_cost, _ = self.processor.estimate_cost(
+ len(batch_data['requests']),
+ total_tokens // len(batch_data['requests']),
+ model
+ )
+
+ # Create job info
+ job = AsyncJobInfo(
+ job_id=batch_job.name,
+ provider='gemini',
+ model=model,
+ status=AsyncAPIStatus.PENDING,
+ created_at=datetime.now(),
+ updated_at=datetime.now(),
+ total_requests=len(batch_data['requests']),
+ cost_estimate=0.0, # No estimate initially
+ metadata={
+ 'batch_info': {
+ 'name': batch_job.name,
+ 'state': batch_job.state.name if hasattr(batch_job, 'state') else 'PENDING',
+ 'src_file': uploaded_file.name
+ },
+ 'source_file': self.gui.file_path # Add this to track which file this job is for
+ }
+ )
+
+ return job
+
+ except ImportError:
+ print("Google Gen AI SDK not installed. Run: pip install google-genai")
+ raise Exception("Google Gen AI SDK not installed. Please run: pip install google-genai")
+ except Exception as e:
+ print(f"Gemini batch submission failed: {e}")
+ print(f"Full error: {traceback.format_exc()}")
+ raise
+
+ def _submit_mistral_batch_sync(self, batch_data, model, api_key):
+ """Submit Mistral batch (synchronous version)"""
+ try:
+ headers = {
+ 'Authorization': f'Bearer {api_key}',
+ 'Content-Type': 'application/json'
+ }
+
+ response = requests.post(
+ 'https://api.mistral.ai/v1/batch/jobs',
+ headers=headers,
+ json=batch_data
+ )
+
+ if response.status_code != 200:
+ raise Exception(f"Batch creation failed: {response.text}")
+
+ batch_info = response.json()
+
+ # Calculate cost estimate
+ total_tokens = sum(r.get('token_count', 15000) for r in batch_data['requests'])
+ async_cost, _ = self.processor.estimate_cost(
+ len(batch_data['requests']),
+ total_tokens // len(batch_data['requests']),
+ model
+ )
+
+ job = AsyncJobInfo(
+ job_id=batch_info['id'],
+ provider='mistral',
+ model=model,
+ status=AsyncAPIStatus.PENDING,
+ created_at=datetime.now(),
+ updated_at=datetime.now(),
+ total_requests=len(batch_data['requests']),
+ cost_estimate=async_cost,
+ metadata={'batch_info': batch_info}
+ )
+
+ return job
+
+ except Exception as e:
+ print(f"Mistral batch submission failed: {e}")
+ raise
+
+ def _submit_groq_batch_sync(self, batch_data, model, api_key):
+ """Submit Groq batch (synchronous version)"""
+ # Groq uses OpenAI-compatible format
+ return self.processor._submit_openai_batch_sync(batch_data, model, api_key)
+
+ def _start_polling(self, job_id):
+ """Start polling for job completion with progress updates"""
+ def poll():
+ try:
+ job = self.processor.check_job_status(job_id)
+ self._refresh_jobs_list()
+
+ # Update progress message
+ if job.total_requests > 0:
+ progress_pct = int((job.completed_requests / job.total_requests) * 100)
+ self._log(f"Progress: {progress_pct}% ({job.completed_requests}/{job.total_requests} chapters)")
+
+ if job.status == AsyncAPIStatus.COMPLETED:
+ self._log(f"✅ Job {job_id} completed!")
+ self._handle_completed_job(job_id)
+ elif job.status in [AsyncAPIStatus.FAILED, AsyncAPIStatus.CANCELLED]:
+ self._log(f"❌ Job {job_id} {job.status.value}")
+ else:
+ # Continue polling with progress update
+ poll_interval = self.poll_interval_var.get() * 1000
+ self.dialog.after(poll_interval, poll)
+
+ except Exception as e:
+ self._log(f"❌ Polling error: {e}")
+
+ # Start polling
+ poll()
+
+ def _handle_completed_job(self, job_id):
+ """Handle a completed job - retrieve results and save"""
+ try:
+ # Retrieve results
+ results = self.processor.retrieve_results(job_id)
+
+ if not results:
+ self._log("❌ No results retrieved from completed job")
+ return
+
+ # Get output directory - same name as input file, in exe location
+ if getattr(sys, 'frozen', False):
+ # Running as compiled exe - use exe directory
+ app_dir = os.path.dirname(sys.executable)
+ else:
+ # Running as script - use script directory
+ app_dir = os.path.dirname(os.path.abspath(__file__))
+
+ base_name = os.path.splitext(os.path.basename(self.gui.file_path))[0]
+ output_dir = os.path.join(app_dir, base_name)
+
+ # Handle existing directory
+ if os.path.exists(output_dir):
+ response = messagebox.askyesnocancel(
+ "Directory Exists",
+ f"The output directory already exists:\n{output_dir}\n\n"
+ "Yes = Overwrite\n"
+ "No = Create new with number\n"
+ "Cancel = Cancel operation"
+ )
+
+ if response is None:
+ return
+ elif response is False:
+ counter = 1
+ while os.path.exists(f"{output_dir}_{counter}"):
+ counter += 1
+ output_dir = f"{output_dir}_{counter}"
+
+ os.makedirs(output_dir, exist_ok=True)
+
+ # Extract ALL resources from EPUB (CSS, fonts, images)
+ self._log("📦 Extracting EPUB resources...")
+ import zipfile
+
+ with zipfile.ZipFile(self.gui.file_path, 'r') as zf:
+ # Create resource directories
+ for res_type in ['css', 'fonts', 'images']:
+ os.makedirs(os.path.join(output_dir, res_type), exist_ok=True)
+
+ # Extract all resources
+ for file_path in zf.namelist():
+ if file_path.endswith('/'):
+ continue
+
+ file_lower = file_path.lower()
+ file_name = os.path.basename(file_path)
+
+ # Skip empty filenames
+ if not file_name:
+ continue
+
+ # Determine resource type and extract
+ if file_lower.endswith('.css'):
+ zf.extract(file_path, os.path.join(output_dir, 'css'))
+ elif file_lower.endswith(('.ttf', '.otf', '.woff', '.woff2')):
+ zf.extract(file_path, os.path.join(output_dir, 'fonts'))
+ elif file_lower.endswith(('.jpg', '.jpeg', '.png', '.gif', '.svg', '.webp')):
+ zf.extract(file_path, os.path.join(output_dir, 'images'))
+
+ # Extract chapter info and metadata from source EPUB
+ self._log("📋 Extracting metadata from source EPUB...")
+
+ import ebooklib
+ from ebooklib import epub
+ from bs4 import BeautifulSoup
+ from TransateKRtoEN import get_content_hash, should_retain_source_extension
+
+ # Extract metadata
+ metadata = {}
+ book = epub.read_epub(self.gui.file_path)
+
+ # Get book metadata
+ if book.get_metadata('DC', 'title'):
+ metadata['title'] = book.get_metadata('DC', 'title')[0][0]
+ if book.get_metadata('DC', 'creator'):
+ metadata['creator'] = book.get_metadata('DC', 'creator')[0][0]
+ if book.get_metadata('DC', 'language'):
+ metadata['language'] = book.get_metadata('DC', 'language')[0][0]
+
+ # Save metadata.json
+ metadata_path = os.path.join(output_dir, 'metadata.json')
+ with open(metadata_path, 'w', encoding='utf-8') as f:
+ json.dump(metadata, f, ensure_ascii=False, indent=2)
+
+ # Map chapter numbers to original info
+ chapter_map = {}
+ chapters_info = []
+ actual_chapter_num = 0
+
+ for item in book.get_items():
+ if item.get_type() == ebooklib.ITEM_DOCUMENT:
+ original_name = item.get_name()
+ original_basename = os.path.splitext(os.path.basename(original_name))[0]
+
+ soup = BeautifulSoup(item.get_content(), 'html.parser')
+ text = soup.get_text().strip()
+
+ if len(text) > 500: # Valid chapter
+ actual_chapter_num += 1
+
+ # Try to find chapter number in content
+ chapter_num = actual_chapter_num
+ for element in soup.find_all(['h1', 'h2', 'h3', 'title']):
+ element_text = element.get_text().strip()
+ match = re.search(r'chapter\s*(\d+)', element_text, re.IGNORECASE)
+ if match:
+ chapter_num = int(match.group(1))
+ break
+
+ # Calculate real content hash
+ content_hash = get_content_hash(text)
+
+ chapter_map[chapter_num] = {
+ 'original_basename': original_basename,
+ 'original_extension': os.path.splitext(original_name)[1],
+ 'content_hash': content_hash,
+ 'text_length': len(text),
+ 'has_images': bool(soup.find_all('img'))
+ }
+
+ chapters_info.append({
+ 'num': chapter_num,
+ 'title': element_text if 'element_text' in locals() else f"Chapter {chapter_num}",
+ 'original_filename': original_name,
+ 'original_basename': original_basename,
+ 'has_images': bool(soup.find_all('img')),
+ 'text_length': len(text),
+ 'content_hash': content_hash
+ })
+
+ # Save chapters_info.json
+ chapters_info_path = os.path.join(output_dir, 'chapters_info.json')
+ with open(chapters_info_path, 'w', encoding='utf-8') as f:
+ json.dump(chapters_info, f, ensure_ascii=False, indent=2)
+
+ # Create realistic progress tracking
+ progress_data = {
+ "version": "3.0",
+ "chapters": {},
+ "chapter_chunks": {},
+ "content_hashes": {},
+ "created": datetime.now().isoformat(),
+ "last_updated": datetime.now().isoformat(),
+ "total_chapters": len(results),
+ "completed_chapters": len(results),
+ "failed_chapters": 0,
+ "async_translated": True
+ }
+
+ # Sort results and save with proper filenames
+ sorted_results = sorted(results, key=lambda x: self._extract_chapter_number(x['custom_id']))
+
+ self._log("💾 Saving translated chapters...")
+ for result in sorted_results:
+ chapter_num = self._extract_chapter_number(result['custom_id'])
+
+ # Get chapter info
+ chapter_info = chapter_map.get(chapter_num, {})
+ original_basename = chapter_info.get('original_basename', f"{chapter_num:04d}")
+ content_hash = chapter_info.get('content_hash', hashlib.md5(f"chapter_{chapter_num}".encode()).hexdigest())
+
+ # Save file with correct name (only once!)
+ retain_ext = should_retain_source_extension()
+ # Preserve compound extensions like .htm.xhtml when retaining
+ orig_name = chapter_info.get('original_filename') or chapter_info.get('original_basename')
+ if retain_ext and orig_name:
+ # Compute full extension suffix beyond the first dot from the left of the basename
+ full = os.path.basename(orig_name)
+ bn, ext1 = os.path.splitext(full)
+ full_ext = ''
+ while ext1:
+ full_ext = ext1 + full_ext
+ bn, ext1 = os.path.splitext(bn)
+ # If no extension found, default to .html
+ suffix = full_ext if full_ext else '.html'
+ filename = f"{original_basename}{suffix}"
+ elif retain_ext:
+ filename = f"{original_basename}.html"
+ else:
+ filename = f"response_{original_basename}.html"
+ file_path = os.path.join(output_dir, filename)
+
+ with open(file_path, 'w', encoding='utf-8') as f:
+ f.write(result['content'])
+
+ # Add realistic progress entry
+ progress_data["chapters"][content_hash] = {
+ "status": "completed",
+ "output_file": filename,
+ "actual_num": chapter_num,
+ "chapter_num": chapter_num,
+ "content_hash": content_hash,
+ "original_basename": original_basename,
+ "started_at": datetime.now().isoformat(),
+ "completed_at": datetime.now().isoformat(),
+ "translation_time": 2.5, # Fake but realistic
+ "token_count": chapter_info.get('text_length', 5000) // 4, # Rough estimate
+ "model": self.gui.model_var.get(),
+ "from_async": True
+ }
+
+ # Add content hash tracking
+ progress_data["content_hashes"][content_hash] = {
+ "chapter_key": content_hash,
+ "chapter_num": chapter_num,
+ "status": "completed",
+ "index": chapter_num - 1
+ }
+
+ # Save realistic progress file
+ progress_file = os.path.join(output_dir, 'translation_progress.json')
+ with open(progress_file, 'w', encoding='utf-8') as f:
+ json.dump(progress_data, f, indent=2)
+
+ self._log(f"✅ Saved {len(sorted_results)} chapters to: {output_dir}")
+
+ messagebox.showinfo(
+ "Async Translation Complete",
+ f"Successfully saved {len(sorted_results)} translated chapters to:\n{output_dir}\n\n"
+ "Ready for EPUB conversion or further processing."
+ )
+
+ except Exception as e:
+ self._log(f"❌ Error handling completed job: {e}")
+ import traceback
+ self._log(traceback.format_exc())
+ messagebox.showerror("Error", f"Failed to process results: {str(e)}")
+
+ def _show_error_details(self, job):
+ """Show details from error file"""
+ if not job.metadata.get('error_file_id'):
+ return
+
+ try:
+ api_key = self.gui.api_key_entry.get().strip()
+ headers = {'Authorization': f'Bearer {api_key}'}
+
+ # Download error file
+ response = requests.get(
+ f'https://api.openai.com/v1/files/{job.metadata["error_file_id"]}/content',
+ headers=headers
+ )
+
+ if response.status_code == 200:
+ # Parse first few errors
+ errors = []
+ for i, line in enumerate(response.text.strip().split('\n')[:5]): # Show first 5 errors
+ if line:
+ try:
+ error_data = json.loads(line)
+ error_msg = error_data.get('error', {}).get('message', 'Unknown error')
+ errors.append(f"• {error_msg}")
+ except:
+ pass
+
+ error_text = '\n'.join(errors)
+ if len(response.text.strip().split('\n')) > 5:
+ newline = '\n'
+ error_text += f"\n\n... and {len(response.text.strip().split(newline)) - 5} more errors"
+
+ messagebox.showerror(
+ "Batch Processing Errors",
+ f"All requests failed with errors:\n\n{error_text}\n\n"
+ "Common causes:\n"
+ "• Invalid API key or insufficient permissions\n"
+ "• Model not available in your region\n"
+ "• Malformed request format"
+ )
+
+ except Exception as e:
+ print(f"Failed to retrieve error details: {e}")
+
+ def _extract_chapter_number(self, custom_id):
+ """Extract chapter number from custom ID"""
+ match = re.search(r'chapter[_-](\d+)', custom_id, re.IGNORECASE)
+ if match:
+ return int(match.group(1))
+ return 0
+
+ # Helper methods for thread-safe UI updates
+ def _log(self, message, level="info"):
+ """Thread-safe logging to GUI"""
+ # Log based on level
+ if level == "error":
+ print(f"❌ {message}") # This will show in GUI
+ elif level == "warning":
+ print(f"⚠️ {message}") # This will show in GUI
+ else:
+ logger.info(message) # This only goes to log file
+ # Also display info messages in GUI
+ if hasattr(self.gui, 'append_log'):
+ self.dialog.after(0, lambda: self.gui.append_log(message))
+
+ def _show_error(self, message):
+ """Thread-safe error dialog"""
+ self._log(f"Error: {message}", level="error")
+ self.dialog.after(0, lambda: messagebox.showerror("Error", message))
+
+ def _show_info(self, title, message):
+ """Thread-safe info dialog"""
+ self._log(f"{title}: {message}", level="info")
+ self.dialog.after(0, lambda: messagebox.showinfo(title, message))
+
+ def _show_warning(self, message):
+ """Thread-safe warning display"""
+ self._log(f"Warning: {message}", level="warning")
+
+
+def show_async_processing_dialog(parent, translator_gui):
+ """Show the async processing dialog
+
+ Args:
+ parent: Parent window
+ translator_gui: Reference to main TranslatorGUI instance
+ """
+ dialog = AsyncProcessingDialog(parent, translator_gui)
+ return dialog.dialog
+
+
+# Integration function for translator_gui.py
+def add_async_processing_button(translator_gui, parent_frame):
+ """Add async processing button to GUI
+
+ This function should be called from translator_gui.py to add the button
+
+ Args:
+ translator_gui: TranslatorGUI instance
+ parent_frame: Frame to add button to
+ """
+ # Create button with appropriate styling
+ async_button = ttk.Button(
+ parent_frame,
+ text="Async Processing (50% Off)",
+ command=lambda: show_async_processing_dialog(translator_gui.master, translator_gui),
+ style="primary.TButton"
+ )
+
+ # Place button appropriately
+ async_button.pack(side=tk.LEFT, padx=5)
+
+ # Store reference
+ translator_gui.async_button = async_button
+
+ return async_button
diff --git a/bubble_detector.py b/bubble_detector.py
new file mode 100644
index 0000000000000000000000000000000000000000..4502a2b1dc81480581d1305210844024f036d9a8
--- /dev/null
+++ b/bubble_detector.py
@@ -0,0 +1,1881 @@
+"""
+bubble_detector.py - Modified version that works in frozen PyInstaller executables
+Replace your bubble_detector.py with this version
+"""
+import os
+import sys
+import json
+import numpy as np
+import cv2
+from typing import List, Tuple, Optional, Dict, Any
+import logging
+import traceback
+import hashlib
+from pathlib import Path
+import threading
+import time
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Check if we're running in a frozen environment
+IS_FROZEN = getattr(sys, 'frozen', False)
+if IS_FROZEN:
+ # In frozen environment, set proper paths for ML libraries
+ MEIPASS = sys._MEIPASS
+ os.environ['TORCH_HOME'] = MEIPASS
+ os.environ['TRANSFORMERS_CACHE'] = os.path.join(MEIPASS, 'transformers')
+ os.environ['HF_HOME'] = os.path.join(MEIPASS, 'huggingface')
+ logger.info(f"Running in frozen environment: {MEIPASS}")
+
+# Modified import checks for frozen environment
+YOLO_AVAILABLE = False
+YOLO = None
+torch = None
+TORCH_AVAILABLE = False
+ONNX_AVAILABLE = False
+TRANSFORMERS_AVAILABLE = False
+RTDetrForObjectDetection = None
+RTDetrImageProcessor = None
+PIL_AVAILABLE = False
+
+# Try to import YOLO dependencies with better error handling
+if IS_FROZEN:
+ # In frozen environment, try harder to import
+ try:
+ # First try to import torch components individually
+ import torch
+ import torch.nn
+ import torch.cuda
+ TORCH_AVAILABLE = True
+ logger.info("✓ PyTorch loaded in frozen environment")
+ except Exception as e:
+ logger.warning(f"PyTorch not available in frozen environment: {e}")
+ TORCH_AVAILABLE = False
+ torch = None
+
+ # Try ultralytics after torch
+ if TORCH_AVAILABLE:
+ try:
+ from ultralytics import YOLO
+ YOLO_AVAILABLE = True
+ logger.info("✓ Ultralytics YOLO loaded in frozen environment")
+ except Exception as e:
+ logger.warning(f"Ultralytics not available in frozen environment: {e}")
+ YOLO_AVAILABLE = False
+
+ # Try transformers
+ try:
+ import transformers
+ # Try specific imports
+ try:
+ from transformers import RTDetrForObjectDetection, RTDetrImageProcessor
+ TRANSFORMERS_AVAILABLE = True
+ logger.info("✓ Transformers RT-DETR loaded in frozen environment")
+ except ImportError:
+ # Try alternative import
+ try:
+ from transformers import AutoModel, AutoImageProcessor
+ RTDetrForObjectDetection = AutoModel
+ RTDetrImageProcessor = AutoImageProcessor
+ TRANSFORMERS_AVAILABLE = True
+ logger.info("✓ Transformers loaded with AutoModel fallback")
+ except:
+ TRANSFORMERS_AVAILABLE = False
+ logger.warning("Transformers RT-DETR not available in frozen environment")
+ except Exception as e:
+ logger.warning(f"Transformers not available in frozen environment: {e}")
+ TRANSFORMERS_AVAILABLE = False
+else:
+ # Normal environment - original import logic
+ try:
+ from ultralytics import YOLO
+ YOLO_AVAILABLE = True
+ except:
+ YOLO_AVAILABLE = False
+ logger.warning("Ultralytics YOLO not available")
+
+ try:
+ import torch
+ # Test if cuda attribute exists
+ _ = torch.cuda
+ TORCH_AVAILABLE = True
+ except (ImportError, AttributeError):
+ TORCH_AVAILABLE = False
+ torch = None
+ logger.warning("PyTorch not available or incomplete")
+
+ try:
+ from transformers import RTDetrForObjectDetection, RTDetrImageProcessor
+ try:
+ from transformers import RTDetrV2ForObjectDetection
+ RTDetrForObjectDetection = RTDetrV2ForObjectDetection
+ except ImportError:
+ pass
+ TRANSFORMERS_AVAILABLE = True
+ except:
+ TRANSFORMERS_AVAILABLE = False
+ logger.info("Transformers not available for RT-DETR")
+
+# Configure ORT memory behavior before importing
+try:
+ os.environ.setdefault('ORT_DISABLE_MEMORY_ARENA', '1')
+except Exception:
+ pass
+# ONNX Runtime - works well in frozen environments
+try:
+ import onnxruntime as ort
+ ONNX_AVAILABLE = True
+ logger.info("✓ ONNX Runtime available")
+except ImportError:
+ ONNX_AVAILABLE = False
+ logger.warning("ONNX Runtime not available")
+
+# PIL
+try:
+ from PIL import Image
+ PIL_AVAILABLE = True
+except ImportError:
+ PIL_AVAILABLE = False
+ logger.info("PIL not available")
+
+
+class BubbleDetector:
+ """
+ Combined YOLOv8 and RT-DETR speech bubble detector for comics and manga.
+ Supports multiple model formats and provides configurable detection.
+ Backward compatible with existing code while adding RT-DETR support.
+ """
+
+ # Process-wide shared RT-DETR to avoid concurrent meta-device loads
+ _rtdetr_init_lock = threading.Lock()
+ _rtdetr_shared_model = None
+ _rtdetr_shared_processor = None
+ _rtdetr_loaded = False
+ _rtdetr_repo_id = 'ogkalu/comic-text-and-bubble-detector'
+
+ # Shared RT-DETR (ONNX) across process to avoid device/context storms
+ _rtdetr_onnx_init_lock = threading.Lock()
+ _rtdetr_onnx_shared_session = None
+ _rtdetr_onnx_loaded = False
+ _rtdetr_onnx_providers = None
+ _rtdetr_onnx_model_path = None
+ # Limit DML concurrent runs to avoid DXGI device hang. Adjustable via env DML_MAX_CONCURRENT
+ try:
+ _rtdetr_onnx_max_concurrent = int(os.environ.get('DML_MAX_CONCURRENT', '1'))
+ except Exception:
+ _rtdetr_onnx_max_concurrent = 1
+ _rtdetr_onnx_sema = threading.Semaphore(max(1, _rtdetr_onnx_max_concurrent))
+
+ def __init__(self, config_path: str = "config.json"):
+ """
+ Initialize the bubble detector.
+
+ Args:
+ config_path: Path to configuration file
+ """
+ self.config_path = config_path
+ self.config = self._load_config()
+
+ # YOLOv8 components (original)
+ self.model = None
+ self.model_loaded = False
+ self.model_type = None # 'yolo', 'onnx', or 'torch'
+ self.onnx_session = None
+
+ # RT-DETR components (new)
+ self.rtdetr_model = None
+ self.rtdetr_processor = None
+ self.rtdetr_loaded = False
+ self.rtdetr_repo = 'ogkalu/comic-text-and-bubble-detector'
+
+ # RT-DETR (ONNX) backend components
+ self.rtdetr_onnx_session = None
+ self.rtdetr_onnx_loaded = False
+ self.rtdetr_onnx_repo = 'ogkalu/comic-text-and-bubble-detector'
+
+ # RT-DETR class definitions
+ self.CLASS_BUBBLE = 0 # Empty speech bubble
+ self.CLASS_TEXT_BUBBLE = 1 # Bubble with text
+ self.CLASS_TEXT_FREE = 2 # Text without bubble
+
+ # Detection settings
+ self.default_confidence = 0.5
+ self.default_iou_threshold = 0.45
+ # Allow override from settings
+ try:
+ ocr_cfg = self.config.get('manga_settings', {}).get('ocr', {}) if isinstance(self.config, dict) else {}
+ self.default_max_detections = int(ocr_cfg.get('bubble_max_detections', 100))
+ self.max_det_yolo = int(ocr_cfg.get('bubble_max_detections_yolo', self.default_max_detections))
+ self.max_det_rtdetr = int(ocr_cfg.get('bubble_max_detections_rtdetr', self.default_max_detections))
+ except Exception:
+ self.default_max_detections = 100
+ self.max_det_yolo = 100
+ self.max_det_rtdetr = 100
+
+ # Cache directory for ONNX conversions
+ self.cache_dir = os.environ.get('BUBBLE_CACHE_DIR', 'models')
+ os.makedirs(self.cache_dir, exist_ok=True)
+
+ # GPU availability
+ self.use_gpu = TORCH_AVAILABLE and torch.cuda.is_available()
+ self.device = 'cuda' if self.use_gpu else 'cpu'
+
+ # Quantization/precision settings
+ adv_cfg = self.config.get('manga_settings', {}).get('advanced', {}) if isinstance(self.config, dict) else {}
+ ocr_cfg = self.config.get('manga_settings', {}).get('ocr', {}) if isinstance(self.config, dict) else {}
+ env_quant = os.environ.get('MODEL_QUANTIZE', 'false').lower() == 'true'
+ self.quantize_enabled = bool(env_quant or adv_cfg.get('quantize_models', False) or ocr_cfg.get('quantize_bubble_detector', False))
+ self.quantize_dtype = str(adv_cfg.get('torch_precision', os.environ.get('TORCH_PRECISION', 'auto'))).lower()
+ # Prefer advanced.onnx_quantize; fall back to env or global quantize
+ self.onnx_quantize_enabled = bool(adv_cfg.get('onnx_quantize', os.environ.get('ONNX_QUANTIZE', 'false').lower() == 'true' or self.quantize_enabled))
+
+ # Stop flag support
+ self.stop_flag = None
+ self._stopped = False
+ self.log_callback = None
+
+ logger.info(f"🗨️ BubbleDetector initialized")
+ logger.info(f" GPU: {'Available' if self.use_gpu else 'Not available'}")
+ logger.info(f" YOLO: {'Available' if YOLO_AVAILABLE else 'Not installed'}")
+ logger.info(f" ONNX: {'Available' if ONNX_AVAILABLE else 'Not installed'}")
+ logger.info(f" RT-DETR: {'Available' if TRANSFORMERS_AVAILABLE else 'Not installed'}")
+ logger.info(f" Quantization: {'ENABLED' if self.quantize_enabled else 'disabled'} (torch_precision={self.quantize_dtype}, onnx_quantize={'on' if self.onnx_quantize_enabled else 'off'})" )
+
+ def _load_config(self) -> Dict[str, Any]:
+ """Load configuration from file."""
+ if os.path.exists(self.config_path):
+ try:
+ with open(self.config_path, 'r', encoding='utf-8') as f:
+ return json.load(f)
+ except Exception as e:
+ logger.warning(f"Failed to load config: {e}")
+ return {}
+
+ def _save_config(self):
+ """Save configuration to file."""
+ try:
+ with open(self.config_path, 'w', encoding='utf-8') as f:
+ json.dump(self.config, f, indent=2)
+ except Exception as e:
+ logger.error(f"Failed to save config: {e}")
+
+ def set_stop_flag(self, stop_flag):
+ """Set the stop flag for checking interruptions"""
+ self.stop_flag = stop_flag
+ self._stopped = False
+
+ def set_log_callback(self, log_callback):
+ """Set log callback for GUI integration"""
+ self.log_callback = log_callback
+
+ def _check_stop(self) -> bool:
+ """Check if stop has been requested"""
+ if self._stopped:
+ return True
+ if self.stop_flag and self.stop_flag.is_set():
+ self._stopped = True
+ return True
+ # Check global manga translator cancellation
+ try:
+ from manga_translator import MangaTranslator
+ if MangaTranslator.is_globally_cancelled():
+ self._stopped = True
+ return True
+ except Exception:
+ pass
+ return False
+
+ def _log(self, message: str, level: str = "info"):
+ """Log message with stop suppression"""
+ # Suppress logs when stopped (allow only essential stop confirmation messages)
+ if self._check_stop():
+ essential_stop_keywords = [
+ "⏹️ Translation stopped by user",
+ "⏹️ Bubble detection stopped",
+ "cleanup", "🧹"
+ ]
+ if not any(keyword in message for keyword in essential_stop_keywords):
+ return
+
+ if self.log_callback:
+ self.log_callback(message, level)
+ else:
+ logger.info(message) if level == 'info' else getattr(logger, level, logger.info)(message)
+
+ def reset_stop_flags(self):
+ """Reset stop flags when starting new processing"""
+ self._stopped = False
+
+ def load_model(self, model_path: str, force_reload: bool = False) -> bool:
+ """
+ Load a YOLOv8 model for bubble detection.
+
+ Args:
+ model_path: Path to model file (.pt, .onnx, or .torchscript)
+ force_reload: Force reload even if model is already loaded
+
+ Returns:
+ True if model loaded successfully, False otherwise
+ """
+ try:
+ # If given a Hugging Face repo ID (e.g., 'owner/name'), fetch detector.onnx into models/
+ if model_path and (('/' in model_path) and not os.path.exists(model_path)):
+ try:
+ from huggingface_hub import hf_hub_download
+ os.makedirs(self.cache_dir, exist_ok=True)
+ logger.info(f"📥 Resolving repo '{model_path}' to detector.onnx in {self.cache_dir}...")
+ resolved = hf_hub_download(repo_id=model_path, filename='detector.onnx', cache_dir=self.cache_dir, local_dir=self.cache_dir, local_dir_use_symlinks=False)
+ if resolved and os.path.exists(resolved):
+ model_path = resolved
+ logger.info(f"✅ Downloaded detector.onnx to: {model_path}")
+ except Exception as repo_err:
+ logger.error(f"Failed to download from repo '{model_path}': {repo_err}")
+ if not os.path.exists(model_path):
+ logger.error(f"Model file not found: {model_path}")
+ return False
+
+ # Check if it's the same model already loaded
+ if self.model_loaded and not force_reload:
+ last_path = self.config.get('last_model_path', '')
+ if last_path == model_path:
+ logger.info("Model already loaded (same path)")
+ return True
+ else:
+ logger.info(f"Model path changed from {last_path} to {model_path}, reloading...")
+ force_reload = True
+
+ # Clear previous model if force reload
+ if force_reload:
+ logger.info("Force reloading model...")
+ self.model = None
+ self.onnx_session = None
+ self.model_loaded = False
+ self.model_type = None
+
+ logger.info(f"📥 Loading bubble detection model: {model_path}")
+
+ # Determine model type by extension
+ ext = Path(model_path).suffix.lower()
+
+ if ext in ['.pt', '.pth']:
+ if not YOLO_AVAILABLE:
+ logger.warning("Ultralytics package not available in this build")
+ logger.info("Bubble detection will be disabled - this is normal for lightweight builds")
+ # Don't return False immediately, try other fallbacks
+ self.model_loaded = False
+ return False
+
+ # Load YOLOv8 model
+ try:
+ self.model = YOLO(model_path)
+ self.model_type = 'yolo'
+
+ # Set to eval mode
+ if hasattr(self.model, 'model'):
+ self.model.model.eval()
+
+ # Move to GPU if available
+ if self.use_gpu and TORCH_AVAILABLE:
+ try:
+ self.model.to('cuda')
+ except Exception as gpu_error:
+ logger.warning(f"Could not move model to GPU: {gpu_error}")
+
+ logger.info("✅ YOLOv8 model loaded successfully")
+ # Apply optional FP16 precision to reduce VRAM if enabled
+ if self.quantize_enabled and self.use_gpu and TORCH_AVAILABLE:
+ try:
+ m = self.model.model if hasattr(self.model, 'model') else self.model
+ m.half()
+ logger.info("🔻 Applied FP16 precision to YOLO model (GPU)")
+ except Exception as _e:
+ logger.warning(f"Could not switch YOLO model to FP16: {_e}")
+
+ except Exception as yolo_error:
+ logger.error(f"Failed to load YOLO model: {yolo_error}")
+ return False
+
+ elif ext == '.onnx':
+ if not ONNX_AVAILABLE:
+ logger.warning("ONNX Runtime not available in this build")
+ logger.info("ONNX model support disabled - this is normal for lightweight builds")
+ return False
+
+ try:
+ # Load ONNX model
+ providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] if self.use_gpu else ['CPUExecutionProvider']
+ session_path = model_path
+ if self.quantize_enabled:
+ try:
+ from onnxruntime.quantization import quantize_dynamic, QuantType
+ quant_path = os.path.splitext(model_path)[0] + ".int8.onnx"
+ if not os.path.exists(quant_path) or os.environ.get('FORCE_ONNX_REBUILD', 'false').lower() == 'true':
+ logger.info("🔻 Quantizing ONNX model weights to INT8 (dynamic)...")
+ quantize_dynamic(model_input=model_path, model_output=quant_path, weight_type=QuantType.QInt8, op_types_to_quantize=['Conv', 'MatMul'])
+ session_path = quant_path
+ self.config['last_onnx_quantized_path'] = quant_path
+ self._save_config()
+ logger.info(f"✅ Using quantized ONNX model: {quant_path}")
+ except Exception as qe:
+ logger.warning(f"ONNX quantization not applied: {qe}")
+ # Use conservative ORT memory options to reduce RAM growth
+ so = ort.SessionOptions()
+ try:
+ so.enable_mem_pattern = False
+ so.enable_cpu_mem_arena = False
+ except Exception:
+ pass
+ self.onnx_session = ort.InferenceSession(session_path, sess_options=so, providers=providers)
+ self.model_type = 'onnx'
+
+ logger.info("✅ ONNX model loaded successfully")
+
+ except Exception as onnx_error:
+ logger.error(f"Failed to load ONNX model: {onnx_error}")
+ return False
+
+ elif ext == '.torchscript':
+ if not TORCH_AVAILABLE:
+ logger.warning("PyTorch not available in this build")
+ logger.info("TorchScript model support disabled - this is normal for lightweight builds")
+ return False
+
+ try:
+ # Add safety check for torch being None
+ if torch is None:
+ logger.error("PyTorch module is None - cannot load TorchScript model")
+ return False
+
+ # Load TorchScript model
+ self.model = torch.jit.load(model_path, map_location='cpu')
+ self.model.eval()
+ self.model_type = 'torch'
+
+ if self.use_gpu:
+ try:
+ self.model = self.model.cuda()
+ except Exception as gpu_error:
+ logger.warning(f"Could not move TorchScript model to GPU: {gpu_error}")
+
+ logger.info("✅ TorchScript model loaded successfully")
+
+ # Optional FP16 precision on GPU
+ if self.quantize_enabled and self.use_gpu and TORCH_AVAILABLE:
+ try:
+ self.model = self.model.half()
+ logger.info("🔻 Applied FP16 precision to TorchScript model (GPU)")
+ except Exception as _e:
+ logger.warning(f"Could not switch TorchScript model to FP16: {_e}")
+
+ except Exception as torch_error:
+ logger.error(f"Failed to load TorchScript model: {torch_error}")
+ return False
+
+ else:
+ logger.error(f"Unsupported model format: {ext}")
+ logger.info("Supported formats: .pt/.pth (YOLOv8), .onnx (ONNX), .torchscript (TorchScript)")
+ return False
+
+ # Only set loaded if we actually succeeded
+ self.model_loaded = True
+ self.config['last_model_path'] = model_path
+ self.config['model_type'] = self.model_type
+ self._save_config()
+
+ return True
+
+ except Exception as e:
+ logger.error(f"Failed to load model: {e}")
+ logger.error(traceback.format_exc())
+ self.model_loaded = False
+
+ # Provide helpful context for .exe users
+ logger.info("Note: If running from .exe, some ML libraries may not be included")
+ logger.info("This is normal for lightweight builds - bubble detection will be disabled")
+
+ return False
+
+ def load_rtdetr_model(self, model_path: str = None, model_id: str = None, force_reload: bool = False) -> bool:
+ """
+ Load RT-DETR model for advanced bubble and text detection.
+ This implementation avoids the 'meta tensor' copy error by:
+ - Serializing the entire load under a class lock (no concurrent loads)
+ - Loading directly onto the target device (CUDA if available) via device_map='auto'
+ - Avoiding .to() on a potentially-meta model; no device migration post-load
+
+ Args:
+ model_path: Optional path to local model
+ model_id: Optional HuggingFace model ID (default: 'ogkalu/comic-text-and-bubble-detector')
+ force_reload: Force reload even if already loaded
+
+ Returns:
+ True if successful, False otherwise
+ """
+ if not TRANSFORMERS_AVAILABLE:
+ logger.error("Transformers library required for RT-DETR. Install with: pip install transformers")
+ return False
+
+ if not PIL_AVAILABLE:
+ logger.error("PIL required for RT-DETR. Install with: pip install pillow")
+ return False
+
+ if self.rtdetr_loaded and not force_reload:
+ logger.info("RT-DETR model already loaded")
+ return True
+
+ # Fast path: if shared already loaded and not forcing reload, attach
+ if BubbleDetector._rtdetr_loaded and not force_reload:
+ self.rtdetr_model = BubbleDetector._rtdetr_shared_model
+ self.rtdetr_processor = BubbleDetector._rtdetr_shared_processor
+ self.rtdetr_loaded = True
+ logger.info("RT-DETR model attached from shared cache")
+ return True
+
+ # Serialize the ENTIRE loading sequence to avoid concurrent init issues
+ with BubbleDetector._rtdetr_init_lock:
+ try:
+ # Re-check after acquiring lock
+ if BubbleDetector._rtdetr_loaded and not force_reload:
+ self.rtdetr_model = BubbleDetector._rtdetr_shared_model
+ self.rtdetr_processor = BubbleDetector._rtdetr_shared_processor
+ self.rtdetr_loaded = True
+ logger.info("RT-DETR model attached from shared cache (post-lock)")
+ return True
+
+ # Use custom model_id if provided, otherwise use default
+ repo_id = model_id if model_id else self.rtdetr_repo
+ logger.info(f"📥 Loading RT-DETR model from {repo_id}...")
+
+ # Ensure TorchDynamo/compile doesn't interfere on some builds
+ try:
+ os.environ.setdefault('TORCHDYNAMO_DISABLE', '1')
+ except Exception:
+ pass
+
+ # Decide device strategy
+ gpu_available = bool(TORCH_AVAILABLE and hasattr(torch, 'cuda') and torch.cuda.is_available())
+ device_map = 'auto' if gpu_available else None
+ # Choose dtype
+ dtype = None
+ if TORCH_AVAILABLE:
+ try:
+ dtype = torch.float16 if gpu_available else torch.float32
+ except Exception:
+ dtype = None
+ low_cpu = True if gpu_available else False
+
+ # Load processor (once)
+ self.rtdetr_processor = RTDetrImageProcessor.from_pretrained(
+ repo_id,
+ size={"width": 640, "height": 640},
+ cache_dir=self.cache_dir if not model_path else None
+ )
+
+ # Prepare kwargs for from_pretrained
+ from_kwargs = {
+ 'cache_dir': self.cache_dir if not model_path else None,
+ 'low_cpu_mem_usage': low_cpu,
+ 'device_map': device_map,
+ }
+ if dtype is not None:
+ from_kwargs['dtype'] = dtype
+
+ # First attempt: load directly to target (CUDA if available)
+ try:
+ self.rtdetr_model = RTDetrForObjectDetection.from_pretrained(
+ model_path if model_path else repo_id,
+ **from_kwargs,
+ )
+ except Exception as primary_err:
+ # Fallback to a simple CPU load (no device move) if CUDA path fails
+ logger.warning(f"RT-DETR primary load failed ({primary_err}); retrying on CPU...")
+ from_kwargs_fallback = {
+ 'cache_dir': self.cache_dir if not model_path else None,
+ 'low_cpu_mem_usage': False,
+ 'device_map': None,
+ }
+ if TORCH_AVAILABLE:
+ from_kwargs_fallback['dtype'] = torch.float32
+ self.rtdetr_model = RTDetrForObjectDetection.from_pretrained(
+ model_path if model_path else repo_id,
+ **from_kwargs_fallback,
+ )
+
+ # Optional dynamic quantization for linear layers (CPU only)
+ if self.quantize_enabled and TORCH_AVAILABLE and (not gpu_available):
+ try:
+ try:
+ import torch.ao.quantization as tq
+ quantize_dynamic = tq.quantize_dynamic # type: ignore
+ except Exception:
+ import torch.quantization as tq # type: ignore
+ quantize_dynamic = tq.quantize_dynamic # type: ignore
+ self.rtdetr_model = quantize_dynamic(self.rtdetr_model, {torch.nn.Linear}, dtype=torch.qint8)
+ logger.info("🔻 Applied dynamic INT8 quantization to RT-DETR linear layers (CPU)")
+ except Exception as qe:
+ logger.warning(f"RT-DETR dynamic quantization skipped: {qe}")
+
+ # Finalize
+ self.rtdetr_model.eval()
+
+ # Sanity check: ensure no parameter is left on 'meta' device
+ try:
+ for n, p in self.rtdetr_model.named_parameters():
+ dev = getattr(p, 'device', None)
+ if dev is not None and getattr(dev, 'type', '') == 'meta':
+ raise RuntimeError(f"Parameter {n} is on 'meta' device after load")
+ except Exception as e:
+ logger.error(f"RT-DETR load sanity check failed: {e}")
+ self.rtdetr_loaded = False
+ return False
+
+ # Publish shared cache
+ BubbleDetector._rtdetr_shared_model = self.rtdetr_model
+ BubbleDetector._rtdetr_shared_processor = self.rtdetr_processor
+ BubbleDetector._rtdetr_loaded = True
+ BubbleDetector._rtdetr_repo_id = repo_id
+
+ self.rtdetr_loaded = True
+
+ # Save the model ID that was used
+ self.config['rtdetr_loaded'] = True
+ self.config['rtdetr_model_id'] = repo_id
+ self._save_config()
+
+ loc = 'CUDA' if gpu_available else 'CPU'
+ logger.info(f"✅ RT-DETR model loaded successfully ({loc})")
+ logger.info(" Classes: Empty bubbles, Text bubbles, Free text")
+
+ # Auto-convert to ONNX for RT-DETR only if explicitly enabled
+ if os.environ.get('AUTO_CONVERT_RTDETR_ONNX', 'false').lower() == 'true':
+ onnx_path = os.path.join(self.cache_dir, 'rtdetr_comic.onnx')
+ if self.convert_to_onnx('rtdetr', onnx_path):
+ logger.info("🚀 RT-DETR converted to ONNX for faster inference")
+ # Store ONNX path for later use
+ self.config['rtdetr_onnx_path'] = onnx_path
+ self._save_config()
+ # Optionally quantize ONNX for reduced RAM
+ if self.onnx_quantize_enabled:
+ try:
+ from onnxruntime.quantization import quantize_dynamic, QuantType
+ quant_path = os.path.splitext(onnx_path)[0] + ".int8.onnx"
+ if not os.path.exists(quant_path) or os.environ.get('FORCE_ONNX_REBUILD', 'false').lower() == 'true':
+ logger.info("🔻 Quantizing RT-DETR ONNX to INT8 (dynamic)...")
+ quantize_dynamic(model_input=onnx_path, model_output=quant_path, weight_type=QuantType.QInt8, op_types_to_quantize=['Conv', 'MatMul'])
+ self.config['rtdetr_onnx_quantized_path'] = quant_path
+ self._save_config()
+ logger.info(f"✅ Quantized RT-DETR ONNX saved to: {quant_path}")
+ except Exception as qe:
+ logger.warning(f"ONNX quantization for RT-DETR skipped: {qe}")
+ else:
+ logger.info("ℹ️ Skipping RT-DETR ONNX export (converter not supported in current environment)")
+
+ return True
+ except Exception as e:
+ logger.error(f"❌ Failed to load RT-DETR: {e}")
+ self.rtdetr_loaded = False
+ return False
+
+ def check_rtdetr_available(self, model_id: str = None) -> bool:
+ """
+ Check if RT-DETR model is available (cached).
+
+ Args:
+ model_id: Optional HuggingFace model ID
+
+ Returns:
+ True if model is cached and available
+ """
+ try:
+ from pathlib import Path
+
+ # Use provided model_id or default
+ repo_id = model_id if model_id else self.rtdetr_repo
+
+ # Check HuggingFace cache
+ cache_dir = Path.home() / ".cache" / "huggingface" / "hub"
+ model_id_formatted = repo_id.replace("/", "--")
+
+ # Look for model folder
+ model_folders = list(cache_dir.glob(f"models--{model_id_formatted}*"))
+
+ if model_folders:
+ for folder in model_folders:
+ if (folder / "snapshots").exists():
+ snapshots = list((folder / "snapshots").iterdir())
+ if snapshots:
+ return True
+
+ return False
+
+ except Exception:
+ return False
+
+ def detect_bubbles(self,
+ image_path: str,
+ confidence: float = None,
+ iou_threshold: float = None,
+ max_detections: int = None,
+ use_rtdetr: bool = None) -> List[Tuple[int, int, int, int]]:
+ """
+ Detect speech bubbles in an image (backward compatible method).
+
+ Args:
+ image_path: Path to image file
+ confidence: Minimum confidence threshold (0-1)
+ iou_threshold: IOU threshold for NMS (0-1)
+ max_detections: Maximum number of detections to return
+ use_rtdetr: If True, use RT-DETR instead of YOLOv8 (if available)
+
+ Returns:
+ List of bubble bounding boxes as (x, y, width, height) tuples
+ """
+ # Check for stop at start
+ if self._check_stop():
+ self._log("⏹️ Bubble detection stopped by user", "warning")
+ return []
+
+ # Decide which model to use
+ if use_rtdetr is None:
+ # Auto-select: prefer RT-DETR if available
+ use_rtdetr = self.rtdetr_loaded
+
+ if use_rtdetr:
+ # Prefer ONNX backend if available, else PyTorch
+ if getattr(self, 'rtdetr_onnx_loaded', False):
+ results = self.detect_with_rtdetr_onnx(
+ image_path=image_path,
+ confidence=confidence,
+ return_all_bubbles=True
+ )
+ return results
+ if self.rtdetr_loaded:
+ results = self.detect_with_rtdetr(
+ image_path=image_path,
+ confidence=confidence,
+ return_all_bubbles=True
+ )
+ return results
+
+ # Original YOLOv8 detection
+ if not self.model_loaded:
+ logger.error("No model loaded. Call load_model() first.")
+ return []
+
+ # Use defaults if not specified
+ confidence = confidence or self.default_confidence
+ iou_threshold = iou_threshold or self.default_iou_threshold
+ max_detections = max_detections or self.default_max_detections
+
+ try:
+ # Load image
+ image = cv2.imread(image_path)
+ if image is None:
+ logger.error(f"Failed to load image: {image_path}")
+ return []
+
+ h, w = image.shape[:2]
+ self._log(f"🔍 Detecting bubbles in {w}x{h} image")
+
+ # Check for stop before inference
+ if self._check_stop():
+ self._log("⏹️ Bubble detection inference stopped by user", "warning")
+ return []
+
+ if self.model_type == 'yolo':
+ # YOLOv8 inference
+ results = self.model(
+ image_path,
+ conf=confidence,
+ iou=iou_threshold,
+ max_det=min(max_detections, getattr(self, 'max_det_yolo', max_detections)),
+ verbose=False
+ )
+
+ bubbles = []
+ for r in results:
+ if r.boxes is not None:
+ for box in r.boxes:
+ # Get box coordinates
+ x1, y1, x2, y2 = box.xyxy[0].cpu().numpy()
+ x, y = int(x1), int(y1)
+ width = int(x2 - x1)
+ height = int(y2 - y1)
+
+ # Get confidence
+ conf = float(box.conf[0])
+
+ # Add to list
+ if len(bubbles) < max_detections:
+ bubbles.append((x, y, width, height))
+
+ logger.debug(f" Bubble: ({x},{y}) {width}x{height} conf={conf:.2f}")
+
+ elif self.model_type == 'onnx':
+ # ONNX inference
+ bubbles = self._detect_with_onnx(image, confidence, iou_threshold, max_detections)
+
+ elif self.model_type == 'torch':
+ # TorchScript inference
+ bubbles = self._detect_with_torchscript(image, confidence, iou_threshold, max_detections)
+
+ else:
+ logger.error(f"Unknown model type: {self.model_type}")
+ return []
+
+ logger.info(f"✅ Detected {len(bubbles)} speech bubbles")
+ time.sleep(0.1) # Brief pause for stability
+ logger.debug("💤 Bubble detection pausing briefly for stability")
+ return bubbles
+
+ except Exception as e:
+ logger.error(f"Detection failed: {e}")
+ logger.error(traceback.format_exc())
+ return []
+
+ def detect_with_rtdetr(self,
+ image_path: str = None,
+ image: np.ndarray = None,
+ confidence: float = None,
+ return_all_bubbles: bool = False) -> Any:
+ """
+ Detect using RT-DETR model with 3-class detection (PyTorch backend).
+
+ Args:
+ image_path: Path to image file
+ image: Image array (BGR format)
+ confidence: Confidence threshold
+ return_all_bubbles: If True, return list of bubble boxes (for compatibility)
+ If False, return dict with all classes
+
+ Returns:
+ List of bubbles if return_all_bubbles=True, else dict with classes
+ """
+ # Check for stop at start
+ if self._check_stop():
+ self._log("⏹️ RT-DETR detection stopped by user", "warning")
+ if return_all_bubbles:
+ return []
+ return {'bubbles': [], 'text_bubbles': [], 'text_free': []}
+
+ if not self.rtdetr_loaded:
+ self._log("RT-DETR not loaded. Call load_rtdetr_model() first.", "warning")
+ if return_all_bubbles:
+ return []
+ return {'bubbles': [], 'text_bubbles': [], 'text_free': []}
+
+ confidence = confidence or self.default_confidence
+
+ try:
+ # Load image
+ if image_path:
+ image = cv2.imread(image_path)
+ elif image is None:
+ logger.error("No image provided")
+ if return_all_bubbles:
+ return []
+ return {'bubbles': [], 'text_bubbles': [], 'text_free': []}
+
+ # Convert BGR to RGB for PIL
+ image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+ pil_image = Image.fromarray(image_rgb)
+
+ # Prepare image for model
+ inputs = self.rtdetr_processor(images=pil_image, return_tensors="pt")
+
+ # Move inputs to the same device as the model and match model dtype for floating tensors
+ model_device = next(self.rtdetr_model.parameters()).device if self.rtdetr_model is not None else (torch.device('cpu') if TORCH_AVAILABLE else 'cpu')
+ model_dtype = None
+ if TORCH_AVAILABLE and self.rtdetr_model is not None:
+ try:
+ model_dtype = next(self.rtdetr_model.parameters()).dtype
+ except Exception:
+ model_dtype = None
+
+ if TORCH_AVAILABLE:
+ new_inputs = {}
+ for k, v in inputs.items():
+ if isinstance(v, torch.Tensor):
+ v = v.to(model_device)
+ if model_dtype is not None and torch.is_floating_point(v):
+ v = v.to(model_dtype)
+ new_inputs[k] = v
+ inputs = new_inputs
+
+ # Run inference with autocast when model is half/bfloat16 on CUDA
+ use_amp = TORCH_AVAILABLE and hasattr(model_device, 'type') and model_device.type == 'cuda' and (model_dtype in (torch.float16, torch.bfloat16))
+ autocast_dtype = model_dtype if model_dtype in (torch.float16, torch.bfloat16) else None
+
+ with torch.no_grad():
+ if use_amp and autocast_dtype is not None:
+ with torch.autocast('cuda', dtype=autocast_dtype):
+ outputs = self.rtdetr_model(**inputs)
+ else:
+ outputs = self.rtdetr_model(**inputs)
+
+ # Brief pause for stability after inference
+ time.sleep(0.1)
+ logger.debug("💤 RT-DETR inference pausing briefly for stability")
+
+ # Post-process results
+ target_sizes = torch.tensor([pil_image.size[::-1]]) if TORCH_AVAILABLE else None
+ if TORCH_AVAILABLE and hasattr(model_device, 'type') and model_device.type == "cuda":
+ target_sizes = target_sizes.to(model_device)
+
+ results = self.rtdetr_processor.post_process_object_detection(
+ outputs,
+ target_sizes=target_sizes,
+ threshold=confidence
+ )[0]
+
+ # Apply per-detector cap if configured
+ cap = getattr(self, 'max_det_rtdetr', self.default_max_detections)
+ if cap and len(results['boxes']) > cap:
+ # Keep top-scoring first
+ scores = results['scores']
+ top_idx = scores.topk(k=cap).indices if hasattr(scores, 'topk') else range(cap)
+ results = {
+ 'boxes': [results['boxes'][i] for i in top_idx],
+ 'scores': [results['scores'][i] for i in top_idx],
+ 'labels': [results['labels'][i] for i in top_idx]
+ }
+
+ logger.info(f"📊 RT-DETR found {len(results['boxes'])} detections above {confidence:.2f} confidence")
+
+ # Organize detections by class
+ detections = {
+ 'bubbles': [], # Empty speech bubbles
+ 'text_bubbles': [], # Bubbles with text
+ 'text_free': [] # Text without bubbles
+ }
+
+ for box, score, label in zip(results['boxes'], results['scores'], results['labels']):
+ x1, y1, x2, y2 = map(int, box.tolist())
+ width = x2 - x1
+ height = y2 - y1
+
+ # Store as (x, y, width, height) to match YOLOv8 format
+ bbox = (x1, y1, width, height)
+
+ label_id = label.item()
+ if label_id == self.CLASS_BUBBLE:
+ detections['bubbles'].append(bbox)
+ elif label_id == self.CLASS_TEXT_BUBBLE:
+ detections['text_bubbles'].append(bbox)
+ elif label_id == self.CLASS_TEXT_FREE:
+ detections['text_free'].append(bbox)
+
+ # Stop early if we hit the configured cap across all classes
+ total_count = len(detections['bubbles']) + len(detections['text_bubbles']) + len(detections['text_free'])
+ if total_count >= (self.config.get('manga_settings', {}).get('ocr', {}).get('bubble_max_detections', self.default_max_detections) if isinstance(self.config, dict) else self.default_max_detections):
+ break
+
+ # Log results
+ total = len(detections['bubbles']) + len(detections['text_bubbles']) + len(detections['text_free'])
+ logger.info(f"✅ RT-DETR detected {total} objects:")
+ logger.info(f" - Empty bubbles: {len(detections['bubbles'])}")
+ logger.info(f" - Text bubbles: {len(detections['text_bubbles'])}")
+ logger.info(f" - Free text: {len(detections['text_free'])}")
+
+ # Return format based on compatibility mode
+ if return_all_bubbles:
+ # Return all bubbles (empty + with text) for backward compatibility
+ all_bubbles = detections['bubbles'] + detections['text_bubbles']
+ return all_bubbles
+ else:
+ return detections
+
+ except Exception as e:
+ logger.error(f"RT-DETR detection failed: {e}")
+ logger.error(traceback.format_exc())
+ if return_all_bubbles:
+ return []
+ return {'bubbles': [], 'text_bubbles': [], 'text_free': []}
+
+ def detect_all_text_regions(self, image_path: str = None, image: np.ndarray = None) -> List[Tuple[int, int, int, int]]:
+ """
+ Detect all text regions using RT-DETR (both in bubbles and free text).
+
+ Returns:
+ List of bounding boxes for all text regions
+ """
+ if not self.rtdetr_loaded:
+ logger.warning("RT-DETR required for text detection")
+ return []
+
+ detections = self.detect_with_rtdetr(image_path=image_path, image=image, return_all_bubbles=False)
+
+ # Combine text bubbles and free text
+ all_text = detections['text_bubbles'] + detections['text_free']
+
+ logger.info(f"📝 Found {len(all_text)} text regions total")
+ return all_text
+
+ def _detect_with_onnx(self, image: np.ndarray, confidence: float,
+ iou_threshold: float, max_detections: int) -> List[Tuple[int, int, int, int]]:
+ """Run detection using ONNX model."""
+ # Preprocess image
+ img_size = 640 # Standard YOLOv8 input size
+ img_resized = cv2.resize(image, (img_size, img_size))
+ img_norm = img_resized.astype(np.float32) / 255.0
+ img_transposed = np.transpose(img_norm, (2, 0, 1))
+ img_batch = np.expand_dims(img_transposed, axis=0)
+
+ # Run inference
+ input_name = self.onnx_session.get_inputs()[0].name
+ outputs = self.onnx_session.run(None, {input_name: img_batch})
+
+ # Process outputs (YOLOv8 format)
+ predictions = outputs[0][0] # Remove batch dimension
+
+ # Filter by confidence and apply NMS
+ bubbles = []
+ boxes = []
+ scores = []
+
+ for pred in predictions.T: # Transpose to get predictions per detection
+ if len(pred) >= 5:
+ x_center, y_center, width, height, obj_conf = pred[:5]
+
+ if obj_conf >= confidence:
+ # Convert to corner coordinates
+ x1 = x_center - width / 2
+ y1 = y_center - height / 2
+
+ # Scale to original image size
+ h, w = image.shape[:2]
+ x1 = int(x1 * w / img_size)
+ y1 = int(y1 * h / img_size)
+ width = int(width * w / img_size)
+ height = int(height * h / img_size)
+
+ boxes.append([x1, y1, x1 + width, y1 + height])
+ scores.append(float(obj_conf))
+
+ # Apply NMS
+ if boxes:
+ indices = cv2.dnn.NMSBoxes(boxes, scores, confidence, iou_threshold)
+ if len(indices) > 0:
+ indices = indices.flatten()[:max_detections]
+ for i in indices:
+ x1, y1, x2, y2 = boxes[i]
+ bubbles.append((x1, y1, x2 - x1, y2 - y1))
+
+ return bubbles
+
+ def _detect_with_torchscript(self, image: np.ndarray, confidence: float,
+ iou_threshold: float, max_detections: int) -> List[Tuple[int, int, int, int]]:
+ """Run detection using TorchScript model."""
+ # Similar to ONNX but using PyTorch tensors
+ img_size = 640
+ img_resized = cv2.resize(image, (img_size, img_size))
+ img_norm = img_resized.astype(np.float32) / 255.0
+ img_tensor = torch.from_numpy(img_norm).permute(2, 0, 1).unsqueeze(0)
+
+ if self.use_gpu:
+ img_tensor = img_tensor.cuda()
+
+ with torch.no_grad():
+ outputs = self.model(img_tensor)
+
+ # Process outputs similar to ONNX
+ # Implementation depends on exact model output format
+ # This is a placeholder - adjust based on your model
+ return []
+
+ def visualize_detections(self, image_path: str, bubbles: List[Tuple[int, int, int, int]] = None,
+ output_path: str = None, use_rtdetr: bool = False) -> np.ndarray:
+ """
+ Visualize detected bubbles on the image.
+
+ Args:
+ image_path: Path to original image
+ bubbles: List of bubble bounding boxes (if None, will detect)
+ output_path: Optional path to save visualization
+ use_rtdetr: Use RT-DETR for visualization with class colors
+
+ Returns:
+ Image with drawn bounding boxes
+ """
+ image = cv2.imread(image_path)
+ if image is None:
+ logger.error(f"Failed to load image: {image_path}")
+ return None
+
+ vis_image = image.copy()
+
+ if use_rtdetr and self.rtdetr_loaded:
+ # RT-DETR visualization with different colors per class
+ detections = self.detect_with_rtdetr(image_path=image_path, return_all_bubbles=False)
+
+ # Colors for each class
+ colors = {
+ 'bubbles': (0, 255, 0), # Green for empty bubbles
+ 'text_bubbles': (255, 0, 0), # Blue for text bubbles
+ 'text_free': (0, 0, 255) # Red for free text
+ }
+
+ # Draw detections
+ for class_name, bboxes in detections.items():
+ color = colors[class_name]
+
+ for i, (x, y, w, h) in enumerate(bboxes):
+ # Draw rectangle
+ cv2.rectangle(vis_image, (x, y), (x + w, y + h), color, 2)
+
+ # Add label
+ label = f"{class_name.replace('_', ' ').title()} {i+1}"
+ label_size, _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
+ cv2.rectangle(vis_image, (x, y - label_size[1] - 4),
+ (x + label_size[0], y), color, -1)
+ cv2.putText(vis_image, label, (x, y - 2),
+ cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1)
+ else:
+ # Original YOLOv8 visualization
+ if bubbles is None:
+ bubbles = self.detect_bubbles(image_path)
+
+ # Draw bounding boxes
+ for i, (x, y, w, h) in enumerate(bubbles):
+ # Draw rectangle
+ color = (0, 255, 0) # Green
+ thickness = 2
+ cv2.rectangle(vis_image, (x, y), (x + w, y + h), color, thickness)
+
+ # Add label
+ label = f"Bubble {i+1}"
+ label_size, _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
+ cv2.rectangle(vis_image, (x, y - label_size[1] - 4), (x + label_size[0], y), color, -1)
+ cv2.putText(vis_image, label, (x, y - 2), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1)
+
+ # Save if output path provided
+ if output_path:
+ cv2.imwrite(output_path, vis_image)
+ logger.info(f"💾 Visualization saved to: {output_path}")
+
+ return vis_image
+
+ def convert_to_onnx(self, model_path: str, output_path: str = None) -> bool:
+ """
+ Convert a YOLOv8 or RT-DETR model to ONNX format.
+
+ Args:
+ model_path: Path to model file or 'rtdetr' for loaded RT-DETR
+ output_path: Path for ONNX output (auto-generated if None)
+
+ Returns:
+ True if conversion successful, False otherwise
+ """
+ try:
+ logger.info(f"🔄 Converting {model_path} to ONNX...")
+
+ # Generate output path if not provided
+ if output_path is None:
+ if model_path == 'rtdetr' and self.rtdetr_loaded:
+ base_name = 'rtdetr_comic'
+ else:
+ base_name = Path(model_path).stem
+ output_path = os.path.join(self.cache_dir, f"{base_name}.onnx")
+
+ # Check if already exists
+ if os.path.exists(output_path) and not os.environ.get('FORCE_ONNX_REBUILD', 'false').lower() == 'true':
+ logger.info(f"✅ ONNX model already exists: {output_path}")
+ return True
+
+ # Handle RT-DETR conversion
+ if model_path == 'rtdetr' and self.rtdetr_loaded:
+ if not TORCH_AVAILABLE:
+ logger.error("PyTorch required for RT-DETR ONNX conversion")
+ return False
+
+ # RT-DETR specific conversion
+ self.rtdetr_model.eval()
+
+ # Create dummy input (pixel values): BxCxHxW
+ dummy_input = torch.randn(1, 3, 640, 640)
+ if self.device == 'cuda':
+ dummy_input = dummy_input.to('cuda')
+
+ # Wrap the model to return only tensors (logits, pred_boxes)
+ class _RTDetrExportWrapper(torch.nn.Module):
+ def __init__(self, mdl):
+ super().__init__()
+ self.mdl = mdl
+ def forward(self, images):
+ out = self.mdl(pixel_values=images)
+ # Handle dict/ModelOutput/tuple outputs
+ logits = None
+ boxes = None
+ try:
+ if isinstance(out, dict):
+ logits = out.get('logits', None)
+ boxes = out.get('pred_boxes', out.get('boxes', None))
+ else:
+ logits = getattr(out, 'logits', None)
+ boxes = getattr(out, 'pred_boxes', getattr(out, 'boxes', None))
+ except Exception:
+ pass
+ if (logits is None or boxes is None) and isinstance(out, (tuple, list)) and len(out) >= 2:
+ logits, boxes = out[0], out[1]
+ return logits, boxes
+
+ wrapper = _RTDetrExportWrapper(self.rtdetr_model)
+ if self.device == 'cuda':
+ wrapper = wrapper.to('cuda')
+
+ # Try PyTorch 2.x dynamo_export first (more tolerant of newer aten ops)
+ try:
+ success = False
+ try:
+ from torch.onnx import dynamo_export
+ try:
+ exp = dynamo_export(wrapper, dummy_input)
+ except TypeError:
+ # Older PyTorch dynamo_export may not support this calling convention
+ exp = dynamo_export(wrapper, dummy_input)
+ # exp may have save(); otherwise, it may expose model_proto
+ try:
+ exp.save(output_path) # type: ignore
+ success = True
+ except Exception:
+ try:
+ import onnx as _onnx
+ _onnx.save(exp.model_proto, output_path) # type: ignore
+ success = True
+ except Exception as _se:
+ logger.warning(f"dynamo_export produced model but could not save: {_se}")
+ except Exception as de:
+ logger.warning(f"dynamo_export failed; falling back to legacy exporter: {de}")
+ if success:
+ logger.info(f"✅ RT-DETR ONNX saved to: {output_path} (dynamo_export)")
+ return True
+ except Exception as de2:
+ logger.warning(f"dynamo_export path error: {de2}")
+
+ # Legacy exporter with opset fallback
+ last_err = None
+ for opset in [19, 18, 17, 16, 15, 14, 13]:
+ try:
+ torch.onnx.export(
+ wrapper,
+ dummy_input,
+ output_path,
+ export_params=True,
+ opset_version=opset,
+ do_constant_folding=True,
+ input_names=['pixel_values'],
+ output_names=['logits', 'boxes'],
+ dynamic_axes={
+ 'pixel_values': {0: 'batch', 2: 'height', 3: 'width'},
+ 'logits': {0: 'batch'},
+ 'boxes': {0: 'batch'}
+ }
+ )
+ logger.info(f"✅ RT-DETR ONNX saved to: {output_path} (opset {opset})")
+ return True
+ except Exception as _e:
+ last_err = _e
+ try:
+ msg = str(_e)
+ except Exception:
+ msg = ''
+ logger.warning(f"RT-DETR ONNX export failed at opset {opset}: {msg}")
+ continue
+
+ logger.error(f"All RT-DETR ONNX export attempts failed. Last error: {last_err}")
+ return False
+
+ # Handle YOLOv8 conversion - FIXED
+ elif YOLO_AVAILABLE and os.path.exists(model_path):
+ logger.info(f"Loading YOLOv8 model from: {model_path}")
+
+ # Load model
+ model = YOLO(model_path)
+
+ # Export to ONNX - this returns the path to the exported model
+ logger.info("Exporting to ONNX format...")
+ exported_path = model.export(format='onnx', imgsz=640, simplify=True)
+
+ # exported_path could be a string or Path object
+ exported_path = str(exported_path) if exported_path else None
+
+ if exported_path and os.path.exists(exported_path):
+ # Move to desired location if different
+ if exported_path != output_path:
+ import shutil
+ logger.info(f"Moving ONNX from {exported_path} to {output_path}")
+ shutil.move(exported_path, output_path)
+
+ logger.info(f"✅ YOLOv8 ONNX saved to: {output_path}")
+ return True
+ else:
+ # Fallback: check if it was created with expected name
+ expected_onnx = model_path.replace('.pt', '.onnx')
+ if os.path.exists(expected_onnx):
+ if expected_onnx != output_path:
+ import shutil
+ shutil.move(expected_onnx, output_path)
+ logger.info(f"✅ YOLOv8 ONNX saved to: {output_path}")
+ return True
+ else:
+ logger.error(f"ONNX export failed - no output file found")
+ return False
+
+ else:
+ logger.error(f"Cannot convert {model_path}: Model not found or dependencies missing")
+ return False
+
+ except Exception as e:
+ logger.error(f"Conversion failed: {e}")
+ # Avoid noisy full stack trace in production logs; return False gracefully
+ return False
+
+ def batch_detect(self, image_paths: List[str], **kwargs) -> Dict[str, List[Tuple[int, int, int, int]]]:
+ """
+ Detect bubbles in multiple images.
+
+ Args:
+ image_paths: List of image paths
+ **kwargs: Detection parameters (confidence, iou_threshold, max_detections, use_rtdetr)
+
+ Returns:
+ Dictionary mapping image paths to bubble lists
+ """
+ results = {}
+
+ for i, image_path in enumerate(image_paths):
+ logger.info(f"Processing image {i+1}/{len(image_paths)}: {os.path.basename(image_path)}")
+ bubbles = self.detect_bubbles(image_path, **kwargs)
+ results[image_path] = bubbles
+
+ return results
+
+ def unload(self, release_shared: bool = False):
+ """Release model resources held by this detector instance.
+ Args:
+ release_shared: If True, also clear class-level shared RT-DETR caches.
+ """
+ try:
+ # Release instance-level models and sessions
+ try:
+ if getattr(self, 'onnx_session', None) is not None:
+ self.onnx_session = None
+ except Exception:
+ pass
+ try:
+ if getattr(self, 'rtdetr_onnx_session', None) is not None:
+ self.rtdetr_onnx_session = None
+ except Exception:
+ pass
+ for attr in ['model', 'rtdetr_model', 'rtdetr_processor']:
+ try:
+ if hasattr(self, attr):
+ setattr(self, attr, None)
+ except Exception:
+ pass
+ for flag in ['model_loaded', 'rtdetr_loaded', 'rtdetr_onnx_loaded']:
+ try:
+ if hasattr(self, flag):
+ setattr(self, flag, False)
+ except Exception:
+ pass
+
+ # Optional: release shared caches
+ if release_shared:
+ try:
+ BubbleDetector._rtdetr_shared_model = None
+ BubbleDetector._rtdetr_shared_processor = None
+ BubbleDetector._rtdetr_loaded = False
+ except Exception:
+ pass
+
+ # Free CUDA cache and trigger GC
+ try:
+ if TORCH_AVAILABLE and torch is not None and torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ except Exception:
+ pass
+ try:
+ import gc
+ gc.collect()
+ except Exception:
+ pass
+ except Exception:
+ # Best-effort only
+ pass
+
+ def get_bubble_masks(self, image_path: str, bubbles: List[Tuple[int, int, int, int]]) -> np.ndarray:
+ """
+ Create a mask image with bubble regions.
+
+ Args:
+ image_path: Path to original image
+ bubbles: List of bubble bounding boxes
+
+ Returns:
+ Binary mask with bubble regions as white (255)
+ """
+ image = cv2.imread(image_path)
+ if image is None:
+ return None
+
+ h, w = image.shape[:2]
+ mask = np.zeros((h, w), dtype=np.uint8)
+
+ # Fill bubble regions
+ for x, y, bw, bh in bubbles:
+ cv2.rectangle(mask, (x, y), (x + bw, y + bh), 255, -1)
+
+ return mask
+
+ def filter_bubbles_by_size(self, bubbles: List[Tuple[int, int, int, int]],
+ min_area: int = 100,
+ max_area: int = None) -> List[Tuple[int, int, int, int]]:
+ """
+ Filter bubbles by area.
+
+ Args:
+ bubbles: List of bubble bounding boxes
+ min_area: Minimum area in pixels
+ max_area: Maximum area in pixels (None for no limit)
+
+ Returns:
+ Filtered list of bubbles
+ """
+ filtered = []
+
+ for x, y, w, h in bubbles:
+ area = w * h
+ if area >= min_area and (max_area is None or area <= max_area):
+ filtered.append((x, y, w, h))
+
+ return filtered
+
+ def merge_overlapping_bubbles(self, bubbles: List[Tuple[int, int, int, int]],
+ overlap_threshold: float = 0.1) -> List[Tuple[int, int, int, int]]:
+ """
+ Merge overlapping bubble detections.
+
+ Args:
+ bubbles: List of bubble bounding boxes
+ overlap_threshold: Minimum overlap ratio to merge
+
+ Returns:
+ Merged list of bubbles
+ """
+ if not bubbles:
+ return []
+
+ # Convert to numpy array for easier manipulation
+ boxes = np.array([(x, y, x+w, y+h) for x, y, w, h in bubbles])
+
+ merged = []
+ used = set()
+
+ for i, box1 in enumerate(boxes):
+ if i in used:
+ continue
+
+ # Start with current box
+ x1, y1, x2, y2 = box1
+
+ # Check for overlaps with remaining boxes
+ for j in range(i + 1, len(boxes)):
+ if j in used:
+ continue
+
+ box2 = boxes[j]
+
+ # Calculate intersection
+ ix1 = max(x1, box2[0])
+ iy1 = max(y1, box2[1])
+ ix2 = min(x2, box2[2])
+ iy2 = min(y2, box2[3])
+
+ if ix1 < ix2 and iy1 < iy2:
+ # Calculate overlap ratio
+ intersection = (ix2 - ix1) * (iy2 - iy1)
+ area1 = (x2 - x1) * (y2 - y1)
+ area2 = (box2[2] - box2[0]) * (box2[3] - box2[1])
+ overlap = intersection / min(area1, area2)
+
+ if overlap >= overlap_threshold:
+ # Merge boxes
+ x1 = min(x1, box2[0])
+ y1 = min(y1, box2[1])
+ x2 = max(x2, box2[2])
+ y2 = max(y2, box2[3])
+ used.add(j)
+
+ merged.append((int(x1), int(y1), int(x2 - x1), int(y2 - y1)))
+
+ return merged
+
+ # ============================
+ # RT-DETR (ONNX) BACKEND
+ # ============================
+ def load_rtdetr_onnx_model(self, model_id: str = None, force_reload: bool = False) -> bool:
+ """
+ Load RT-DETR ONNX model using onnxruntime. Downloads detector.onnx and config.json
+ from the provided Hugging Face repo if not already cached.
+ """
+ if not ONNX_AVAILABLE:
+ logger.error("ONNX Runtime not available for RT-DETR ONNX backend")
+ return False
+ try:
+ # If singleton mode and already loaded, just attach shared session
+ try:
+ adv = (self.config or {}).get('manga_settings', {}).get('advanced', {}) if isinstance(self.config, dict) else {}
+ singleton = bool(adv.get('use_singleton_models', True))
+ except Exception:
+ singleton = True
+ if singleton and BubbleDetector._rtdetr_onnx_loaded and not force_reload and BubbleDetector._rtdetr_onnx_shared_session is not None:
+ self.rtdetr_onnx_session = BubbleDetector._rtdetr_onnx_shared_session
+ self.rtdetr_onnx_loaded = True
+ return True
+
+ repo = model_id or self.rtdetr_onnx_repo
+ try:
+ from huggingface_hub import hf_hub_download
+ except Exception as e:
+ logger.error(f"huggingface-hub required to fetch RT-DETR ONNX: {e}")
+ return False
+
+ # Ensure local models dir (use configured cache_dir directly: e.g., 'models')
+ cache_dir = self.cache_dir
+ os.makedirs(cache_dir, exist_ok=True)
+
+ # Download files into models/ and avoid symlinks so the file is visible there
+ try:
+ _ = hf_hub_download(repo_id=repo, filename='config.json', cache_dir=cache_dir, local_dir=cache_dir, local_dir_use_symlinks=False)
+ except Exception:
+ pass
+ onnx_fp = hf_hub_download(repo_id=repo, filename='detector.onnx', cache_dir=cache_dir, local_dir=cache_dir, local_dir_use_symlinks=False)
+ BubbleDetector._rtdetr_onnx_model_path = onnx_fp
+
+ # Pick providers: prefer CUDA if available; otherwise CPU. Do NOT use DML.
+ providers = ['CPUExecutionProvider']
+ try:
+ avail = ort.get_available_providers() if ONNX_AVAILABLE else []
+ if 'CUDAExecutionProvider' in avail:
+ providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']
+ except Exception:
+ pass
+
+ # Session options with reduced memory arena and optional thread limiting in singleton mode
+ so = ort.SessionOptions()
+ try:
+ so.enable_mem_pattern = False
+ so.enable_cpu_mem_arena = False
+ except Exception:
+ pass
+ # If singleton models mode is enabled in config, limit ORT threading to reduce CPU spikes
+ try:
+ adv = (self.config or {}).get('manga_settings', {}).get('advanced', {}) if isinstance(self.config, dict) else {}
+ if bool(adv.get('use_singleton_models', True)):
+ so.intra_op_num_threads = 1
+ so.inter_op_num_threads = 1
+ try:
+ so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
+ except Exception:
+ pass
+ try:
+ so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_BASIC
+ except Exception:
+ pass
+ except Exception:
+ pass
+
+ # Create session (serialize creation in singleton mode to avoid device storms)
+ if singleton:
+ with BubbleDetector._rtdetr_onnx_init_lock:
+ # Re-check after acquiring lock
+ if BubbleDetector._rtdetr_onnx_loaded and BubbleDetector._rtdetr_onnx_shared_session is not None and not force_reload:
+ self.rtdetr_onnx_session = BubbleDetector._rtdetr_onnx_shared_session
+ self.rtdetr_onnx_loaded = True
+ return True
+ sess = ort.InferenceSession(onnx_fp, providers=providers, sess_options=so)
+ BubbleDetector._rtdetr_onnx_shared_session = sess
+ BubbleDetector._rtdetr_onnx_loaded = True
+ BubbleDetector._rtdetr_onnx_providers = providers
+ self.rtdetr_onnx_session = sess
+ self.rtdetr_onnx_loaded = True
+ else:
+ self.rtdetr_onnx_session = ort.InferenceSession(onnx_fp, providers=providers, sess_options=so)
+ self.rtdetr_onnx_loaded = True
+ logger.info("✅ RT-DETR (ONNX) model ready")
+ return True
+ except Exception as e:
+ logger.error(f"Failed to load RT-DETR ONNX: {e}")
+ self.rtdetr_onnx_session = None
+ self.rtdetr_onnx_loaded = False
+ return False
+
+ def detect_with_rtdetr_onnx(self,
+ image_path: str = None,
+ image: np.ndarray = None,
+ confidence: float = 0.3,
+ return_all_bubbles: bool = False) -> Any:
+ """Detect using RT-DETR ONNX backend.
+ Returns bubbles list if return_all_bubbles else dict by classes similar to PyTorch path.
+ """
+ if not self.rtdetr_onnx_loaded or self.rtdetr_onnx_session is None:
+ logger.warning("RT-DETR ONNX not loaded")
+ return [] if return_all_bubbles else {'bubbles': [], 'text_bubbles': [], 'text_free': []}
+ try:
+ # Acquire image
+ if image_path is not None:
+ import cv2
+ image = cv2.imread(image_path)
+ if image is None:
+ raise RuntimeError(f"Failed to read image: {image_path}")
+ image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+ else:
+ if image is None:
+ raise RuntimeError("No image provided")
+ # Assume image is BGR np.ndarray if from OpenCV
+ try:
+ import cv2
+ image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+ except Exception:
+ image_rgb = image
+
+ # To PIL then resize 640x640 as in reference
+ from PIL import Image as _PILImage
+ pil_image = _PILImage.fromarray(image_rgb)
+ im_resized = pil_image.resize((640, 640))
+ arr = np.asarray(im_resized, dtype=np.float32) / 255.0
+ arr = np.transpose(arr, (2, 0, 1)) # (3,H,W)
+ im_data = arr[np.newaxis, ...]
+
+ w, h = pil_image.size
+ orig_size = np.array([[w, h]], dtype=np.int64)
+
+ # Run with a concurrency guard when using DML to prevent device hangs
+ providers = BubbleDetector._rtdetr_onnx_providers or []
+ def _do_run(session):
+ return session.run(None, {
+ 'images': im_data,
+ 'orig_target_sizes': orig_size
+ })
+ if 'DmlExecutionProvider' in providers:
+ acquired = False
+ try:
+ BubbleDetector._rtdetr_onnx_sema.acquire()
+ acquired = True
+ outputs = _do_run(self.rtdetr_onnx_session)
+ except Exception as dml_err:
+ msg = str(dml_err)
+ if '887A0005' in msg or '887A0006' in msg or 'Dml' in msg:
+ # Rebuild CPU session and retry once
+ try:
+ base_path = BubbleDetector._rtdetr_onnx_model_path
+ if base_path:
+ so = ort.SessionOptions()
+ so.enable_mem_pattern = False
+ so.enable_cpu_mem_arena = False
+ cpu_providers = ['CPUExecutionProvider']
+ # Serialize rebuild
+ with BubbleDetector._rtdetr_onnx_init_lock:
+ sess = ort.InferenceSession(base_path, providers=cpu_providers, sess_options=so)
+ BubbleDetector._rtdetr_onnx_shared_session = sess
+ BubbleDetector._rtdetr_onnx_providers = cpu_providers
+ self.rtdetr_onnx_session = sess
+ outputs = _do_run(self.rtdetr_onnx_session)
+ else:
+ raise
+ except Exception:
+ raise
+ else:
+ raise
+ finally:
+ if acquired:
+ try:
+ BubbleDetector._rtdetr_onnx_sema.release()
+ except Exception:
+ pass
+ else:
+ outputs = _do_run(self.rtdetr_onnx_session)
+
+ # outputs expected: labels, boxes, scores
+ labels, boxes, scores = outputs[:3]
+ if labels.ndim == 2 and labels.shape[0] == 1:
+ labels = labels[0]
+ if scores.ndim == 2 and scores.shape[0] == 1:
+ scores = scores[0]
+ if boxes.ndim == 3 and boxes.shape[0] == 1:
+ boxes = boxes[0]
+
+ detections = {'bubbles': [], 'text_bubbles': [], 'text_free': []}
+ bubbles_all = []
+ for lab, box, scr in zip(labels, boxes, scores):
+ if float(scr) < float(confidence):
+ continue
+ x1, y1, x2, y2 = map(int, box)
+ bbox = (x1, y1, x2 - x1, y2 - y1)
+ label_id = int(lab)
+ if label_id == self.CLASS_BUBBLE:
+ detections['bubbles'].append(bbox)
+ bubbles_all.append(bbox)
+ elif label_id == self.CLASS_TEXT_BUBBLE:
+ detections['text_bubbles'].append(bbox)
+ bubbles_all.append(bbox)
+ elif label_id == self.CLASS_TEXT_FREE:
+ detections['text_free'].append(bbox)
+
+ return bubbles_all if return_all_bubbles else detections
+ except Exception as e:
+ logger.error(f"RT-DETR ONNX detection failed: {e}")
+ return [] if return_all_bubbles else {'bubbles': [], 'text_bubbles': [], 'text_free': []}
+
+
+# Standalone utility functions
+def download_model_from_huggingface(repo_id: str = "ogkalu/comic-speech-bubble-detector-yolov8m",
+ filename: str = "comic-speech-bubble-detector-yolov8m.pt",
+ cache_dir: str = "models") -> str:
+ """
+ Download model from Hugging Face Hub.
+
+ Args:
+ repo_id: Hugging Face repository ID
+ filename: Model filename in the repository
+ cache_dir: Local directory to cache the model
+
+ Returns:
+ Path to downloaded model file
+ """
+ try:
+ from huggingface_hub import hf_hub_download
+
+ os.makedirs(cache_dir, exist_ok=True)
+
+ logger.info(f"📥 Downloading {filename} from {repo_id}...")
+
+ model_path = hf_hub_download(
+ repo_id=repo_id,
+ filename=filename,
+ cache_dir=cache_dir,
+ local_dir=cache_dir
+ )
+
+ logger.info(f"✅ Model downloaded to: {model_path}")
+ return model_path
+
+ except ImportError:
+ logger.error("huggingface-hub package required. Install with: pip install huggingface-hub")
+ return None
+ except Exception as e:
+ logger.error(f"Download failed: {e}")
+ return None
+
+
+def download_rtdetr_model(cache_dir: str = "models") -> bool:
+ """
+ Download RT-DETR model for advanced detection.
+
+ Args:
+ cache_dir: Directory to cache the model
+
+ Returns:
+ True if successful
+ """
+ if not TRANSFORMERS_AVAILABLE:
+ logger.error("Transformers required. Install with: pip install transformers")
+ return False
+
+ try:
+ logger.info("📥 Downloading RT-DETR model...")
+ from transformers import RTDetrForObjectDetection, RTDetrImageProcessor
+
+ # This will download and cache the model
+ processor = RTDetrImageProcessor.from_pretrained(
+ "ogkalu/comic-text-and-bubble-detector",
+ cache_dir=cache_dir
+ )
+ model = RTDetrForObjectDetection.from_pretrained(
+ "ogkalu/comic-text-and-bubble-detector",
+ cache_dir=cache_dir
+ )
+
+ logger.info("✅ RT-DETR model downloaded successfully")
+ return True
+
+ except Exception as e:
+ logger.error(f"Download failed: {e}")
+ return False
+
+
+# Example usage and testing
+if __name__ == "__main__":
+ import sys
+
+ # Create detector
+ detector = BubbleDetector()
+
+ if len(sys.argv) > 1:
+ if sys.argv[1] == "download":
+ # Download model from Hugging Face
+ model_path = download_model_from_huggingface()
+ if model_path:
+ print(f"YOLOv8 model downloaded to: {model_path}")
+
+ # Also download RT-DETR
+ if download_rtdetr_model():
+ print("RT-DETR model downloaded")
+
+ elif sys.argv[1] == "detect" and len(sys.argv) > 3:
+ # Detect bubbles in an image
+ model_path = sys.argv[2]
+ image_path = sys.argv[3]
+
+ # Load appropriate model
+ if 'rtdetr' in model_path.lower():
+ if detector.load_rtdetr_model():
+ # Use RT-DETR
+ results = detector.detect_with_rtdetr(image_path)
+ print(f"RT-DETR Detection:")
+ print(f" Empty bubbles: {len(results['bubbles'])}")
+ print(f" Text bubbles: {len(results['text_bubbles'])}")
+ print(f" Free text: {len(results['text_free'])}")
+ else:
+ if detector.load_model(model_path):
+ bubbles = detector.detect_bubbles(image_path, confidence=0.5)
+ print(f"YOLOv8 detected {len(bubbles)} bubbles:")
+ for i, (x, y, w, h) in enumerate(bubbles):
+ print(f" Bubble {i+1}: position=({x},{y}) size=({w}x{h})")
+
+ # Optionally visualize
+ if len(sys.argv) > 4:
+ output_path = sys.argv[4]
+ detector.visualize_detections(image_path, output_path=output_path,
+ use_rtdetr='rtdetr' in model_path.lower())
+
+ elif sys.argv[1] == "test-both" and len(sys.argv) > 2:
+ # Test both models
+ image_path = sys.argv[2]
+
+ # Load YOLOv8
+ yolo_path = "models/comic-speech-bubble-detector-yolov8m.pt"
+ if os.path.exists(yolo_path):
+ detector.load_model(yolo_path)
+ yolo_bubbles = detector.detect_bubbles(image_path, use_rtdetr=False)
+ print(f"YOLOv8: {len(yolo_bubbles)} bubbles")
+
+ # Load RT-DETR
+ if detector.load_rtdetr_model():
+ rtdetr_bubbles = detector.detect_bubbles(image_path, use_rtdetr=True)
+ print(f"RT-DETR: {len(rtdetr_bubbles)} bubbles")
+
+ else:
+ print("Usage:")
+ print(" python bubble_detector.py download")
+ print(" python bubble_detector.py detect [output_path]")
+ print(" python bubble_detector.py test-both ")
+
+ else:
+ print("Bubble Detector Module (YOLOv8 + RT-DETR)")
+ print("Usage:")
+ print(" python bubble_detector.py download")
+ print(" python bubble_detector.py detect [output_path]")
+ print(" python bubble_detector.py test-both ")
diff --git a/chapter_extraction_manager.py b/chapter_extraction_manager.py
new file mode 100644
index 0000000000000000000000000000000000000000..f3cefa5cd83c34cd2081c9697c34378d0af799a7
--- /dev/null
+++ b/chapter_extraction_manager.py
@@ -0,0 +1,403 @@
+#!/usr/bin/env python3
+"""
+Chapter Extraction Manager - Manages chapter extraction in subprocess to prevent GUI freezing
+"""
+
+import subprocess
+import sys
+import os
+import json
+import threading
+import queue
+import time
+from pathlib import Path
+
+
+class ChapterExtractionManager:
+ """
+ Manages chapter extraction in a separate process to prevent GUI freezing
+ Similar to GlossaryManager but for chapter extraction
+ """
+
+ def __init__(self, log_callback=None):
+ """
+ Initialize the extraction manager
+
+ Args:
+ log_callback: Function to call with log messages (for GUI integration)
+ """
+ self.log_callback = log_callback
+ self.process = None
+ self.output_queue = queue.Queue()
+ self.error_queue = queue.Queue()
+ self.result = None
+ self.is_running = False
+ self.stop_requested = False
+
+ def extract_chapters_async(self, epub_path, output_dir, extraction_mode="smart",
+ progress_callback=None, completion_callback=None):
+ """
+ Start chapter extraction in a subprocess
+
+ Args:
+ epub_path: Path to EPUB file
+ output_dir: Output directory for extracted content
+ extraction_mode: Extraction mode (smart, comprehensive, full, enhanced)
+ progress_callback: Function to call with progress updates
+ completion_callback: Function to call when extraction completes
+ """
+ if self.is_running:
+ self._log("⚠️ Chapter extraction already in progress")
+ return False
+
+ self.is_running = True
+ self.stop_requested = False
+ self.result = None
+
+ # Start extraction in a thread that manages the subprocess
+ thread = threading.Thread(
+ target=self._run_extraction_subprocess,
+ args=(epub_path, output_dir, extraction_mode, progress_callback, completion_callback),
+ daemon=True
+ )
+ thread.start()
+
+ return True
+
+ def _run_extraction_subprocess(self, epub_path, output_dir, extraction_mode,
+ progress_callback, completion_callback):
+ """
+ Run the extraction subprocess and handle its output
+ """
+ try:
+ # Build command differently for frozen vs dev mode
+ if getattr(sys, 'frozen', False):
+ # In a frozen one-file build, sys.executable is our GUI .exe, not Python.
+ # Use an internal worker-mode flag handled by translator_gui.py to run the worker.
+ cmd = [
+ sys.executable,
+ '--run-chapter-extraction',
+ epub_path,
+ output_dir,
+ extraction_mode
+ ]
+ else:
+ # In dev mode, invoke the worker script with the Python interpreter
+ base_dir = Path(__file__).parent
+ worker_script = base_dir / "chapter_extraction_worker.py"
+ cmd = [
+ sys.executable,
+ str(worker_script),
+ epub_path,
+ output_dir,
+ extraction_mode
+ ]
+
+ # Set environment to force UTF-8 encoding
+ env = os.environ.copy()
+ env['PYTHONIOENCODING'] = 'utf-8'
+ env['PYTHONLEGACYWINDOWSSTDIO'] = '0' # Use new Windows console API
+
+ self._log(f"🚀 Starting chapter extraction subprocess...")
+ self._log(f"📚 EPUB: {os.path.basename(epub_path)}")
+ self._log(f"📂 Output: {output_dir}")
+ self._log(f"⚙️ Mode: {extraction_mode}")
+
+ # Start the subprocess with UTF-8 encoding
+ self.process = subprocess.Popen(
+ cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ encoding='utf-8',
+ errors='replace', # Replace invalid chars instead of failing
+ bufsize=1,
+ universal_newlines=True,
+ env=env # Pass the environment with UTF-8 settings
+ )
+
+ # Read output in real-time
+ while True:
+ if self.stop_requested:
+ self._terminate_process()
+ break
+
+ # Check if process is still running
+ if self.process.poll() is not None:
+ break
+
+ # Read stdout line by line with error handling
+ try:
+ line = self.process.stdout.readline()
+ if not line:
+ continue
+
+ line = line.strip()
+ if not line:
+ continue
+ except UnicodeDecodeError as e:
+ self._log(f"⚠️ Encoding error reading output: {e}")
+ continue
+
+ # Skip all processing if stop is requested to suppress logs
+ if self.stop_requested:
+ continue
+
+ # Parse output based on prefix
+ if line.startswith("[PROGRESS]"):
+ # Progress update
+ message = line[10:].strip()
+ if progress_callback:
+ progress_callback(message)
+ self._log(f"📊 {message}")
+
+ elif line.startswith("[INFO]"):
+ # Information message
+ message = line[6:].strip()
+ self._log(f"ℹ️ {message}")
+
+ elif line.startswith("[ERROR]"):
+ # Error message
+ message = line[7:].strip()
+ self._log(f"❌ {message}")
+ self.error_queue.put(message)
+
+ elif line.startswith("[RESULT]"):
+ # Final result as JSON
+ try:
+ json_str = line[8:].strip()
+ self.result = json.loads(json_str)
+
+ if self.result.get("success"):
+ self._log(f"✅ Extraction completed successfully!")
+ self._log(f"📚 Extracted {self.result.get('chapters', 0)} chapters")
+ else:
+ error = self.result.get("error", "Unknown error")
+ self._log(f"❌ Extraction failed: {error}")
+
+ except json.JSONDecodeError as e:
+ self._log(f"⚠️ Failed to parse result: {e}")
+
+ elif line.startswith("["):
+ # Other prefixed messages - skip
+ pass
+ else:
+ # Regular output - only log if not too verbose
+ if not any(skip in line for skip in ["📁 Searching for", "📁 Found", "📁 ✓", "📁 ✗"]):
+ self._log(line)
+
+ # Get any remaining output - but only process if not stopped
+ if not self.stop_requested:
+ remaining_output, remaining_error = self.process.communicate(timeout=1)
+
+ # Process any remaining output
+ if remaining_output:
+ for line in remaining_output.strip().split('\n'):
+ if line and not line.startswith("["):
+ self._log(line)
+
+ # Check for errors
+ if remaining_error:
+ for line in remaining_error.strip().split('\n'):
+ if line:
+ self._log(f"⚠️ {line}")
+
+ # Check final status
+ if self.process.returncode != 0:
+ self._log(f"⚠️ Process exited with code {self.process.returncode}")
+ else:
+ # If stopped, just clean up without processing output
+ try:
+ self.process.communicate(timeout=0.1)
+ except subprocess.TimeoutExpired:
+ pass # Ignore timeout when cleaning up
+
+ except subprocess.TimeoutExpired:
+ if not self.stop_requested:
+ self._log("⚠️ Subprocess communication timeout")
+ self._terminate_process()
+
+ except Exception as e:
+ # Only log errors if not stopping (unless it's a critical error)
+ if not self.stop_requested or "Subprocess error" in str(e):
+ self._log(f"❌ Subprocess error: {e}")
+ self.result = {
+ "success": False,
+ "error": str(e) if not self.stop_requested else "Extraction stopped by user"
+ }
+
+ finally:
+ self.is_running = False
+ # Store process reference before clearing it in case termination is needed
+ process_ref = self.process
+ self.process = None
+
+ # If process is still running, try to clean it up
+ if process_ref and process_ref.poll() is None:
+ try:
+ process_ref.terminate()
+ time.sleep(0.1) # Brief wait
+ if process_ref.poll() is None:
+ process_ref.kill()
+ except Exception:
+ pass # Ignore cleanup errors in finally block
+
+ # Ensure result is never None
+ if self.result is None:
+ if self.stop_requested:
+ self.result = {
+ "success": False,
+ "error": "Extraction stopped by user"
+ }
+ else:
+ self.result = {
+ "success": False,
+ "error": "Extraction process ended unexpectedly"
+ }
+
+ # Call completion callback
+ if completion_callback:
+ completion_callback(self.result)
+
+ def stop_extraction(self):
+ """Stop the extraction process"""
+ if not self.is_running:
+ return False
+
+ # Set stop flag first to suppress subsequent logs
+ self.stop_requested = True
+ self._log("🛑 Stopping chapter extraction...")
+
+ # Store process reference to avoid race condition
+ process_ref = self.process
+
+ # Give it a moment to stop gracefully
+ time.sleep(0.5)
+
+ # Force terminate if still running and process still exists
+ if process_ref:
+ self._terminate_process_ref(process_ref)
+
+ return True
+
+ def _terminate_process(self):
+ """Terminate the subprocess using current process reference"""
+ if self.process:
+ self._terminate_process_ref(self.process)
+
+ def _terminate_process_ref(self, process_ref):
+ """Terminate a specific process reference"""
+ if not process_ref:
+ return
+
+ try:
+ # Check if process is still alive before attempting termination
+ if process_ref.poll() is None:
+ process_ref.terminate()
+ # Give it a moment to terminate
+ time.sleep(0.5)
+
+ # Force kill if still running
+ if process_ref.poll() is None:
+ process_ref.kill()
+ time.sleep(0.1) # Brief wait after kill
+
+ # Only log termination if not stopping (user already knows they stopped it)
+ if not self.stop_requested:
+ self._log("✅ Process terminated")
+ else:
+ # Only log if not stopping
+ if not self.stop_requested:
+ self._log("✅ Process already terminated")
+ except Exception as e:
+ # Always log termination errors as they might indicate a problem
+ self._log(f"⚠️ Error terminating process: {e}")
+
+ def _log(self, message):
+ """Log a message using the callback if available"""
+ # Suppress logs when stop is requested (except for stop/termination messages)
+ if self.stop_requested and not any(keyword in message for keyword in ["🛑", "✅ Process terminated", "❌ Subprocess error"]):
+ return
+
+ if self.log_callback:
+ self.log_callback(message)
+ else:
+ print(message)
+
+ def is_extraction_running(self):
+ """Check if extraction is currently running"""
+ return self.is_running
+
+ def get_result(self):
+ """Get the extraction result if available"""
+ return self.result
+
+
+# Example usage
+if __name__ == "__main__":
+ import tkinter as tk
+ from tkinter import filedialog
+
+ def test_extraction():
+ """Test the extraction manager"""
+
+ # Create a simple GUI for testing
+ root = tk.Tk()
+ root.title("Chapter Extraction Test")
+ root.geometry("800x600")
+
+ # Text widget for logs
+ text = tk.Text(root, wrap=tk.WORD)
+ text.pack(fill=tk.BOTH, expand=True, padx=10, pady=10)
+
+ # Log callback
+ def log_message(msg):
+ text.insert(tk.END, msg + "\n")
+ text.see(tk.END)
+ root.update_idletasks()
+
+ # Progress callback
+ def progress_update(msg):
+ log_message(f"📊 Progress: {msg}")
+
+ # Completion callback
+ def extraction_complete(result):
+ if result and result.get("success"):
+ log_message(f"✅ Extraction completed!")
+ log_message(f" Chapters: {result.get('chapters', 0)}")
+ else:
+ log_message(f"❌ Extraction failed!")
+
+ # Create manager
+ manager = ChapterExtractionManager(log_callback=log_message)
+
+ # File selection
+ epub_path = filedialog.askopenfilename(
+ title="Select EPUB file",
+ filetypes=[("EPUB files", "*.epub"), ("All files", "*.*")]
+ )
+
+ if epub_path:
+ output_dir = os.path.splitext(os.path.basename(epub_path))[0]
+
+ # Start extraction
+ manager.extract_chapters_async(
+ epub_path,
+ output_dir,
+ extraction_mode="smart",
+ progress_callback=progress_update,
+ completion_callback=extraction_complete
+ )
+
+ # Button to stop
+ stop_btn = tk.Button(
+ root,
+ text="Stop Extraction",
+ command=lambda: manager.stop_extraction()
+ )
+ stop_btn.pack(pady=5)
+
+ root.mainloop()
+
+ # Run test
+ test_extraction()
\ No newline at end of file
diff --git a/chapter_extraction_worker.py b/chapter_extraction_worker.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2b0ecfebe96e3d26a62db65a4969570a683e9dc
--- /dev/null
+++ b/chapter_extraction_worker.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+"""
+Chapter Extraction Worker - Runs chapter extraction in a separate process to prevent GUI freezing
+"""
+
+import sys
+import os
+import io
+
+# Force UTF-8 encoding for stdout/stderr on Windows
+sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
+sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace')
+import json
+import zipfile
+import time
+import traceback
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+def run_chapter_extraction(epub_path, output_dir, extraction_mode="smart", progress_callback=None):
+ """
+ Run chapter extraction in this worker process
+
+ Args:
+ epub_path: Path to EPUB file
+ output_dir: Output directory for extracted content
+ extraction_mode: Extraction mode (smart, comprehensive, full, enhanced)
+ progress_callback: Callback function for progress updates (uses print for IPC)
+
+ Returns:
+ dict: Extraction results including chapters and metadata
+ """
+ try:
+ # Import here to avoid loading heavy modules until needed
+ from TransateKRtoEN import ChapterExtractor
+
+ # Create progress callback that prints to stdout for IPC
+ def worker_progress_callback(message):
+ # Use special prefix for progress messages
+ print(f"[PROGRESS] {message}", flush=True)
+
+ # Create extractor with progress callback
+ extractor = ChapterExtractor(progress_callback=worker_progress_callback)
+
+ # Set extraction mode
+ os.environ["EXTRACTION_MODE"] = extraction_mode
+
+ # Open EPUB and extract chapters
+ print(f"[INFO] Starting extraction of: {epub_path}", flush=True)
+ print(f"[INFO] Output directory: {output_dir}", flush=True)
+ print(f"[INFO] Extraction mode: {extraction_mode}", flush=True)
+
+ with zipfile.ZipFile(epub_path, 'r') as zf:
+ # Extract metadata first
+ metadata = extractor._extract_epub_metadata(zf)
+ print(f"[INFO] Extracted metadata: {list(metadata.keys())}", flush=True)
+
+ # Extract chapters
+ chapters = extractor.extract_chapters(zf, output_dir)
+
+ print(f"[INFO] Extracted {len(chapters)} chapters", flush=True)
+
+ # The extract_chapters method already handles OPF sorting internally
+ # Just log if OPF was used
+ opf_path = os.path.join(output_dir, 'content.opf')
+ if os.path.exists(opf_path):
+ print(f"[INFO] OPF file available for chapter ordering", flush=True)
+
+ # CRITICAL: Save the full chapters with body content!
+ # This is what the main process needs to load
+ chapters_full_path = os.path.join(output_dir, "chapters_full.json")
+ try:
+ with open(chapters_full_path, 'w', encoding='utf-8') as f:
+ json.dump(chapters, f, ensure_ascii=False)
+ print(f"[INFO] Saved full chapters data to: {chapters_full_path}", flush=True)
+ except Exception as e:
+ print(f"[WARNING] Could not save full chapters: {e}", flush=True)
+ # Fall back to saving individual files
+ for chapter in chapters:
+ try:
+ chapter_file = f"chapter_{chapter['num']:04d}_{chapter.get('filename', 'content').replace('/', '_')}.html"
+ chapter_path = os.path.join(output_dir, chapter_file)
+ with open(chapter_path, 'w', encoding='utf-8') as f:
+ f.write(chapter.get('body', ''))
+ print(f"[INFO] Saved chapter {chapter['num']} to {chapter_file}", flush=True)
+ except Exception as ce:
+ print(f"[WARNING] Could not save chapter {chapter.get('num')}: {ce}", flush=True)
+
+ # Return results as JSON for IPC
+ result = {
+ "success": True,
+ "chapters": len(chapters),
+ "metadata": metadata,
+ "chapter_info": [
+ {
+ "num": ch.get("num"),
+ "title": ch.get("title"),
+ "has_images": ch.get("has_images", False),
+ "file_size": ch.get("file_size", 0),
+ "content_hash": ch.get("content_hash", "")
+ }
+ for ch in chapters
+ ]
+ }
+
+ # Output result as JSON
+ print(f"[RESULT] {json.dumps(result)}", flush=True)
+ return result
+
+ except Exception as e:
+ # Send error information
+ error_info = {
+ "success": False,
+ "error": str(e),
+ "traceback": traceback.format_exc()
+ }
+ print(f"[ERROR] {str(e)}", flush=True)
+ print(f"[RESULT] {json.dumps(error_info)}", flush=True)
+ return error_info
+
+
+def main():
+ """Main entry point for worker process"""
+
+ # Parse command line arguments
+ if len(sys.argv) < 3:
+ print("[ERROR] Usage: chapter_extraction_worker.py [extraction_mode]", flush=True)
+ sys.exit(1)
+
+ epub_path = sys.argv[1]
+ output_dir = sys.argv[2]
+ extraction_mode = sys.argv[3] if len(sys.argv) > 3 else "smart"
+
+ # Validate inputs
+ if not os.path.exists(epub_path):
+ print(f"[ERROR] EPUB file not found: {epub_path}", flush=True)
+ sys.exit(1)
+
+ # Create output directory if needed
+ os.makedirs(output_dir, exist_ok=True)
+
+ # Run extraction
+ result = run_chapter_extraction(epub_path, output_dir, extraction_mode)
+
+ # Exit with appropriate code
+ sys.exit(0 if result.get("success", False) else 1)
+
+
+if __name__ == "__main__":
+ # Ensure freeze support for Windows frozen exe
+ try:
+ import multiprocessing
+ multiprocessing.freeze_support()
+ except Exception:
+ pass
+ main()
diff --git a/chapter_splitter.py b/chapter_splitter.py
new file mode 100644
index 0000000000000000000000000000000000000000..342330217c3f10a121e5478ea01d55001b735b63
--- /dev/null
+++ b/chapter_splitter.py
@@ -0,0 +1,195 @@
+import re
+from bs4 import BeautifulSoup
+import tiktoken
+
+class ChapterSplitter:
+ """Split large chapters into smaller chunks while preserving structure"""
+
+ def __init__(self, model_name="gpt-3.5-turbo", target_tokens=80000, compression_factor=1.0):
+ """
+ Initialize splitter with token counter
+ target_tokens: Target size for each chunk (leaving room for system prompt & history)
+ compression_factor: Expected compression ratio from source to target language (0.7-1.0)
+ """
+ try:
+ self.enc = tiktoken.encoding_for_model(model_name)
+ except:
+ self.enc = tiktoken.get_encoding("cl100k_base")
+ self.target_tokens = target_tokens
+ self.compression_factor = compression_factor
+
+ def count_tokens(self, text):
+ """Count tokens in text"""
+ try:
+ return len(self.enc.encode(text))
+ except:
+ # Fallback estimation
+ return len(text) // 4
+
+ def split_chapter(self, chapter_html, max_tokens=None):
+ """
+ Split a chapter into smaller chunks if it exceeds token limit
+ Returns: List of (chunk_html, chunk_index, total_chunks)
+ """
+ if max_tokens is None:
+ max_tokens = self.target_tokens
+
+ # Apply compression factor to output token limit
+ # If compression_factor is 0.7 and max_tokens is 4096,
+ # we expect output to be 4096 * 0.7 = 2867 tokens
+ effective_max_tokens = int(max_tokens * self.compression_factor)
+
+ # First check if splitting is needed
+ total_tokens = self.count_tokens(chapter_html)
+ if total_tokens <= effective_max_tokens:
+ return [(chapter_html, 1, 1)] # No split needed
+
+ # Parse HTML
+ soup = BeautifulSoup(chapter_html, 'html.parser')
+
+ # Try to find natural break points
+ chunks = []
+ current_chunk = []
+ current_tokens = 0
+
+ # Get all direct children of body, or all top-level elements
+ if soup.body:
+ elements = list(soup.body.children)
+ else:
+ elements = list(soup.children)
+
+ for element in elements:
+ if isinstance(element, str) and element.strip() == '':
+ continue
+
+ element_html = str(element)
+ element_tokens = self.count_tokens(element_html)
+
+ # If single element is too large, try to split it
+ if element_tokens > effective_max_tokens:
+ sub_chunks = self._split_large_element(element, effective_max_tokens)
+ for sub_chunk in sub_chunks:
+ chunks.append(sub_chunk)
+ else:
+ # Check if adding this element would exceed limit
+ if current_tokens + element_tokens > effective_max_tokens and current_chunk:
+ # Save current chunk
+ chunks.append(self._create_chunk_html(current_chunk))
+ current_chunk = [element_html]
+ current_tokens = element_tokens
+ else:
+ current_chunk.append(element_html)
+ current_tokens += element_tokens
+
+ # Don't forget the last chunk
+ if current_chunk:
+ chunks.append(self._create_chunk_html(current_chunk))
+
+ # Return chunks with metadata
+ total_chunks = len(chunks)
+ return [(chunk, i+1, total_chunks) for i, chunk in enumerate(chunks)]
+
+ def _split_large_element(self, element, max_tokens):
+ """Split a single large element (like a long paragraph)"""
+ chunks = []
+
+ if element.name == 'p' or not hasattr(element, 'children'):
+ # For paragraphs or text elements, split by sentences
+ text = element.get_text()
+ sentences = re.split(r'(?<=[.!?])\s+', text)
+
+ current_chunk = []
+ current_tokens = 0
+
+ for sentence in sentences:
+ sentence_tokens = self.count_tokens(sentence)
+
+ if current_tokens + sentence_tokens > max_tokens * 0.8 and current_chunk:
+ # Create paragraph with current sentences
+ chunk_text = ' '.join(current_chunk)
+ chunks.append(f"{chunk_text}
")
+ current_chunk = [sentence]
+ current_tokens = sentence_tokens
+ else:
+ current_chunk.append(sentence)
+ current_tokens += sentence_tokens
+
+ if current_chunk:
+ chunk_text = ' '.join(current_chunk)
+ chunks.append(f"{chunk_text}
")
+
+ else:
+ # For other elements, try to split by children
+ children = list(element.children)
+ current_chunk = []
+ current_tokens = 0
+
+ for child in children:
+ child_html = str(child)
+ child_tokens = self.count_tokens(child_html)
+
+ if current_tokens + child_tokens > max_tokens * 0.8 and current_chunk:
+ # Wrap in parent element type
+ wrapper = BeautifulSoup(f"<{element.name}>", 'html.parser')
+ wrapper_elem = wrapper.find(element.name)
+ for item in current_chunk:
+ wrapper_elem.append(BeautifulSoup(item, 'html.parser'))
+ chunks.append(str(wrapper))
+
+ current_chunk = [child_html]
+ current_tokens = child_tokens
+ else:
+ current_chunk.append(child_html)
+ current_tokens += child_tokens
+
+ if current_chunk:
+ wrapper = BeautifulSoup(f"<{element.name}>", 'html.parser')
+ wrapper_elem = wrapper.find(element.name)
+ for item in current_chunk:
+ wrapper_elem.append(BeautifulSoup(item, 'html.parser'))
+ chunks.append(str(wrapper))
+
+ return chunks
+
+ def _create_chunk_html(self, elements):
+ """Create a valid HTML chunk from list of elements"""
+ # Join elements and wrap in basic HTML structure if needed
+ content = '\n'.join(elements)
+
+ # Check if it already has body tags
+ if ' 1:
+ # Keep first body, move all content from others into it
+ main_body = bodies[0]
+ for extra_body in bodies[1:]:
+ for child in list(extra_body.children):
+ main_body.append(child)
+ extra_body.decompose()
+
+ return str(soup)
+
+ return merged
diff --git a/check_epub_directory.py b/check_epub_directory.py
new file mode 100644
index 0000000000000000000000000000000000000000..907074ed108151562ec7910a307a095ff11cab43
--- /dev/null
+++ b/check_epub_directory.py
@@ -0,0 +1,152 @@
+import os
+import re
+
+def diagnose_epub_directory(directory="."):
+ """Diagnose issues with EPUB output directory"""
+
+ print(f"\n{'='*60}")
+ print(f"EPUB Directory Diagnostic Tool")
+ print(f"{'='*60}\n")
+
+ # Get absolute path
+ abs_path = os.path.abspath(directory)
+ print(f"📁 Checking directory: {abs_path}")
+
+ # Check if directory exists
+ if not os.path.exists(abs_path):
+ print(f"❌ ERROR: Directory does not exist!")
+ return
+
+ if not os.path.isdir(abs_path):
+ print(f"❌ ERROR: Path is not a directory!")
+ return
+
+ # List contents
+ try:
+ contents = os.listdir(abs_path)
+ print(f"✅ Directory is accessible")
+ print(f"📊 Total items: {len(contents)}\n")
+ except Exception as e:
+ print(f"❌ ERROR: Cannot read directory: {e}")
+ return
+
+ # Categorize files
+ html_files = []
+ response_files = []
+ css_files = []
+ image_files = []
+ directories = []
+ other_files = []
+
+ for item in contents:
+ item_path = os.path.join(abs_path, item)
+
+ if os.path.isdir(item_path):
+ directories.append(item)
+ elif item.endswith('.html'):
+ html_files.append(item)
+ if item.startswith('response_'):
+ response_files.append(item)
+ elif item.endswith('.css'):
+ css_files.append(item)
+ elif item.lower().endswith(('.jpg', '.jpeg', '.png', '.gif', '.svg')):
+ image_files.append(item)
+ else:
+ other_files.append(item)
+
+ # Report findings
+ print("📋 Directory Contents Summary:")
+ print(f" • HTML files: {len(html_files)}")
+ print(f" • Response files (translated chapters): {len(response_files)}")
+ print(f" • CSS files: {len(css_files)}")
+ print(f" • Image files: {len(image_files)}")
+ print(f" • Subdirectories: {len(directories)}")
+ print(f" • Other files: {len(other_files)}")
+
+ # Check for required items
+ print(f"\n📍 Checking Required Items:")
+
+ # Check for metadata.json
+ if 'metadata.json' in contents:
+ print(" ✅ metadata.json found")
+ else:
+ print(" ❌ metadata.json NOT FOUND")
+
+ # Check for response files
+ if response_files:
+ print(f" ✅ {len(response_files)} translated chapter files found")
+
+ # Analyze chapter numbers
+ chapter_nums = []
+ for f in response_files:
+ m = re.match(r'response_(\d+)_', f)
+ if m:
+ chapter_nums.append(int(m.group(1)))
+
+ if chapter_nums:
+ chapter_nums.sort()
+ print(f" 📖 Chapter range: {min(chapter_nums)} to {max(chapter_nums)}")
+
+ # Check for missing chapters
+ expected = set(range(min(chapter_nums), max(chapter_nums) + 1))
+ actual = set(chapter_nums)
+ missing = expected - actual
+ if missing:
+ print(f" ⚠️ Missing chapters: {sorted(missing)}")
+ else:
+ print(" ❌ No response_*.html files found!")
+
+ if html_files:
+ print(f"\n 🔍 Found {len(html_files)} HTML files with different names:")
+ for i, f in enumerate(html_files[:5]):
+ print(f" {i+1}. {f}")
+ if len(html_files) > 5:
+ print(f" ... and {len(html_files) - 5} more")
+
+ # Check subdirectories
+ if directories:
+ print(f"\n📂 Subdirectories found:")
+ for d in directories:
+ print(f" • {d}/")
+
+ # Check contents of important subdirectories
+ if d in ['css', 'images', 'fonts']:
+ try:
+ sub_contents = os.listdir(os.path.join(abs_path, d))
+ print(f" Contains {len(sub_contents)} items")
+ except:
+ print(f" Cannot read contents")
+
+ # Sample file check
+ if response_files:
+ print(f"\n🔍 Checking a sample chapter file...")
+ sample_file = response_files[0]
+ sample_path = os.path.join(abs_path, sample_file)
+
+ try:
+ with open(sample_path, 'r', encoding='utf-8') as f:
+ content = f.read()
+ print(f" ✅ {sample_file} is readable")
+ print(f" 📏 File size: {len(content):,} characters")
+
+ # Check for basic HTML structure
+ if '' in content or ' 1:
+ diagnose_epub_directory(sys.argv[1])
+ else:
+ diagnose_epub_directory(".")
\ No newline at end of file
diff --git a/direct_imports.py b/direct_imports.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae1202fd39a7913499a6a056c629b50808b46302
--- /dev/null
+++ b/direct_imports.py
@@ -0,0 +1,38 @@
+import sys
+import os
+
+# Add the current directory to Python path so we can import our modules
+current_dir = os.path.dirname(os.path.abspath(__file__))
+if current_dir not in sys.path:
+ sys.path.insert(0, current_dir)
+
+# When running as executable, modules might be in _MEIPASS
+if hasattr(sys, '_MEIPASS'):
+ meipass_dir = sys._MEIPASS
+ if meipass_dir not in sys.path:
+ sys.path.insert(0, meipass_dir)
+
+# Now we can safely import our modules
+try:
+ from extract_glossary_from_epub import main as glossary_main
+except ImportError as e:
+ print(f"Failed to import glossary module: {e}")
+ glossary_main = None
+
+try:
+ from TransateKRtoEN import main as translation_main
+except ImportError as e:
+ print(f"Failed to import translation module: {e}")
+ translation_main = None
+
+try:
+ from epub_converter import fallback_compile_epub
+except ImportError as e:
+ print(f"Failed to import epub converter: {e}")
+ fallback_compile_epub = None
+
+try:
+ from scan_html_folder import scan_html_folder
+except ImportError as e:
+ print(f"Failed to import scanner: {e}")
+ scan_html_folder = None
diff --git a/enhanced_text_extractor.py b/enhanced_text_extractor.py
new file mode 100644
index 0000000000000000000000000000000000000000..31c02bdb17619366b59bf2fb0548a84d088ee7cf
--- /dev/null
+++ b/enhanced_text_extractor.py
@@ -0,0 +1,597 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Enhanced Text Extractor Module with CJK Support
+Provides superior text extraction from HTML with proper Unicode handling
+Optimized for Korean, Japanese, and Chinese content extraction
+"""
+
+import os
+import re
+import html
+import unicodedata
+from typing import Tuple, Optional
+import chardet
+
+# BEAUTIFUL SOUP IMPORT MONKEY FIX - Import BeautifulSoup BEFORE html2text
+# This prevents certain parser initialization issues
+try:
+ from bs4 import BeautifulSoup
+ # Force BeautifulSoup to initialize its parsers
+ _ = BeautifulSoup("", 'html.parser')
+except ImportError:
+ BeautifulSoup = None
+ raise ImportError("BeautifulSoup is required. Install with: pip install beautifulsoup4")
+
+# Now import html2text AFTER BeautifulSoup
+try:
+ import html2text
+except ImportError:
+ html2text = None
+ raise ImportError("html2text is required. Install with: pip install html2text")
+
+
+class EnhancedTextExtractor:
+ """Enhanced text extraction with proper Unicode and CJK handling"""
+
+ # Unicode preservation mappings
+ UNICODE_QUOTES = {
+ # Western quotes
+ '“': '\u201c', # Left double quotation mark
+ '”': '\u201d', # Right double quotation mark
+ '‘': '\u2018', # Left single quotation mark
+ '’': '\u2019', # Right single quotation mark
+ '"': '"', # Standard double quote
+ ''': "'", # Standard apostrophe
+
+ # CJK quotes and punctuation
+ '「': '「', # Japanese left corner bracket
+ '」': '」', # Japanese right corner bracket
+ '『': '『', # Japanese left white corner bracket
+ '』': '』', # Japanese right white corner bracket
+ '（': '（', # Fullwidth left parenthesis
+ '）': '）', # Fullwidth right parenthesis
+ '【': '【', # Left black lenticular bracket
+ '】': '】', # Right black lenticular bracket
+ '《': '《', # Left double angle bracket
+ '》': '》', # Right double angle bracket
+ '；': '；', # Fullwidth semicolon
+ '：': '：', # Fullwidth colon
+ '。': '。', # Ideographic full stop
+ '？': '？', # Fullwidth question mark
+ '！': '！', # Fullwidth exclamation mark
+ '、': '、', # Ideographic comma
+
+ # Numeric entities
+ '“': '\u201c', # Left double quote (numeric)
+ '”': '\u201d', # Right double quote (numeric)
+ '‘': '\u2018', # Left single quote (numeric)
+ '’': '\u2019', # Right single quote (numeric)
+
+ # Common CJK entities
+ '…': '…', # Horizontal ellipsis
+ '—': '—', # Em dash
+ '–': '–', # En dash
+ ' ': '\u00A0', # Non-breaking space
+ }
+
+ # CJK-specific punctuation to preserve
+ CJK_PUNCTUATION = {
+ '。', '、', '！', '？', '…', '—', '～', '・',
+ '「', '」', '『', '』', '（', '）', '【', '】',
+ '《', '》', '〈', '〉', '〔', '〕', '［', '］',
+ '：', '；', '"', '"', ''', ''',
+ '，', '．', '？', '！', '：', '；',
+ '"', '"', '‚', '„', '«', '»',
+ }
+
+ # Quote protection markers
+ QUOTE_MARKERS = {
+ '"': '␥', # Opening double quote marker
+ '"': '␦', # Closing double quote marker
+ '"': '␦', # Alternative closing quote
+ "'": '␣', # Opening single quote marker
+ "'": '', # Closing single quote marker
+ "'": '', # Alternative closing quote
+ }
+
+
+ def __init__(self, filtering_mode: str = "smart", preserve_structure: bool = True):
+ """Initialize the enhanced text extractor"""
+ if not html2text:
+ raise ImportError("html2text is required for enhanced extraction")
+
+ if not BeautifulSoup:
+ raise ImportError("BeautifulSoup is required for enhanced extraction")
+
+ self.filtering_mode = filtering_mode
+ self.preserve_structure = preserve_structure
+ self.h2t = None
+ self.detected_language = None
+
+ self._configure_html2text()
+
+ def _detect_encoding(self, content: bytes) -> str:
+ """Detect the encoding of the content"""
+ try:
+ # Try chardet detection
+ detected = chardet.detect(content)
+ if detected['confidence'] > 0.7:
+ return detected['encoding']
+ except Exception:
+ pass
+
+ # Try common CJK encodings in order
+ for encoding in ['utf-8', 'gb2312', 'gbk', 'gb18030', 'big5', 'shift_jis', 'euc-kr', 'euc-jp']:
+ try:
+ content.decode(encoding)
+ return encoding
+ except Exception:
+ continue
+
+ return 'utf-8' # Default fallback
+
+ def _detect_content_language(self, text: str) -> str:
+ """Detect the primary language of content"""
+ if not text:
+ return 'unknown'
+
+ # Take a sample of the text
+ sample = text[:5000]
+
+ # Count characters by script
+ korean_chars = sum(1 for char in sample if 0xAC00 <= ord(char) <= 0xD7AF)
+ japanese_kana = sum(1 for char in sample if (0x3040 <= ord(char) <= 0x309F) or (0x30A0 <= ord(char) <= 0x30FF))
+ chinese_chars = sum(1 for char in sample if 0x4E00 <= ord(char) <= 0x9FFF)
+ latin_chars = sum(1 for char in sample if 0x0041 <= ord(char) <= 0x007A)
+
+ # Determine primary language
+ if korean_chars > 50:
+ return 'korean'
+ elif japanese_kana > 20:
+ return 'japanese'
+ elif chinese_chars > 50 and japanese_kana < 10:
+ return 'chinese'
+ elif latin_chars > 100:
+ return 'english'
+ else:
+ return 'unknown'
+
+ def _configure_html2text(self):
+ """Configure html2text with optimal Unicode and CJK settings"""
+ self.h2t = html2text.HTML2Text()
+
+ # Core settings for Unicode preservation
+ self.h2t.unicode_snob = True
+ self.h2t.escape_snob = True
+ self.h2t.use_automatic_links = False
+
+ # Layout settings
+ self.h2t.body_width = 0
+ self.h2t.single_line_break = False
+
+ # Content filtering
+ self.h2t.ignore_links = False
+ self.h2t.ignore_images = False
+ self.h2t.ignore_anchors = False
+ self.h2t.skip_internal_links = False
+ self.h2t.ignore_tables = False
+
+ # Image handling - CRITICAL: Force html2text to preserve img tags as HTML
+ self.h2t.images_as_html = True # Keep images as ![]()
tags instead of ![]()
+ self.h2t.images_to_alt = False # Don't convert to alt text only
+ self.h2t.images_with_size = True # Include width/height attributes
+
+ # Additional settings
+ self.h2t.wrap_links = False
+ self.h2t.wrap_list_items = False
+ self.h2t.protect_links = True
+
+ # Structure preservation settings
+ if self.preserve_structure:
+ self.h2t.bypass_tables = False
+ self.h2t.ignore_emphasis = False
+ self.h2t.mark_code = True
+ self.h2t.ul_item_mark = '•'
+ else:
+ self.h2t.bypass_tables = True
+ self.h2t.ignore_emphasis = True
+ self.h2t.mark_code = False
+
+ def _decode_entities(self, text: str) -> str:
+ """Decode HTML entities to Unicode characters with CJK support"""
+ if not text:
+ return text
+
+ # First pass: Apply known CJK-aware replacements
+ for entity, unicode_char in self.UNICODE_QUOTES.items():
+ text = text.replace(entity, unicode_char)
+
+ # Second pass: standard HTML unescape
+ text = html.unescape(text)
+
+ # Third pass: handle numeric entities
+ def decode_decimal(match):
+ try:
+ code = int(match.group(1))
+ if code < 0x110000:
+ return chr(code)
+ except Exception:
+ pass
+ return match.group(0)
+
+ def decode_hex(match):
+ try:
+ code = int(match.group(1), 16)
+ if code < 0x110000:
+ return chr(code)
+ except Exception:
+ pass
+ return match.group(0)
+
+ text = re.sub(r'&#(\d+);?', decode_decimal, text)
+ text = re.sub(r'&#x([0-9a-fA-F]+);?', decode_hex, text)
+
+ # Fourth pass: handle special CJK entities
+ cjk_special_entities = {
+ '⟨': '〈', '⟩': '〉',
+ '⌈': '⌈', '⌉': '⌉',
+ '⌊': '⌊', '⌋': '⌋',
+ }
+
+ for entity, char in cjk_special_entities.items():
+ text = text.replace(entity, char)
+
+ return text
+
+ def _normalize_unicode(self, text: str) -> str:
+ """Normalize Unicode with CJK awareness"""
+ if self.detected_language in ['korean', 'japanese', 'chinese']:
+ return text
+ else:
+ return unicodedata.normalize('NFC', text)
+
+ def _protect_quotes(self, text: str) -> str:
+ """Protect quotes by replacing with special markers"""
+ for original, marker in self.QUOTE_MARKERS.items():
+ text = text.replace(original, marker)
+ return text
+
+ def _restore_quotes(self, text: str) -> str:
+ """Restore quotes from special markers"""
+ for original, marker in self.QUOTE_MARKERS.items():
+ text = text.replace(marker, original)
+ return text
+
+
+
+ def _preprocess_html_for_quotes(self, html_content: str) -> str:
+ """Pre-process HTML to protect quotes from conversion"""
+ def protect_quotes_in_text(match):
+ text = match.group(1)
+ return f'>{self._protect_quotes(text)}<'
+
+ # Apply to all text between tags
+ html_content = re.sub(r'>([^<]+)<', protect_quotes_in_text, html_content)
+ return html_content
+
+ def _protect_quotes_in_soup(self, soup: BeautifulSoup) -> None:
+ """Protect quotes in BeautifulSoup object before processing"""
+ for element in soup.find_all(string=True):
+ if element.parent.name not in ['script', 'style', 'noscript']:
+ original_text = str(element)
+ protected_text = self._protect_quotes(original_text)
+ element.replace_with(protected_text)
+
+ def _minimal_parser_fix(self, html_content: str) -> str:
+ """Apply minimal fixes only for parser errors"""
+ # Fix tags with ="" pattern
+ html_content = re.sub(r'<[^>]*?=\s*""\s*[^>]*?>', '', html_content)
+
+ # Fix malformed closing tags
+ html_content = re.sub(r'', r'', html_content)
+ html_content = re.sub(r'', '', html_content)
+ html_content = re.sub(r'', r'', html_content)
+
+ # Fix orphaned brackets
+ html_content = re.sub(r'<(?![a-zA-Z/!?])', '<', html_content)
+ html_content = re.sub(r'(?', '>', html_content)
+
+ # Fix unclosed tags at the end
+ if html_content.rstrip().endswith('<'):
+ html_content = html_content.rstrip()[:-1]
+
+ # Remove nested opening brackets
+ html_content = re.sub(r'<[^>]*?<[^>]*?>', '', html_content)
+
+ return html_content
+
+ def _clean_text_cjk_aware(self, text: str, preserve_structure: bool) -> str:
+ """Clean extracted text with CJK awareness"""
+ if not preserve_structure and self.detected_language not in ['korean', 'japanese', 'chinese']:
+ # Only do aggressive cleanup for non-CJK text
+ text = re.sub(r'^#+\s*', '', text, flags=re.MULTILINE)
+ text = re.sub(r'\*\*(.*?)\*\*', r'\1', text)
+ text = re.sub(r'\*(.*?)\*', r'\1', text)
+ text = re.sub(r'__(.*?)__', r'\1', text)
+ text = re.sub(r'_(.*?)_', r'\1', text)
+ text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', text)
+ text = re.sub(r'!\[([^\]]*)\]\([^)]+\)', '', text)
+ text = re.sub(r'`([^`]+)`', r'\1', text)
+ text = re.sub(r'```[^`]*```', '', text, flags=re.DOTALL)
+ text = re.sub(r'^[-*+]\s+', '', text, flags=re.MULTILINE)
+ text = re.sub(r'^\d+\.\s+', '', text, flags=re.MULTILINE)
+ text = re.sub(r'^>\s+', '', text, flags=re.MULTILINE)
+ text = re.sub(r'^[-_*]{3,}$', '', text, flags=re.MULTILINE)
+
+ # Clean whitespace
+ if self.detected_language in ['korean', 'japanese', 'chinese']:
+ text = re.sub(r'\n{3,}', '\n\n', text)
+ text = re.sub(r'[ ]{3,}', ' ', text)
+ else:
+ text = re.sub(r'\n{3,}', '\n\n', text)
+ text = re.sub(r' {2,}', ' ', text)
+
+ # Remove invisible characters
+ invisible_chars = ['\u200b', '\u200c', '\u200d', '\ufeff', '\u2060']
+ for char in invisible_chars:
+ text = text.replace(char, '')
+
+ return text.strip()
+
+ def _extract_title(self, soup: BeautifulSoup) -> Optional[str]:
+ """Extract chapter title from various sources"""
+ # Try title tag first
+ if soup.title and soup.title.string:
+ title = soup.title.string.strip()
+ title = self._decode_entities(title)
+ return title
+
+ # Try headers in order
+ for header_tag in ['h1', 'h2', 'h3', 'h4']:
+ headers = soup.find_all(header_tag)
+ for header in headers:
+ title = header.get_text(strip=True)
+ if title:
+ title = self._decode_entities(title)
+ if self._is_chapter_title(title):
+ return title
+
+ return None
+
+ def _is_chapter_title(self, text: str) -> bool:
+ """Check if text looks like a chapter title"""
+ if not text or len(text) > 200:
+ return False
+
+ # Common chapter patterns
+ patterns = [
+ r'第.{1,10}[章回話话]',
+ r'Chapter\s+\d+',
+ r'제\s*\d+\s*화',
+ r'第\d+話',
+ r'\d+\s*화',
+ r'EP\.?\s*\d+',
+ r'Part\s+\d+',
+ ]
+
+ for pattern in patterns:
+ if re.search(pattern, text, re.IGNORECASE):
+ return True
+
+ # Check if it's short and doesn't contain too much punctuation
+ if len(text) < 100:
+ punct_count = sum(1 for c in text if c in '.,;:!?。、！？')
+ if punct_count < len(text) * 0.2:
+ return True
+
+ return False
+
+ def _extract_body_content(self, soup: BeautifulSoup, full_html: str) -> str:
+ """Extract body content while preserving Unicode"""
+ # Remove script and style elements first
+ for element in soup(['script', 'style', 'noscript']):
+ element.decompose()
+
+ if soup.body:
+ return str(soup.body)
+ else:
+ return str(soup)
+
+ def extract_chapter_content(self, html_content: str, extraction_mode: str = None) -> Tuple[str, str, Optional[str]]:
+ """Extract chapter content with proper Unicode and CJK handling"""
+ try:
+ # Use instance filtering_mode if not overridden
+ if extraction_mode is None:
+ extraction_mode = self.filtering_mode
+
+ # Handle encoding if content is bytes
+ if isinstance(html_content, bytes):
+ encoding = self._detect_encoding(html_content)
+ html_content = html_content.decode(encoding, errors='replace')
+
+ # Pre-process HTML to protect quotes
+ html_content = self._preprocess_html_for_quotes(html_content)
+
+ # Pre-process HTML to decode all entities
+ html_content = self._decode_entities(html_content)
+
+ # Detect language early
+ self.detected_language = self._detect_content_language(html_content)
+ print(f"🌐 Detected language: {self.detected_language}")
+
+ # Parse with BeautifulSoup
+ parser = 'html.parser'
+ if self.detected_language in ['korean', 'japanese', 'chinese']:
+ # For CJK content, lxml might handle encoding better if available
+ try:
+ import lxml
+ parser = 'lxml'
+ except ImportError:
+ pass
+
+ soup = BeautifulSoup(html_content, parser)
+
+ # Protect quotes before any processing
+ self._protect_quotes_in_soup(soup)
+
+ # Extract title
+ chapter_title = self._extract_title(soup)
+
+ # Respect GUI toggles to exclude headers/titles BEFORE conversion
+ try:
+ batch_translate_active = os.getenv('BATCH_TRANSLATE_HEADERS', '0') == '1'
+ ignore_title_tag = os.getenv('IGNORE_TITLE', '0') == '1' and batch_translate_active
+ ignore_header_tags = os.getenv('IGNORE_HEADER', '0') == '1' and batch_translate_active
+ if ignore_title_tag and soup.title:
+ # Remove
so it isn't included when using full extraction
+ soup.title.decompose()
+ if ignore_header_tags:
+ # Remove visible headers from body prior to conversion
+ for tag_name in ['h1', 'h2', 'h3']:
+ for hdr in soup.find_all(tag_name):
+ hdr.decompose()
+ except Exception:
+ # Non-fatal – proceed with original soup if anything goes wrong
+ pass
+
+ # Determine content to convert (after removals)
+ if extraction_mode == "full":
+ content_to_convert = str(soup)
+ else:
+ content_to_convert = self._extract_body_content(soup, html_content)
+
+ # Convert using html2text
+ content_to_convert = self._decode_entities(content_to_convert)
+
+ # Convert to text with error handling
+ try:
+ clean_text = self.h2t.handle(content_to_convert)
+ except (AssertionError, UnboundLocalError) as e:
+ error_msg = str(e)
+ if "cannot access local variable" in error_msg or "we should not get here!" in error_msg or "unexpected call to parse_endtag" in error_msg or "unexpected call to parse_starttag" in error_msg:
+ print(f"⚠️ html2text encountered malformed HTML: {error_msg}")
+ print(f"⚠️ Applying minimal fixes...")
+ # Apply minimal fixes
+ content_to_convert = self._minimal_parser_fix(content_to_convert)
+ try:
+ clean_text = self.h2t.handle(content_to_convert)
+ print(f"✅ Successfully processed after minimal fixes")
+ except Exception as e2:
+ print(f"⚠️ html2text still failing: {e2}")
+ # Last resort fallback
+ clean_text = soup.get_text(separator='\n', strip=True)
+ print(f"✅ Used BeautifulSoup fallback")
+ else:
+ # Re-raise if it's a different error
+ raise
+ except Exception as e:
+ print(f"⚠️ Unexpected error in html2text: {e}")
+ # Fallback to BeautifulSoup
+ clean_text = soup.get_text(separator='\n', strip=True)
+
+ # Normalize only if appropriate
+ clean_text = self._normalize_unicode(clean_text)
+
+ # Clean based on settings and language
+ clean_text = self._clean_text_cjk_aware(clean_text, self.preserve_structure)
+
+ # Restore protected quotes
+ clean_text = self._restore_quotes(clean_text)
+
+ # For enhanced mode, both display and translation content are the same
+ return clean_text, clean_text, chapter_title
+
+ except Exception as e:
+ print(f"❌ Enhanced extraction failed: {e}")
+ raise
+
+
+# Test function
+def test_cjk_preservation():
+ """Test that CJK characters and quotes are properly preserved"""
+ test_cases = [
+ # Korean test with quotes
+ '''<html>
+ <head><title>제국의 붉은 사신
+
+ "왜 이러는 겁니까? 우리가 무슨 잘못을 했다고!"
+ "......"
+ "한 번만 살려주시오! 가족을 지키려면 어쩔 수 없었소!"
+ "응애! 응애! 응애!"
+ "미안하구나. 모든 죄는 내가 짊어지고 사마."
+
+
' in translation_result:
+ trans_soup = BeautifulSoup(translation_result, 'html.parser')
+
+ # Try to get the full container first
+ full_container = trans_soup.find('div', class_=['translated-text-only', 'image-with-translation'])
+
+ if full_container:
+ # Clone the container to avoid issues
+ new_container = BeautifulSoup(str(full_container), 'html.parser').find('div')
+ img_tag.replace_with(new_container)
+ print(f" ✅ Replaced image with full translation container")
+ else:
+ # Fallback: manually build the structure
+ trans_div = trans_soup.find('div', class_='image-translation')
+ if trans_div:
+ container = soup.new_tag('div', **{'class': 'translated-text-only' if hide_label else 'image-with-translation'})
+ img_tag.replace_with(container)
+
+ if not hide_label:
+ new_img = soup.new_tag('img', src=img_src)
+ if img_info.get('alt'):
+ new_img['alt'] = img_info.get('alt')
+ container.append(new_img)
+
+ # Clone the translation div content
+ new_trans_div = soup.new_tag('div', **{'class': 'image-translation'})
+ # Copy all children from trans_div to new_trans_div
+ for child in trans_div.children:
+ if hasattr(child, 'name'):
+ new_trans_div.append(BeautifulSoup(str(child), 'html.parser'))
+ else:
+ new_trans_div.append(str(child))
+
+ container.append(new_trans_div)
+ print(f" ✅ Built container with translation div")
+ else:
+ print(f" ⚠️ No translation div found in result")
+ continue
+ else:
+ # Plain text translation - build structure manually
+ container = soup.new_tag('div', **{'class': 'translated-text-only' if hide_label else 'image-with-translation'})
+ img_tag.replace_with(container)
+
+ if not hide_label:
+ new_img = soup.new_tag('img', src=img_src)
+ if img_info.get('alt'):
+ new_img['alt'] = img_info.get('alt')
+ container.append(new_img)
+
+ # Create translation div with content
+ translation_div = soup.new_tag('div', **{'class': 'image-translation'})
+ if not hide_label:
+ label_p = soup.new_tag('p')
+ label_em = soup.new_tag('em')
+ #label_em.string = "[Image text translation:]"
+ label_p.append(label_em)
+ translation_div.append(label_p)
+
+ trans_p = soup.new_tag('p')
+ trans_p.string = translation_result
+ translation_div.append(trans_p)
+ container.append(translation_div)
+ print(f" ✅ Created plain text translation structure")
+
+ translated_count += 1
+
+ # Save to translated_images folder
+ trans_filename = f"ch{actual_num:03d}_img{idx:02d}_translation.html"
+ trans_filepath = os.path.join(image_translator.translated_images_dir, trans_filename)
+
+ # Extract just the translation content for saving
+ save_soup = BeautifulSoup(translation_result, 'html.parser')
+ save_div = save_soup.find('div', class_='image-translation')
+ if not save_div:
+ # Create a simple div for plain text
+ save_div = f''
+
+ with open(trans_filepath, 'w', encoding='utf-8') as f:
+ f.write(f"""
+
+
+
+ Chapter {actual_num} - Image {idx} Translation
+
+
+
+ {save_div} + +""") + + print(f" ✅ Saved translation to: {trans_filename}") + else: + print(f" ⚠️ Could not find image tag in HTML for: {img_src}") + + if translated_count > 0: + print(f" 🖼️ Successfully translated {translated_count} images") + + # Debug output + final_html = str(soup) + trans_count = final_html.count('
{translation_result}
Chapter {actual_num} - Image {idx}
+Original: {os.path.basename(img_path)}
++ {save_div} + +""") + + print(f" ✅ Saved translation to: {trans_filename}") + else: + print(f" ⚠️ Could not find image tag in HTML for: {img_src}") + + if translated_count > 0: + print(f" 🖼️ Successfully translated {translated_count} images") + + # Debug output + final_html = str(soup) + trans_count = final_html.count('
')
+ print(f" 📊 Final HTML has {trans_count} translation divs")
+ print(f" 📊 image_translations dict has {len(image_translations)} entries")
+
+ prog = image_translator.load_progress()
+ if "image_chunks" in prog:
+ completed_images = []
+ for img_key, img_data in prog["image_chunks"].items():
+ if len(img_data["completed"]) == img_data["total"]:
+ completed_images.append(img_key)
+
+ for img_key in completed_images:
+ del prog["image_chunks"][img_key]
+
+ if completed_images:
+ image_translator.save_progress(prog)
+ print(f" 🧹 Cleaned up progress for {len(completed_images)} completed images")
+
+ image_translator.save_translation_log(actual_num, image_translations)
+
+ return str(soup), image_translations
+ else:
+ print(f" ℹ️ No images were successfully translated")
+
+ return chapter_html, {}
+
+def detect_novel_numbering(chapters):
+ """Detect if the novel uses 0-based or 1-based chapter numbering with improved accuracy"""
+ print("[DEBUG] Detecting novel numbering system...")
+
+ if not chapters:
+ return False
+
+ if isinstance(chapters[0], str):
+ print("[DEBUG] Text file detected, skipping numbering detection")
+ return False
+
+ patterns = PatternManager.FILENAME_EXTRACT_PATTERNS
+
+ # Special check for prefix_suffix pattern like "0000_1.xhtml"
+ prefix_suffix_pattern = r'^(\d+)_(\d+)[_\.]'
+
+ # Track chapter numbers from different sources
+ filename_numbers = []
+ content_numbers = []
+ has_prefix_suffix = False
+ prefix_suffix_numbers = []
+
+ for idx, chapter in enumerate(chapters):
+ extracted_num = None
+
+ # Check filename patterns
+ if 'original_basename' in chapter and chapter['original_basename']:
+ filename = chapter['original_basename']
+ elif 'filename' in chapter:
+ filename = os.path.basename(chapter['filename'])
+ else:
+ continue
+
+ # First check for prefix_suffix pattern
+ prefix_match = re.search(prefix_suffix_pattern, filename, re.IGNORECASE)
+ if prefix_match:
+ has_prefix_suffix = True
+ # Use the SECOND number (after underscore)
+ suffix_num = int(prefix_match.group(2))
+ prefix_suffix_numbers.append(suffix_num)
+ extracted_num = suffix_num
+ print(f"[DEBUG] Prefix_suffix pattern matched: {filename} -> Chapter {suffix_num}")
+ else:
+ # Try other patterns
+ for pattern in patterns:
+ match = re.search(pattern, filename)
+ if match:
+ extracted_num = int(match.group(1))
+ #print(f"[DEBUG] Pattern '{pattern}' matched: {filename} -> Chapter {extracted_num}")
+ break
+
+ if extracted_num is not None:
+ filename_numbers.append(extracted_num)
+
+ # Also check chapter content for chapter declarations
+ if 'body' in chapter:
+ # Look for "Chapter N" in the first 1000 characters
+ content_preview = chapter['body'][:1000]
+ content_match = re.search(r'Chapter\s+(\d+)', content_preview, re.IGNORECASE)
+ if content_match:
+ content_num = int(content_match.group(1))
+ content_numbers.append(content_num)
+ print(f"[DEBUG] Found 'Chapter {content_num}' in content")
+
+ # Decision logic with improved heuristics
+
+ # 1. If using prefix_suffix pattern, trust those numbers exclusively
+ if has_prefix_suffix and prefix_suffix_numbers:
+ min_suffix = min(prefix_suffix_numbers)
+ if min_suffix >= 1:
+ print(f"[DEBUG] ✅ 1-based novel detected (prefix_suffix pattern starts at {min_suffix})")
+ return False
+ else:
+ print(f"[DEBUG] ✅ 0-based novel detected (prefix_suffix pattern starts at {min_suffix})")
+ return True
+
+ # 2. If we have content numbers, prefer those over filename numbers
+ if content_numbers:
+ min_content = min(content_numbers)
+ # Check if we have a good sequence starting from 0 or 1
+ if 0 in content_numbers and 1 in content_numbers:
+ print(f"[DEBUG] ✅ 0-based novel detected (found both Chapter 0 and Chapter 1 in content)")
+ return True
+ elif min_content == 1:
+ print(f"[DEBUG] ✅ 1-based novel detected (content chapters start at 1)")
+ return False
+
+ # 3. Fall back to filename numbers
+ if filename_numbers:
+ min_filename = min(filename_numbers)
+ max_filename = max(filename_numbers)
+
+ # Check for a proper sequence
+ # If we have 0,1,2,3... it's likely 0-based
+ # If we have 1,2,3,4... it's likely 1-based
+
+ # Count how many chapters we have in sequence starting from 0
+ zero_sequence_count = 0
+ for i in range(len(chapters)):
+ if i in filename_numbers:
+ zero_sequence_count += 1
+ else:
+ break
+
+ # Count how many chapters we have in sequence starting from 1
+ one_sequence_count = 0
+ for i in range(1, len(chapters) + 1):
+ if i in filename_numbers:
+ one_sequence_count += 1
+ else:
+ break
+
+ print(f"[DEBUG] Zero-based sequence length: {zero_sequence_count}")
+ print(f"[DEBUG] One-based sequence length: {one_sequence_count}")
+
+ # If we have a better sequence starting from 1, it's 1-based
+ if one_sequence_count > zero_sequence_count and min_filename >= 1:
+ print(f"[DEBUG] ✅ 1-based novel detected (better sequence match starting from 1)")
+ return False
+
+ # If we have any 0 in filenames and it's part of a sequence
+ if 0 in filename_numbers and zero_sequence_count >= 3:
+ print(f"[DEBUG] ✅ 0-based novel detected (found 0 in sequence)")
+ return True
+
+ # 4. Default to 1-based if uncertain
+ print(f"[DEBUG] ✅ Defaulting to 1-based novel (insufficient evidence for 0-based)")
+ return False
+
+def validate_chapter_continuity(chapters):
+ """Validate chapter continuity and warn about issues"""
+ if not chapters:
+ print("No chapters to translate")
+ return
+
+ issues = []
+
+ # Get all chapter numbers
+ chapter_nums = [c['num'] for c in chapters]
+ actual_nums = [c.get('actual_chapter_num', c['num']) for c in chapters]
+
+ # Check for duplicates
+ duplicates = [num for num in chapter_nums if chapter_nums.count(num) > 1]
+ if duplicates:
+ issues.append(f"Duplicate chapter numbers found: {set(duplicates)}")
+
+ # Check for gaps in sequence
+ min_num = min(chapter_nums)
+ max_num = max(chapter_nums)
+ expected = set(range(min_num, max_num + 1))
+ actual = set(chapter_nums)
+ missing = expected - actual
+
+ if missing:
+ issues.append(f"Missing chapter numbers: {sorted(missing)}")
+ # Show gaps more clearly
+ gaps = []
+ sorted_missing = sorted(missing)
+ if sorted_missing:
+ start = sorted_missing[0]
+ end = sorted_missing[0]
+ for num in sorted_missing[1:]:
+ if num == end + 1:
+ end = num
+ else:
+ gaps.append(f"{start}-{end}" if start != end else str(start))
+ start = end = num
+ gaps.append(f"{start}-{end}" if start != end else str(start))
+ issues.append(f"Gap ranges: {', '.join(gaps)}")
+
+ # Check for duplicate titles
+ title_map = {}
+ for c in chapters:
+ title_lower = c['title'].lower().strip()
+ if title_lower in title_map:
+ title_map[title_lower].append(c['num'])
+ else:
+ title_map[title_lower] = [c['num']]
+
+ for title, nums in title_map.items():
+ if len(nums) > 1:
+ issues.append(f"Duplicate title '{title}' in chapters: {nums}")
+
+ # Print summary
+ print("\n" + "="*60)
+ print("📚 CHAPTER VALIDATION SUMMARY")
+ print("="*60)
+ print(f"Total chapters: {len(chapters)}")
+ print(f"Chapter range: {min_num} to {max_num}")
+ print(f"Expected count: {max_num - min_num + 1}")
+ print(f"Actual count: {len(chapters)}")
+
+ if len(chapters) != (max_num - min_num + 1):
+ print(f"⚠️ Chapter count mismatch - missing {(max_num - min_num + 1) - len(chapters)} chapters")
+
+ if issues:
+ print("\n⚠️ Issues found:")
+ for issue in issues:
+ print(f" - {issue}")
+ else:
+ print("✅ No continuity issues detected")
+
+ print("="*60 + "\n")
+
+def validate_epub_structure(output_dir):
+ """Validate that all necessary EPUB structure files are present"""
+ print("🔍 Validating EPUB structure...")
+
+ required_files = {
+ 'container.xml': 'META-INF container file (critical)',
+ '*.opf': 'OPF package file (critical)',
+ '*.ncx': 'Navigation file (recommended)'
+ }
+
+ found_files = {}
+ missing_files = []
+
+ container_path = os.path.join(output_dir, 'container.xml')
+ if os.path.exists(container_path):
+ found_files['container.xml'] = 'Found'
+ print(" ✅ container.xml - Found")
+ else:
+ missing_files.append('container.xml')
+ print(" ❌ container.xml - Missing (CRITICAL)")
+
+ opf_files = []
+ ncx_files = []
+
+ for file in os.listdir(output_dir):
+ if file.lower().endswith('.opf'):
+ opf_files.append(file)
+ elif file.lower().endswith('.ncx'):
+ ncx_files.append(file)
+
+ if opf_files:
+ found_files['opf'] = opf_files
+ print(f" ✅ OPF file(s) - Found: {', '.join(opf_files)}")
+ else:
+ missing_files.append('*.opf')
+ print(" ❌ OPF file - Missing (CRITICAL)")
+
+ if ncx_files:
+ found_files['ncx'] = ncx_files
+ print(f" ✅ NCX file(s) - Found: {', '.join(ncx_files)}")
+ else:
+ missing_files.append('*.ncx')
+ print(" ⚠️ NCX file - Missing (navigation may not work)")
+
+ html_files = [f for f in os.listdir(output_dir) if f.lower().endswith('.html') and f.startswith('response_')]
+ if html_files:
+ print(f" ✅ Translated chapters - Found: {len(html_files)} files")
+ else:
+ print(" ⚠️ No translated chapter files found")
+
+ critical_missing = [f for f in missing_files if f in ['container.xml', '*.opf']]
+
+ if not critical_missing:
+ print("✅ EPUB structure validation PASSED")
+ print(" All critical files present for EPUB reconstruction")
+ return True
+ else:
+ print("❌ EPUB structure validation FAILED")
+ print(f" Missing critical files: {', '.join(critical_missing)}")
+ print(" EPUB reconstruction may fail without these files")
+ return False
+
+def check_epub_readiness(output_dir):
+ """Check if the output directory is ready for EPUB compilation"""
+ print("📋 Checking EPUB compilation readiness...")
+
+ issues = []
+
+ if not validate_epub_structure(output_dir):
+ issues.append("Missing critical EPUB structure files")
+
+ html_files = [f for f in os.listdir(output_dir) if f.lower().endswith('.html') and f.startswith('response_')]
+ if not html_files:
+ issues.append("No translated chapter files found")
+ else:
+ print(f" ✅ Found {len(html_files)} translated chapters")
+
+ metadata_path = os.path.join(output_dir, 'metadata.json')
+ if os.path.exists(metadata_path):
+ print(" ✅ Metadata file present")
+ try:
+ with open(metadata_path, 'r', encoding='utf-8') as f:
+ metadata = json.load(f)
+ if 'title' not in metadata:
+ issues.append("Metadata missing title")
+ except Exception as e:
+ issues.append(f"Metadata file corrupted: {e}")
+ else:
+ issues.append("Missing metadata.json file")
+
+ resource_dirs = ['css', 'fonts', 'images']
+ found_resources = 0
+ for res_dir in resource_dirs:
+ res_path = os.path.join(output_dir, res_dir)
+ if os.path.exists(res_path):
+ files = [f for f in os.listdir(res_path) if os.path.isfile(os.path.join(res_path, f))]
+ if files:
+ found_resources += len(files)
+ print(f" ✅ Found {len(files)} {res_dir} files")
+
+ if found_resources > 0:
+ print(f" ✅ Total resources: {found_resources} files")
+ else:
+ print(" ⚠️ No resource files found (this may be normal)")
+
+ if not issues:
+ print("🎉 EPUB compilation readiness: READY")
+ print(" All necessary files present for EPUB creation")
+ return True
+ else:
+ print("⚠️ EPUB compilation readiness: ISSUES FOUND")
+ for issue in issues:
+ print(f" • {issue}")
+ return False
+
+def cleanup_previous_extraction(output_dir):
+ """Clean up any files from previous extraction runs (preserves CSS files)"""
+ # Remove 'css' from cleanup_items to preserve CSS files
+ cleanup_items = [
+ 'images', # Removed 'css' from this list
+ '.resources_extracted'
+ ]
+
+ epub_structure_files = [
+ 'container.xml', 'content.opf', 'toc.ncx'
+ ]
+
+ cleaned_count = 0
+
+ # Clean up directories (except CSS)
+ for item in cleanup_items:
+ if item.startswith('.'):
+ continue
+ item_path = os.path.join(output_dir, item)
+ try:
+ if os.path.isdir(item_path):
+ shutil.rmtree(item_path)
+ print(f"🧹 Removed directory: {item}")
+ cleaned_count += 1
+ except Exception as e:
+ print(f"⚠️ Could not remove directory {item}: {e}")
+
+ # Clean up EPUB structure files
+ for epub_file in epub_structure_files:
+ file_path = os.path.join(output_dir, epub_file)
+ try:
+ if os.path.isfile(file_path):
+ os.remove(file_path)
+ print(f"🧹 Removed EPUB file: {epub_file}")
+ cleaned_count += 1
+ except Exception as e:
+ print(f"⚠️ Could not remove {epub_file}: {e}")
+
+ # Clean up any loose .opf and .ncx files
+ try:
+ for file in os.listdir(output_dir):
+ if file.lower().endswith(('.opf', '.ncx')):
+ file_path = os.path.join(output_dir, file)
+ if os.path.isfile(file_path):
+ os.remove(file_path)
+ print(f"🧹 Removed EPUB file: {file}")
+ cleaned_count += 1
+ except Exception as e:
+ print(f"⚠️ Error scanning for EPUB files: {e}")
+
+ # Remove extraction marker
+ marker_path = os.path.join(output_dir, '.resources_extracted')
+ try:
+ if os.path.isfile(marker_path):
+ os.remove(marker_path)
+ print(f"🧹 Removed extraction marker")
+ cleaned_count += 1
+ except Exception as e:
+ print(f"⚠️ Could not remove extraction marker: {e}")
+
+ # Check if CSS files exist and inform user they're being preserved
+ css_path = os.path.join(output_dir, 'css')
+ if os.path.exists(css_path):
+ try:
+ css_files = [f for f in os.listdir(css_path) if os.path.isfile(os.path.join(css_path, f))]
+ if css_files:
+ print(f"📚 Preserving {len(css_files)} CSS files")
+ except Exception:
+ pass
+
+ if cleaned_count > 0:
+ print(f"🧹 Cleaned up {cleaned_count} items from previous runs (CSS files preserved)")
+
+ return cleaned_count
+
+# =====================================================
+# API AND TRANSLATION UTILITIES
+# =====================================================
+def send_with_interrupt(messages, client, temperature, max_tokens, stop_check_fn, chunk_timeout=None, request_id=None, context=None):
+ """Send API request with interrupt capability and optional timeout retry.
+ Optional context parameter is passed through to the client to improve payload labeling.
+ """
+ # Import UnifiedClientError at function level to avoid scoping issues
+ from unified_api_client import UnifiedClientError
+
+ # The client.send() call will handle multi-key rotation automatically
+
+ # Generate request_id if not provided
+ #if request_id is None:
+ # request_id = str(uuid.uuid4())[:8]
+
+ result_queue = queue.Queue()
+
+ def api_call():
+ try:
+ start_time = time.time()
+
+ # Check if client.send accepts request_id parameter
+ send_params = {
+ 'messages': messages,
+ 'temperature': temperature,
+ 'max_tokens': max_tokens
+ }
+ # Add context if supported
+ sig = inspect.signature(client.send)
+ if 'context' in sig.parameters and context is not None:
+ send_params['context'] = context
+
+ # Add request_id if the client supports it
+ sig = inspect.signature(client.send)
+ #if 'request_id' in sig.parameters:
+ # send_params['request_id'] = request_id
+
+ result = client.send(**send_params)
+ elapsed = time.time() - start_time
+ result_queue.put((result, elapsed))
+ except Exception as e:
+ result_queue.put(e)
+
+ api_thread = threading.Thread(target=api_call)
+ api_thread.daemon = True
+ api_thread.start()
+
+ timeout = chunk_timeout if chunk_timeout is not None else 86400
+ check_interval = 0.5
+ elapsed = 0
+
+ while elapsed < timeout:
+ try:
+ result = result_queue.get(timeout=check_interval)
+ if isinstance(result, Exception):
+ # For expected errors like rate limits, preserve the error type without extra traceback
+ if hasattr(result, 'error_type') and result.error_type == "rate_limit":
+ raise result
+ elif "429" in str(result) or "rate limit" in str(result).lower():
+ # Convert generic exceptions to UnifiedClientError for rate limits
+ raise UnifiedClientError(str(result), error_type="rate_limit")
+ else:
+ raise result
+ if isinstance(result, tuple):
+ api_result, api_time = result
+ if chunk_timeout and api_time > chunk_timeout:
+ # Set cleanup flag when chunk timeout occurs
+ if hasattr(client, '_in_cleanup'):
+ client._in_cleanup = True
+ if hasattr(client, 'cancel_current_operation'):
+ client.cancel_current_operation()
+ raise UnifiedClientError(f"API call took {api_time:.1f}s (timeout: {chunk_timeout}s)")
+ return api_result
+ return result
+ except queue.Empty:
+ if stop_check_fn():
+ # Set cleanup flag when user stops
+ if hasattr(client, '_in_cleanup'):
+ client._in_cleanup = True
+ if hasattr(client, 'cancel_current_operation'):
+ client.cancel_current_operation()
+ raise UnifiedClientError("Translation stopped by user")
+ elapsed += check_interval
+
+ # Set cleanup flag when timeout occurs
+ if hasattr(client, '_in_cleanup'):
+ client._in_cleanup = True
+ if hasattr(client, 'cancel_current_operation'):
+ client.cancel_current_operation()
+ raise UnifiedClientError(f"API call timed out after {timeout} seconds")
+
+def handle_api_error(processor, error, chunk_info=""):
+ """Handle API errors with multi-key support"""
+ error_str = str(error)
+
+ # Check for rate limit
+ if "429" in error_str or "rate limit" in error_str.lower():
+ if processor.config.use_multi_api_keys:
+ print(f"⚠️ Rate limit hit {chunk_info}, client should rotate to next key")
+ stats = processor.client.get_stats()
+ print(f"📊 API Stats - Active keys: {stats.get('active_keys', 0)}/{stats.get('total_keys', 0)}")
+
+ if stats.get('active_keys', 0) == 0:
+ print("⏳ All API keys are cooling down - will wait and retry")
+ print(f"🔄 Multi-key error handling: Rate limit processed, preparing for key rotation...")
+ time.sleep(0.1) # Brief pause after rate limit detection for stability
+ return True # Always retry
+ else:
+ print(f"⚠️ Rate limit hit {chunk_info}, waiting before retry...")
+ time.sleep(60)
+ print(f"🔄 Single-key error handling: Rate limit wait completed, ready for retry...")
+ time.sleep(0.1) # Brief pause after rate limit wait for stability
+ return True # Always retry
+
+ # Other errors
+ print(f"❌ API Error {chunk_info}: {error_str}")
+ return False
+
+def parse_token_limit(env_value):
+ """Parse token limit from environment variable"""
+ if not env_value or env_value.strip() == "":
+ return None, "unlimited"
+
+ env_value = env_value.strip()
+ if env_value.lower() == "unlimited":
+ return None, "unlimited"
+
+ if env_value.isdigit() and int(env_value) > 0:
+ limit = int(env_value)
+ return limit, str(limit)
+
+ return 1000000, "1000000 (default)"
+
+def build_system_prompt(user_prompt, glossary_path=None):
+ """Build the system prompt with glossary - TRUE BRUTE FORCE VERSION"""
+ append_glossary = os.getenv("APPEND_GLOSSARY", "1") == "1"
+ actual_glossary_path = glossary_path
+
+
+ system = user_prompt if user_prompt else ""
+
+ if append_glossary and actual_glossary_path and os.path.exists(actual_glossary_path):
+ try:
+ print(f"[DEBUG] ✅ Loading glossary from: {os.path.abspath(actual_glossary_path)}")
+
+ # Try to load as JSON first
+ try:
+ with open(actual_glossary_path, "r", encoding="utf-8") as gf:
+ glossary_data = json.load(gf)
+ glossary_text = json.dumps(glossary_data, ensure_ascii=False, indent=2)
+ print(f"[DEBUG] Loaded as JSON")
+ except json.JSONDecodeError:
+ # If JSON fails, just read as raw text
+ #print(f"[DEBUG] JSON parse failed, reading as raw text")
+ with open(actual_glossary_path, "r", encoding="utf-8") as gf:
+ glossary_text = gf.read()
+
+ if system:
+ system += "\n\n"
+
+ custom_prompt = os.getenv("APPEND_GLOSSARY_PROMPT", "Character/Term Glossary (use these translations consistently):").strip()
+ if not custom_prompt:
+ custom_prompt = "Character/Term Glossary (use these translations consistently):"
+
+ system += f"{custom_prompt}\n{glossary_text}"
+
+ print(f"[DEBUG] ✅ Entire glossary appended!")
+ print(f"[DEBUG] Glossary text length: {len(glossary_text)} characters")
+
+ except Exception as e:
+ print(f"[ERROR] Could not load glossary: {e}")
+ import traceback
+ print(f"[ERROR] Full traceback: {traceback.format_exc()}")
+ else:
+ if not append_glossary:
+ #print(f"[DEBUG] ❌ Glossary append disabled")
+ pass
+ elif not actual_glossary_path:
+ print(f"[DEBUG] ❌ No glossary path provided")
+ elif not os.path.exists(actual_glossary_path):
+ print(f"[DEBUG] ❌ Glossary file does not exist: {actual_glossary_path}")
+
+ print(f"🎯 Final system prompt length: {len(system)} characters")
+
+ return system
+
+def translate_title(title, client, system_prompt, user_prompt, temperature=0.3):
+ """Translate the book title using the configured settings"""
+ if not title or not title.strip():
+ return title
+
+ print(f"📚 Processing book title: {title}")
+
+ try:
+ if os.getenv("TRANSLATE_BOOK_TITLE", "1") == "0":
+ print(f"📚 Book title translation disabled - keeping original")
+ return title
+
+ # Check if we're using a translation service (not AI)
+ client_type = getattr(client, 'client_type', '')
+ is_translation_service = client_type in ['deepl', 'google_translate']
+
+ if is_translation_service:
+ # For translation services, send only the text without AI prompts
+ print(f"📚 Using translation service ({client_type}) - sending text directly")
+ messages = [
+ {"role": "user", "content": title}
+ ]
+ max_tokens = int(os.getenv("MAX_OUTPUT_TOKENS", "8192"))
+ translated_title, _ = client.send(messages, temperature=temperature, max_tokens=max_tokens)
+ else:
+ # For AI services, use prompts as before
+ book_title_prompt = os.getenv("BOOK_TITLE_PROMPT",
+ "Translate this book title to English while retaining any acronyms:")
+
+ # Get the system prompt for book titles, with fallback to default
+ book_title_system_prompt = os.getenv("BOOK_TITLE_SYSTEM_PROMPT",
+ "You are a translator. Respond with only the translated text, nothing else. Do not add any explanation or additional content.")
+
+ messages = [
+ {"role": "system", "content": book_title_system_prompt},
+ {"role": "user", "content": f"{book_title_prompt}\n\n{title}"}
+ ]
+ max_tokens = int(os.getenv("MAX_OUTPUT_TOKENS", "8192"))
+ translated_title, _ = client.send(messages, temperature=temperature, max_tokens=max_tokens)
+
+ print(f"[DEBUG] Raw API response: '{translated_title}'")
+ print(f"[DEBUG] Response length: {len(translated_title)} (original: {len(title)})")
+ newline = '\n'
+ print(f"[DEBUG] Has newlines: {repr(translated_title) if newline in translated_title else 'No'}")
+
+ translated_title = translated_title.strip()
+
+ if ((translated_title.startswith('"') and translated_title.endswith('"')) or
+ (translated_title.startswith("'") and translated_title.endswith("'"))):
+ translated_title = translated_title[1:-1].strip()
+
+ if '\n' in translated_title:
+ print(f"⚠️ API returned multi-line content, keeping original title")
+ return title
+
+ # Check for JSON-like structured content, but allow simple brackets like [END]
+ if (any(char in translated_title for char in ['{', '}']) or
+ '"role":' in translated_title or
+ '"content":' in translated_title or
+ ('[[' in translated_title and ']]' in translated_title)): # Only flag double brackets
+ print(f"⚠️ API returned structured content, keeping original title")
+ return title
+
+ if any(tag in translated_title.lower() for tag in ['= 2:
+ return True
+
+ # Single strong error indicator in very short response
+ if len(content_str) < 50 and error_count >= 1:
+ return True
+
+ return False
+
+
+# Additional helper function for debugging
+def get_failure_reason(content):
+ """
+ Returns the specific reason why content was marked as qa_failed
+ Useful for debugging and logging
+ """
+ if not content:
+ return "Empty content"
+
+ content_str = str(content).strip()
+ content_lower = content_str.lower()
+
+ # Check each category and return the first match
+ failure_categories = {
+ "Explicit Failure Marker": [
+ "[TRANSLATION FAILED - ORIGINAL TEXT PRESERVED]",
+ "[IMAGE TRANSLATION FAILED]",
+ "API response unavailable",
+ "[]"
+ ],
+ "HTTP Error": [
+ "authentication_error", "rate_limit_error", "api_error"
+ ],
+ "Content Filter": [
+ "content_filter", "safety filter", "blocked by safety"
+ ],
+ "Timeout": [
+ "timeout", "timed out", "apitimeouterror"
+ ],
+ "Rate Limit": [
+ "rate limit exceeded", "quota exceeded", "too many requests"
+ ],
+ "Refusal Pattern": [
+ "i cannot", "i can't", "unable to process"
+ ],
+ "Empty Response": [
+ '"text": ""', "choices: [ { text: ''"
+ ]
+ }
+
+ for category, markers in failure_categories.items():
+ for marker in markers:
+ if marker in content_str or marker in content_lower:
+ return f"{category}: {marker}"
+
+ if len(content_str) < 50:
+ return f"Short response with error indicators: {content_str[:30]}..."
+
+ return "Unknown failure pattern"
+
+def convert_enhanced_text_to_html(plain_text, chapter_info=None):
+ """Convert markdown/plain text back to HTML after translation (for enhanced mode)
+
+ This function handles the conversion of translated markdown back to HTML.
+ The input is the TRANSLATED text that was originally extracted using html2text.
+ """
+ import re
+
+ preserve_structure = chapter_info.get('preserve_structure', False) if chapter_info else False
+
+ # First, try to use markdown2 for proper markdown conversion
+ try:
+ import markdown2
+
+ # Check if the text contains markdown patterns
+ has_markdown = any([
+ '##' in plain_text, # Headers
+ '**' in plain_text, # Bold
+ '*' in plain_text and not '**' in plain_text, # Italic
+ '[' in plain_text and '](' in plain_text, # Links
+ '```' in plain_text, # Code blocks
+ '> ' in plain_text, # Blockquotes
+ '- ' in plain_text or '* ' in plain_text or '1. ' in plain_text # Lists
+ ])
+
+ if has_markdown or preserve_structure:
+ # Use markdown2 for proper conversion
+ html = markdown2.markdown(plain_text, extras=[
+ 'cuddled-lists', # Lists without blank lines
+ 'fenced-code-blocks', # Code blocks with ```
+ 'break-on-newline', # Treat single newlines as
+ 'smarty-pants', # Smart quotes and dashes + 'tables', # Markdown tables + ]) + + # Post-process to ensure proper paragraph structure + if not '
') + continue + + # Check for markdown headers + if line.startswith('#'): + match = re.match(r'^(#+)\s*(.+)$', line) + if match: + level = min(len(match.group(1)), 6) + header_text = match.group(2).strip() + html_parts.append(f'{header_text} ')
+ continue
+
+ # Check for blockquotes
+ if line.startswith('> '):
+ quote_text = line[2:].strip()
+ html_parts.append(f'{list_text} ')
+ continue
+
+ if re.match(r'^\d+\.\s+', line):
+ list_text = re.sub(r'^\d+\.\s+', '', line)
+ html_parts.append(f'{list_text} ')
+ continue
+
+ # Convert inline markdown
+ # Bold
+ line = re.sub(r'\*\*(.+?)\*\*', r'\1', line)
+ line = re.sub(r'__(.+?)__', r'\1', line)
+
+ # Italic
+ line = re.sub(r'\*(.+?)\*', r'\1', line)
+ line = re.sub(r'_(.+?)_', r'\1', line)
+
+ # Links
+ line = re.sub(r'\[([^\]]+)\]\(([^)]+)\)', r'\1', line)
+
+ # Code inline
+ line = re.sub(r'`([^`]+)`', r''):
+ if not in_list:
+ # Determine list type based on context (simplified)
+ list_type = 'ul' # Default to unordered
+ final_html.append(f'<{list_type}>')
+ in_list = True
+ final_html.append(part)
+ else:
+ if in_list:
+ final_html.append(f'')
+ in_list = False
+ final_html.append(part)
+
+ # Close any open list
+ if in_list:
+ final_html.append(f'')
+
+ return '\n'.join(final_html)
+# =====================================================
+# MAIN TRANSLATION FUNCTION
+# =====================================================
+def main(log_callback=None, stop_callback=None):
+ """Main translation function with enhanced duplicate detection and progress tracking"""
+
+ config = TranslationConfig()
+ builtins._DISABLE_ZERO_DETECTION = config.DISABLE_ZERO_DETECTION
+
+ if config.DISABLE_ZERO_DETECTION:
+ print("=" * 60)
+ print("⚠️ 0-BASED DETECTION DISABLED BY USER")
+ print("⚠️ All chapter numbers will be used exactly as found")
+ print("=" * 60)
+
+ args = None
+ chapters_completed = 0
+ chunks_completed = 0
+
+ args = None
+ chapters_completed = 0
+ chunks_completed = 0
+
+ input_path = config.input_path
+ if not input_path and len(sys.argv) > 1:
+ input_path = sys.argv[1]
+
+ is_text_file = input_path.lower().endswith('.txt')
+
+ if is_text_file:
+ os.environ["IS_TEXT_FILE_TRANSLATION"] = "1"
+
+ import json as _json
+ _original_load = _json.load
+
+ def debug_json_load(fp, *args, **kwargs):
+ result = _original_load(fp, *args, **kwargs)
+ if isinstance(result, list) and len(result) > 0:
+ if isinstance(result[0], dict) and 'original_name' in result[0]:
+ print(f"[DEBUG] Loaded glossary list with {len(result)} items from {fp.name if hasattr(fp, 'name') else 'unknown'}")
+ return result
+
+ _json.load = debug_json_load
+
+ if log_callback:
+ set_output_redirect(log_callback)
+
+ def check_stop():
+ if stop_callback and stop_callback():
+ print("❌ Translation stopped by user request.")
+ return True
+ return is_stop_requested()
+
+ if config.EMERGENCY_RESTORE:
+ print("✅ Emergency paragraph restoration is ENABLED")
+ else:
+ print("⚠️ Emergency paragraph restoration is DISABLED")
+
+ print(f"[DEBUG] REMOVE_AI_ARTIFACTS environment variable: {os.getenv('REMOVE_AI_ARTIFACTS', 'NOT SET')}")
+ print(f"[DEBUG] REMOVE_AI_ARTIFACTS parsed value: {config.REMOVE_AI_ARTIFACTS}")
+ if config.REMOVE_AI_ARTIFACTS:
+ print("⚠️ AI artifact removal is ENABLED - will clean AI response artifacts")
+ else:
+ print("✅ AI artifact removal is DISABLED - preserving all content as-is")
+
+ if '--epub' in sys.argv or (len(sys.argv) > 1 and sys.argv[1].endswith(('.epub', '.txt'))):
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument('epub', help='Input EPUB or text file')
+ args = parser.parse_args()
+ input_path = args.epub
+
+ is_text_file = input_path.lower().endswith('.txt')
+
+ if is_text_file:
+ file_base = os.path.splitext(os.path.basename(input_path))[0]
+ else:
+ epub_base = os.path.splitext(os.path.basename(input_path))[0]
+ file_base = epub_base
+
+ out = file_base
+ os.makedirs(out, exist_ok=True)
+ print(f"[DEBUG] Created output folder → {out}")
+
+ cleanup_previous_extraction(out)
+
+ os.environ["EPUB_OUTPUT_DIR"] = out
+ payloads_dir = out
+
+ # clear history if CONTEXTUAL is disabled
+ if not config.CONTEXTUAL:
+ history_file = os.path.join(payloads_dir, "translation_history.json")
+ if os.path.exists(history_file):
+ os.remove(history_file)
+ print("[DEBUG] CONTEXTUAL disabled - cleared translation history")
+
+ history_manager = HistoryManager(payloads_dir)
+ chapter_splitter = ChapterSplitter(model_name=config.MODEL)
+ chunk_context_manager = ChunkContextManager()
+ progress_manager = ProgressManager(payloads_dir)
+
+ # Create ChapterExtractor with progress callback if available
+ chapter_progress_callback = None
+ if log_callback:
+ # Create a wrapper that formats progress messages for the log
+ def chapter_progress_callback(msg):
+ log_callback(f"📊 {msg}")
+
+ chapter_extractor = ChapterExtractor(progress_callback=chapter_progress_callback)
+ glossary_manager = GlossaryManager()
+
+ history_file = os.path.join(payloads_dir, "translation_history.json")
+ if os.path.exists(history_file):
+ os.remove(history_file)
+ print(f"[DEBUG] Purged translation history → {history_file}")
+
+ print("🔍 Checking for deleted output files...")
+ progress_manager.cleanup_missing_files(out)
+ progress_manager.save()
+
+ if check_stop():
+ return
+
+ if not config.API_KEY:
+ print("❌ Error: Set API_KEY, OPENAI_API_KEY, or OPENAI_OR_Gemini_API_KEY in your environment.")
+ return
+
+ #print(f"[DEBUG] Found API key: {config.API_KEY[:10]}...")
+ print(f"[DEBUG] Using model = {config.MODEL}")
+ print(f"[DEBUG] Max output tokens = {config.MAX_OUTPUT_TOKENS}")
+
+ client = UnifiedClient(model=config.MODEL, api_key=config.API_KEY, output_dir=out)
+ if hasattr(client, 'use_multi_keys') and client.use_multi_keys:
+ stats = client.get_stats()
+ print(f"🔑 Multi-key mode active: {stats.get('total_keys', 0)} keys loaded")
+ print(f" Active keys: {stats.get('active_keys', 0)}")
+ else:
+ print(f"🔑 Single-key mode: Using {config.MODEL}")
+ # Reset cleanup state when starting new translation
+ if hasattr(client, 'reset_cleanup_state'):
+ client.reset_cleanup_state()
+
+ if is_text_file:
+ print("📄 Processing text file...")
+ try:
+ txt_processor = TextFileProcessor(input_path, out)
+ chapters = txt_processor.extract_chapters()
+ txt_processor.save_original_structure()
+
+ metadata = {
+ "title": os.path.splitext(os.path.basename(input_path))[0],
+ "type": "text",
+ "chapter_count": len(chapters)
+ }
+ except ImportError as e:
+ print(f"❌ Error: Text file processor not available: {e}")
+ if log_callback:
+ log_callback(f"❌ Error: Text file processor not available: {e}")
+ return
+ except Exception as e:
+ print(f"❌ Error processing text file: {e}")
+ if log_callback:
+ log_callback(f"❌ Error processing text file: {e}")
+ return
+ else:
+ # Check if we should use async extraction (for GUI mode)
+ use_async_extraction = os.getenv("USE_ASYNC_CHAPTER_EXTRACTION", "0") == "1"
+
+ if use_async_extraction and log_callback:
+ print("🚀 Using async chapter extraction (subprocess mode)...")
+ from chapter_extraction_manager import ChapterExtractionManager
+
+ # Create manager with log callback
+ extraction_manager = ChapterExtractionManager(log_callback=log_callback)
+
+ # Get extraction mode
+ extraction_mode = os.getenv("EXTRACTION_MODE", "smart").lower()
+
+ # Define completion callback
+ extraction_result = {"completed": False, "result": None}
+
+ def on_extraction_complete(result):
+ extraction_result["completed"] = True
+ extraction_result["result"] = result
+
+ # Safety check for None result
+ if result is None:
+ log_callback("❌ Chapter extraction failed: No result returned")
+ return
+
+ if result.get("success"):
+ log_callback(f"✅ Chapter extraction completed: {result.get('chapters', 0)} chapters")
+ else:
+ log_callback(f"❌ Chapter extraction failed: {result.get('error', 'Unknown error')}")
+
+ # Start async extraction
+ extraction_manager.extract_chapters_async(
+ input_path,
+ out,
+ extraction_mode=extraction_mode,
+ progress_callback=lambda msg: log_callback(f"📊 {msg}"),
+ completion_callback=on_extraction_complete
+ )
+
+ # Wait for completion (with timeout)
+ timeout = 300 # 5 minutes timeout
+ start_time = time.time()
+
+ while not extraction_result["completed"]:
+ if check_stop():
+ extraction_manager.stop_extraction()
+ return
+
+ if time.time() - start_time > timeout:
+ log_callback("⚠️ Chapter extraction timeout")
+ extraction_manager.stop_extraction()
+ return
+
+ time.sleep(0.1) # Check every 100ms
+
+ # Check if extraction was successful
+ if not extraction_result["result"] or not extraction_result["result"].get("success"):
+ log_callback("❌ Chapter extraction failed")
+ return
+
+ # Load the extracted data
+ metadata_path = os.path.join(out, "metadata.json")
+ if os.path.exists(metadata_path):
+ with open(metadata_path, 'r', encoding='utf-8') as f:
+ metadata = json.load(f)
+ else:
+ metadata = extraction_result["result"].get("metadata", {})
+
+ # The async extraction should have saved chapters directly, similar to the sync version
+ # We need to reconstruct the chapters list with body content
+
+ # Check if the extraction actually created a chapters.json file with full content
+ chapters_full_path = os.path.join(out, "chapters_full.json")
+ chapters_info_path = os.path.join(out, "chapters_info.json")
+
+ chapters = []
+
+ # First try to load full chapters if saved
+ if os.path.exists(chapters_full_path):
+ log_callback("Loading full chapters data...")
+ with open(chapters_full_path, 'r', encoding='utf-8') as f:
+ chapters = json.load(f)
+ log_callback(f"✅ Loaded {len(chapters)} chapters with content")
+
+ elif os.path.exists(chapters_info_path):
+ # Fall back to loading from individual files
+ log_callback("Loading chapter info and searching for content files...")
+ with open(chapters_info_path, 'r', encoding='utf-8') as f:
+ chapters_info = json.load(f)
+
+ # List all files in the output directory
+ all_files = os.listdir(out)
+ log_callback(f"Found {len(all_files)} files in output directory")
+
+ # Try to match chapter files
+ for info in chapters_info:
+ chapter_num = info['num']
+ found = False
+
+ # Try different naming patterns
+ patterns = [
+ f"chapter_{chapter_num:04d}_", # With leading zeros
+ f"chapter_{chapter_num}_", # Without leading zeros
+ f"ch{chapter_num:04d}_", # Shortened with zeros
+ f"ch{chapter_num}_", # Shortened without zeros
+ f"{chapter_num:04d}_", # Just number with zeros
+ f"{chapter_num}_" # Just number
+ ]
+
+ for pattern in patterns:
+ # Find files matching this pattern (any extension)
+ matching_files = [f for f in all_files if f.startswith(pattern)]
+
+ if matching_files:
+ # Prefer HTML/XHTML files
+ html_files = [f for f in matching_files if f.endswith(('.html', '.xhtml', '.htm'))]
+ if html_files:
+ chapter_file = html_files[0]
+ else:
+ chapter_file = matching_files[0]
+
+ chapter_path = os.path.join(out, chapter_file)
+
+ try:
+ with open(chapter_path, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ chapters.append({
+ "num": chapter_num,
+ "title": info.get("title", f"Chapter {chapter_num}"),
+ "body": content,
+ "filename": info.get("original_filename", ""),
+ "has_images": info.get("has_images", False),
+ "file_size": len(content),
+ "content_hash": info.get("content_hash", "")
+ })
+ found = True
+ break
+ except Exception as e:
+ log_callback(f"⚠️ Error reading {chapter_file}: {e}")
+
+ if not found:
+ log_callback(f"⚠️ No file found for Chapter {chapter_num}")
+ # Log available files for debugging
+ if len(all_files) < 50:
+ similar_files = [f for f in all_files if str(chapter_num) in f]
+ if similar_files:
+ log_callback(f" Similar files: {similar_files[:3]}")
+
+ if not chapters:
+ log_callback("❌ No chapters could be loaded!")
+ log_callback(f"❌ Output directory: {out}")
+ log_callback(f"❌ Files in directory: {len(os.listdir(out))} files")
+ # Show first few files for debugging
+ sample_files = os.listdir(out)[:10]
+ log_callback(f"❌ Sample files: {sample_files}")
+ return
+
+ # Sort chapters by OPF spine order if available
+ opf_path = os.path.join(out, 'content.opf')
+ if os.path.exists(opf_path) and chapters:
+ log_callback("📋 Sorting chapters according to OPF spine order...")
+ # Use the existing chapter_extractor instance to sort
+ chapters = chapter_extractor._sort_by_opf_spine(chapters, opf_path)
+ log_callback("✅ Chapters sorted according to OPF reading order")
+ else:
+ print("🚀 Using comprehensive chapter extraction with resource handling...")
+ with zipfile.ZipFile(input_path, 'r') as zf:
+ metadata = chapter_extractor._extract_epub_metadata(zf)
+ chapters = chapter_extractor.extract_chapters(zf, out)
+
+ print(f"\n📚 Extraction Summary:")
+ print(f" Total chapters extracted: {len(chapters)}")
+ if chapters:
+ nums = [c.get('num', 0) for c in chapters]
+ print(f" Chapter range: {min(nums)} to {max(nums)}")
+
+ # Check for gaps in the sequence
+ expected_count = max(nums) - min(nums) + 1
+ if len(chapters) < expected_count:
+ print(f"\n⚠️ Potential missing chapters detected:")
+ print(f" Expected {expected_count} chapters (from {min(nums)} to {max(nums)})")
+ print(f" Actually found: {len(chapters)} chapters")
+ print(f" Potentially missing: {expected_count - len(chapters)} chapters")
+
+ validate_chapter_continuity(chapters)
+
+ print("\n" + "="*50)
+ validate_epub_structure(out)
+ print("="*50 + "\n")
+
+ progress_manager.migrate_to_content_hash(chapters)
+ progress_manager.save()
+
+ if check_stop():
+ return
+
+ metadata_path = os.path.join(out, "metadata.json")
+ if os.path.exists(metadata_path):
+ with open(metadata_path, 'r', encoding='utf-8') as mf:
+ metadata = json.load(mf)
+
+ metadata["chapter_count"] = len(chapters)
+ metadata["chapter_titles"] = {str(c["num"]): c["title"] for c in chapters}
+
+ print(f"[DEBUG] Initializing client with model = {config.MODEL}")
+ client = UnifiedClient(api_key=config.API_KEY, model=config.MODEL, output_dir=out)
+ if hasattr(client, 'use_multi_keys') and client.use_multi_keys:
+ stats = client.get_stats()
+ print(f"🔑 Multi-key mode active: {stats.get('total_keys', 0)} keys loaded")
+ print(f" Active keys: {stats.get('active_keys', 0)}")
+ else:
+ print(f"🔑 Single-key mode: Using {config.MODEL}")
+
+ # Reset cleanup state when starting new translation
+ if hasattr(client, 'reset_cleanup_state'):
+ client.reset_cleanup_state()
+
+ if "title" in metadata and config.TRANSLATE_BOOK_TITLE and not metadata.get("title_translated", False):
+ original_title = metadata["title"]
+ print(f"📚 Original title: {original_title}")
+
+ if not check_stop():
+ translated_title = translate_title(
+ original_title,
+ client,
+ None,
+ None,
+ config.TEMP
+ )
+
+ metadata["original_title"] = original_title
+ metadata["title"] = translated_title
+ metadata["title_translated"] = True
+
+ print(f"📚 Translated title: {translated_title}")
+ else:
+ print("❌ Title translation skipped due to stop request")
+
+ # Translate other metadata fields if configured
+ translate_metadata_fields_str = os.getenv('TRANSLATE_METADATA_FIELDS', '{}')
+ metadata_translation_mode = os.getenv('METADATA_TRANSLATION_MODE', 'together')
+
+ try:
+ translate_metadata_fields = json.loads(translate_metadata_fields_str)
+
+ if translate_metadata_fields and any(translate_metadata_fields.values()):
+ # Filter out fields that should be translated (excluding already translated fields)
+ fields_to_translate = {}
+ skipped_fields = []
+
+ for field_name, should_translate in translate_metadata_fields.items():
+ if should_translate and field_name != 'title' and field_name in metadata:
+ # Check if already translated
+ if metadata.get(f"{field_name}_translated", False):
+ skipped_fields.append(field_name)
+ print(f"✓ Skipping {field_name} - already translated")
+ else:
+ fields_to_translate[field_name] = should_translate
+
+ if fields_to_translate:
+ print("\n" + "="*50)
+ print("📋 METADATA TRANSLATION PHASE")
+ print("="*50)
+ print(f"🌐 Translating {len(fields_to_translate)} metadata fields...")
+
+ # Get ALL configuration from environment - NO DEFAULTS
+ system_prompt = os.getenv('BOOK_TITLE_SYSTEM_PROMPT', '')
+ if not system_prompt:
+ print("❌ No system prompt configured, skipping metadata translation")
+ else:
+ # Get field-specific prompts
+ field_prompts_str = os.getenv('METADATA_FIELD_PROMPTS', '{}')
+ try:
+ field_prompts = json.loads(field_prompts_str)
+ except:
+ field_prompts = {}
+
+ if not field_prompts and not field_prompts.get('_default'):
+ print("❌ No field prompts configured, skipping metadata translation")
+ else:
+ # Get language configuration
+ lang_behavior = os.getenv('LANG_PROMPT_BEHAVIOR', 'auto')
+ forced_source_lang = os.getenv('FORCED_SOURCE_LANG', 'Korean')
+ output_language = os.getenv('OUTPUT_LANGUAGE', 'English')
+
+ # Determine source language
+ source_lang = metadata.get('language', '').lower()
+ if lang_behavior == 'never':
+ lang_str = ""
+ elif lang_behavior == 'always':
+ lang_str = forced_source_lang
+ else: # auto
+ if 'zh' in source_lang or 'chinese' in source_lang:
+ lang_str = 'Chinese'
+ elif 'ja' in source_lang or 'japanese' in source_lang:
+ lang_str = 'Japanese'
+ elif 'ko' in source_lang or 'korean' in source_lang:
+ lang_str = 'Korean'
+ else:
+ lang_str = ''
+
+ # Check if batch translation is enabled for parallel processing
+ batch_translate_enabled = os.getenv('BATCH_TRANSLATION', '0') == '1'
+ batch_size = int(os.getenv('BATCH_SIZE', '50')) # Default batch size
+
+ if batch_translate_enabled and len(fields_to_translate) > 1:
+ print(f"⚡ Using parallel metadata translation mode ({len(fields_to_translate)} fields, batch size: {batch_size})...")
+
+ # Import ThreadPoolExecutor for parallel processing
+ from concurrent.futures import ThreadPoolExecutor, as_completed
+ import threading
+
+ # Thread-safe results storage
+ translation_results = {}
+ results_lock = threading.Lock()
+
+ def translate_metadata_field(field_name, original_value):
+ """Translate a single metadata field"""
+ try:
+ print(f"\n📋 Translating {field_name}: {original_value[:100]}..."
+ if len(str(original_value)) > 100 else f"\n📋 Translating {field_name}: {original_value}")
+
+ # Get field-specific prompt
+ prompt_template = field_prompts.get(field_name, field_prompts.get('_default', ''))
+
+ if not prompt_template:
+ print(f"⚠️ No prompt configured for field '{field_name}', skipping")
+ return None
+
+ # Replace variables in prompt
+ field_prompt = prompt_template.replace('{source_lang}', lang_str)
+ field_prompt = field_prompt.replace('{output_lang}', output_language)
+ field_prompt = field_prompt.replace('English', output_language)
+ field_prompt = field_prompt.replace('{field_value}', str(original_value))
+
+ # Check if we're using a translation service (not AI)
+ client_type = getattr(client, 'client_type', '')
+ is_translation_service = client_type in ['deepl', 'google_translate']
+
+ if is_translation_service:
+ # For translation services, send only the field value without AI prompts
+ print(f"🌐 Using translation service ({client_type}) - sending field directly")
+ messages = [
+ {"role": "user", "content": str(original_value)}
+ ]
+ else:
+ # For AI services, use prompts as before
+ messages = [
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": f"{field_prompt}\n\n{original_value}"}
+ ]
+
+ # Add delay for rate limiting
+ if config.DELAY > 0:
+ time.sleep(config.DELAY)
+
+ # Make API call
+ content, finish_reason = client.send(
+ messages,
+ temperature=config.TEMP,
+ max_tokens=config.MAX_OUTPUT_TOKENS
+ )
+ translated_value = content.strip()
+
+ # Store result thread-safely
+ with results_lock:
+ translation_results[field_name] = {
+ 'original': original_value,
+ 'translated': translated_value,
+ 'success': True
+ }
+
+ print(f"✅ Translated {field_name}: {translated_value}")
+ return translated_value
+
+ except Exception as e:
+ print(f"❌ Failed to translate {field_name}: {e}")
+ with results_lock:
+ translation_results[field_name] = {
+ 'original': original_value,
+ 'translated': None,
+ 'success': False,
+ 'error': str(e)
+ }
+ return None
+
+ # Execute parallel translations with limited workers
+ max_workers = min(len(fields_to_translate), batch_size)
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ # Submit all translation tasks
+ futures = {}
+ for field_name in fields_to_translate:
+ if field_name in metadata and not check_stop():
+ original_value = metadata[field_name]
+ future = executor.submit(translate_metadata_field, field_name, original_value)
+ futures[future] = field_name
+
+ # Wait for completion
+ for future in as_completed(futures):
+ if check_stop():
+ print("❌ Metadata translation stopped by user")
+ break
+
+ # Apply results to metadata
+ for field_name, result in translation_results.items():
+ if result['success'] and result['translated']:
+ metadata[f"original_{field_name}"] = result['original']
+ metadata[field_name] = result['translated']
+ metadata[f"{field_name}_translated"] = True
+
+ else:
+ # Sequential translation mode (individual translation)
+ mode_desc = "sequential" if not batch_translate_enabled else "sequential (single field)"
+ print(f"📝 Using {mode_desc} translation mode...")
+
+ for field_name in fields_to_translate:
+ if not check_stop() and field_name in metadata:
+ original_value = metadata[field_name]
+ print(f"\n📋 Translating {field_name}: {original_value[:100]}..."
+ if len(str(original_value)) > 100 else f"\n📋 Translating {field_name}: {original_value}")
+
+ # Get field-specific prompt
+ prompt_template = field_prompts.get(field_name, field_prompts.get('_default', ''))
+
+ if not prompt_template:
+ print(f"⚠️ No prompt configured for field '{field_name}', skipping")
+ continue
+
+ # Replace variables in prompt
+ field_prompt = prompt_template.replace('{source_lang}', lang_str)
+ field_prompt = field_prompt.replace('{output_lang}', output_language)
+ field_prompt = field_prompt.replace('English', output_language)
+ field_prompt = field_prompt.replace('{field_value}', str(original_value))
+
+ # Check if we're using a translation service (not AI)
+ client_type = getattr(client, 'client_type', '')
+ is_translation_service = client_type in ['deepl', 'google_translate']
+
+ if is_translation_service:
+ # For translation services, send only the field value without AI prompts
+ print(f"🌐 Using translation service ({client_type}) - sending field directly")
+ messages = [
+ {"role": "user", "content": str(original_value)}
+ ]
+ else:
+ # For AI services, use prompts as before
+ messages = [
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": f"{field_prompt}\n\n{original_value}"}
+ ]
+
+ try:
+ # Add delay using the config instance from main()
+ if config.DELAY > 0: # ✅ FIXED - use config.DELAY instead of config.SEND_INTERVAL
+ time.sleep(config.DELAY)
+
+ # Use the same client instance from main()
+ # ✅ FIXED - Properly unpack tuple response and provide max_tokens
+ content, finish_reason = client.send(
+ messages,
+ temperature=config.TEMP,
+ max_tokens=config.MAX_OUTPUT_TOKENS # ✅ FIXED - provide max_tokens to avoid NoneType error
+ )
+ translated_value = content.strip() # ✅ FIXED - use content from unpacked tuple
+
+ metadata[f"original_{field_name}"] = original_value
+ metadata[field_name] = translated_value
+ metadata[f"{field_name}_translated"] = True
+
+ print(f"✅ Translated {field_name}: {translated_value}")
+
+ except Exception as e:
+ print(f"❌ Failed to translate {field_name}: {e}")
+
+ else:
+ if check_stop():
+ print("❌ Metadata translation stopped by user")
+ break
+ else:
+ print("📋 No additional metadata fields to translate")
+
+ except Exception as e:
+ print(f"⚠️ Error processing metadata translation settings: {e}")
+ import traceback
+ traceback.print_exc()
+
+ with open(metadata_path, 'w', encoding='utf-8') as mf:
+ json.dump(metadata, mf, ensure_ascii=False, indent=2)
+ print(f"💾 Saved metadata with {'translated' if metadata.get('title_translated', False) else 'original'} title")
+
+ print("\n" + "="*50)
+ print("📑 GLOSSARY GENERATION PHASE")
+ print("="*50)
+
+ print(f"📑 DEBUG: ENABLE_AUTO_GLOSSARY = '{os.getenv('ENABLE_AUTO_GLOSSARY', 'NOT SET')}'")
+ print(f"📑 DEBUG: MANUAL_GLOSSARY = '{config.MANUAL_GLOSSARY}'")
+ print(f"📑 DEBUG: Manual glossary exists? {os.path.isfile(config.MANUAL_GLOSSARY) if config.MANUAL_GLOSSARY else False}")
+
+ # Check if glossary.csv already exists in the source folder
+ existing_glossary_csv = os.path.join(out, "glossary.csv")
+ existing_glossary_json = os.path.join(out, "glossary.json")
+ print(f"📑 DEBUG: Existing glossary.csv? {os.path.exists(existing_glossary_csv)}")
+ print(f"📑 DEBUG: Existing glossary.json? {os.path.exists(existing_glossary_json)}")
+
+ if config.MANUAL_GLOSSARY and os.path.isfile(config.MANUAL_GLOSSARY):
+ ext = os.path.splitext(config.MANUAL_GLOSSARY)[1].lower()
+ target_name = "glossary.csv" if ext == ".csv" else "glossary.json"
+ target_path = os.path.join(out, target_name)
+ if os.path.abspath(config.MANUAL_GLOSSARY) != os.path.abspath(target_path):
+ shutil.copy(config.MANUAL_GLOSSARY, target_path)
+ print("📑 Using manual glossary from:", config.MANUAL_GLOSSARY)
+ else:
+ print("📑 Using existing glossary:", config.MANUAL_GLOSSARY)
+ elif os.path.exists(existing_glossary_csv) or os.path.exists(existing_glossary_json):
+ print("📑 Existing glossary file detected in source folder - skipping automatic generation")
+ if os.path.exists(existing_glossary_csv):
+ print(f"📑 Using existing glossary.csv: {existing_glossary_csv}")
+ elif os.path.exists(existing_glossary_json):
+ print(f"📑 Using existing glossary.json: {existing_glossary_json}")
+ elif os.getenv("ENABLE_AUTO_GLOSSARY", "0") == "1":
+ model = os.getenv("MODEL", "gpt-4")
+ if is_traditional_translation_api(model):
+ print("📑 Automatic glossary generation disabled")
+ print(f" {model} does not support glossary extraction")
+ print(" Traditional translation APIs cannot identify character names/terms")
+ else:
+ print("📑 Starting automatic glossary generation...")
+ try:
+ # Use the new process-safe glossary worker
+ from glossary_process_worker import generate_glossary_in_process
+ import concurrent.futures
+ import multiprocessing
+
+ instructions = ""
+
+ # Get extraction workers setting
+ extraction_workers = int(os.getenv("EXTRACTION_WORKERS", "1"))
+ if extraction_workers == 1:
+ # Auto-detect for better performance
+ extraction_workers = min(os.cpu_count() or 4, 4)
+ print(f"📑 Using {extraction_workers} CPU cores for glossary generation")
+
+ # Collect environment variables to pass to subprocess
+ env_vars = {}
+ important_vars = [
+ 'EXTRACTION_WORKERS', 'GLOSSARY_MIN_FREQUENCY', 'GLOSSARY_MAX_NAMES',
+ 'GLOSSARY_MAX_TITLES', 'GLOSSARY_BATCH_SIZE', 'GLOSSARY_STRIP_HONORIFICS',
+ 'GLOSSARY_FUZZY_THRESHOLD', 'GLOSSARY_MAX_TEXT_SIZE', 'AUTO_GLOSSARY_PROMPT',
+ 'GLOSSARY_USE_SMART_FILTER', 'GLOSSARY_USE_LEGACY_CSV', 'GLOSSARY_PARALLEL_ENABLED',
+ 'GLOSSARY_FILTER_MODE', 'GLOSSARY_SKIP_FREQUENCY_CHECK', 'GLOSSARY_SKIP_ALL_VALIDATION',
+ 'MODEL', 'API_KEY', 'OPENAI_API_KEY', 'GEMINI_API_KEY', 'MAX_OUTPUT_TOKENS',
+ 'GLOSSARY_TEMPERATURE', 'MANUAL_GLOSSARY', 'ENABLE_AUTO_GLOSSARY'
+ ]
+
+ for var in important_vars:
+ if var in os.environ:
+ env_vars[var] = os.environ[var]
+
+ # Create a Queue for real-time log streaming
+ manager = multiprocessing.Manager()
+ log_queue = manager.Queue()
+
+ # Use ProcessPoolExecutor for true parallelism (completely bypasses GIL)
+ print("📑 Starting glossary generation in separate process...")
+ with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor:
+ # Submit to separate process WITH log queue
+ future = executor.submit(
+ generate_glossary_in_process,
+ out,
+ chapters,
+ instructions,
+ env_vars,
+ log_queue # Pass the queue for real-time logs
+ )
+
+ # Poll for completion and stream logs in real-time
+ poll_count = 0
+ while not future.done():
+ poll_count += 1
+
+ # Check for logs from subprocess and print them immediately
+ try:
+ while not log_queue.empty():
+ log_line = log_queue.get_nowait()
+ print(log_line) # Print to GUI
+ except:
+ pass
+
+ # Super short sleep to yield to GUI
+ time.sleep(0.001)
+
+ # Check for stop every 100 polls
+ if poll_count % 100 == 0:
+ if check_stop():
+ print("📑 ❌ Glossary generation cancelled")
+ executor.shutdown(wait=False, cancel_futures=True)
+ return
+
+ # Get any remaining logs from queue
+ try:
+ while not log_queue.empty():
+ log_line = log_queue.get_nowait()
+ print(log_line)
+ except:
+ pass
+
+ # Get result
+ if future.done():
+ try:
+ result = future.result(timeout=0.1)
+ if isinstance(result, dict):
+ if result.get('success'):
+ print(f"📑 ✅ Glossary generation completed successfully")
+ else:
+ print(f"📑 ❌ Glossary generation failed: {result.get('error')}")
+ if result.get('traceback'):
+ print(f"📑 Error details:\n{result.get('traceback')}")
+ except Exception as e:
+ print(f"📑 ❌ Error retrieving glossary result: {e}")
+
+ print("✅ Automatic glossary generation COMPLETED")
+
+ # Handle deferred glossary appending
+ if os.getenv('DEFER_GLOSSARY_APPEND') == '1':
+ print("📑 Processing deferred glossary append to system prompt...")
+
+ glossary_path = find_glossary_file(out)
+ if glossary_path and os.path.exists(glossary_path):
+ try:
+ glossary_block = None
+ if glossary_path.lower().endswith('.csv'):
+ with open(glossary_path, 'r', encoding='utf-8') as f:
+ glossary_block = f.read()
+ else:
+ with open(glossary_path, 'r', encoding='utf-8') as f:
+ glossary_data = json.load(f)
+
+ formatted_entries = {}
+ if isinstance(glossary_data, dict) and 'entries' in glossary_data:
+ formatted_entries = glossary_data['entries']
+ elif isinstance(glossary_data, dict):
+ formatted_entries = {k: v for k, v in glossary_data.items() if k != "metadata"}
+
+ if formatted_entries:
+ glossary_block = json.dumps(formatted_entries, ensure_ascii=False, indent=2)
+ else:
+ glossary_block = None
+
+ if glossary_block:
+ glossary_prompt = os.getenv('GLOSSARY_APPEND_PROMPT',
+ "Character/Term Glossary (use these translations consistently):")
+
+ current_prompt = config.PROMPT
+ if current_prompt:
+ current_prompt += "\n\n"
+ current_prompt += f"{glossary_prompt}\n{glossary_block}"
+
+ config.PROMPT = current_prompt
+
+ print(f"✅ Added auto-generated glossary to system prompt ({os.path.basename(glossary_path)})")
+
+ if 'DEFER_GLOSSARY_APPEND' in os.environ:
+ del os.environ['DEFER_GLOSSARY_APPEND']
+ if 'GLOSSARY_APPEND_PROMPT' in os.environ:
+ del os.environ['GLOSSARY_APPEND_PROMPT']
+ else:
+ print("⚠️ Auto-generated glossary has no entries - skipping append")
+ if 'DEFER_GLOSSARY_APPEND' in os.environ:
+ del os.environ['DEFER_GLOSSARY_APPEND']
+ if 'GLOSSARY_APPEND_PROMPT' in os.environ:
+ del os.environ['GLOSSARY_APPEND_PROMPT']
+ except Exception as e:
+ print(f"⚠️ Failed to append auto-generated glossary: {e}")
+ else:
+ print("⚠️ No glossary file found after automatic generation")
+
+ except Exception as e:
+ print(f"❌ Glossary generation failed: {e}")
+ else:
+ print("📑 Automatic glossary generation disabled")
+ # Don't create an empty glossary - let any existing manual glossary remain
+
+ glossary_file = find_glossary_file(out)
+ if glossary_file and os.path.exists(glossary_file):
+ try:
+ if glossary_file.lower().endswith('.csv'):
+ # Quick CSV stats
+ with open(glossary_file, 'r', encoding='utf-8') as f:
+ lines = [ln.strip() for ln in f.readlines() if ln.strip()]
+ entry_count = max(0, len(lines) - 1) if lines and ',' in lines[0] else len(lines)
+ print(f"📑 Glossary ready (CSV) with {entry_count} entries")
+ print("📑 Sample glossary lines:")
+ for ln in lines[1:4]:
+ print(f" • {ln}")
+ else:
+ with open(glossary_file, 'r', encoding='utf-8') as f:
+ glossary_data = json.load(f)
+
+ if isinstance(glossary_data, dict):
+ if 'entries' in glossary_data and isinstance(glossary_data['entries'], dict):
+ entry_count = len(glossary_data['entries'])
+ sample_items = list(glossary_data['entries'].items())[:3]
+ else:
+ entry_count = len(glossary_data)
+ sample_items = list(glossary_data.items())[:3]
+
+ print(f"📑 Glossary ready with {entry_count} entries")
+ print("📑 Sample glossary entries:")
+ for key, value in sample_items:
+ print(f" • {key} → {value}")
+
+ elif isinstance(glossary_data, list):
+ print(f"📑 Glossary ready with {len(glossary_data)} entries")
+ print("📑 Sample glossary entries:")
+ for i, entry in enumerate(glossary_data[:3]):
+ if isinstance(entry, dict):
+ original = entry.get('original_name', '?')
+ translated = entry.get('name', original)
+ print(f" • {original} → {translated}")
+ else:
+ print(f"⚠️ Unexpected glossary format: {type(glossary_data)}")
+
+ except Exception as e:
+ print(f"⚠️ Failed to inspect glossary file: {e}")
+ else:
+ print("📑 No glossary file found")
+
+ print("="*50)
+ print("🚀 STARTING MAIN TRANSLATION PHASE")
+ print("="*50 + "\n")
+
+ glossary_path = find_glossary_file(out)
+ if glossary_path and os.path.exists(glossary_path) and glossary_path.lower().endswith('.json'):
+ try:
+ with open(glossary_path, 'r', encoding='utf-8') as f:
+ g_data = json.load(f)
+
+ print(f"[DEBUG] Glossary type before translation: {type(g_data)}")
+ if isinstance(g_data, list):
+ print(f"[DEBUG] Glossary is a list")
+ except Exception as e:
+ print(f"[DEBUG] Error checking glossary: {e}")
+ glossary_path = find_glossary_file(out)
+ system = build_system_prompt(config.SYSTEM_PROMPT, glossary_path)
+ base_msg = [{"role": "system", "content": system}]
+ # Preserve the original system prompt to avoid in-place mutations
+ original_system_prompt = system
+ last_summary_block_text = None # Will hold the last rolling summary text for the NEXT chapter only
+
+ image_translator = None
+
+ if config.ENABLE_IMAGE_TRANSLATION:
+ print(f"🖼️ Image translation enabled for model: {config.MODEL}")
+ print("🖼️ Image translation will use your custom system prompt and glossary")
+ image_translator = ImageTranslator(
+ client,
+ out,
+ config.PROFILE_NAME,
+ system,
+ config.TEMP,
+ log_callback ,
+ progress_manager,
+ history_manager,
+ chunk_context_manager
+ )
+
+ known_vision_models = [
+ 'gemini-1.5-pro', 'gemini-1.5-flash', 'gemini-2.0-flash', 'gemini-2.5-flash', 'gemini-2.5-pro',
+ 'gpt-4-turbo', 'gpt-4o', 'gpt-4.1-mini', 'gpt-4.1-nano', 'o4-mini', 'gpt-4.1-mini'
+ ]
+
+ if config.MODEL.lower() not in known_vision_models:
+ print(f"⚠️ Note: {config.MODEL} may not have vision capabilities. Image translation will be attempted anyway.")
+ else:
+ print("ℹ️ Image translation disabled by user")
+
+ total_chapters = len(chapters)
+
+ # Only detect numbering if the toggle is not disabled
+ if config.DISABLE_ZERO_DETECTION:
+ print(f"📊 0-based detection disabled by user setting")
+ uses_zero_based = False
+ # Important: Set a flag that can be checked throughout the codebase
+ config._force_disable_zero_detection = True
+ else:
+ if chapters:
+ uses_zero_based = detect_novel_numbering(chapters)
+ print(f"📊 Novel numbering detected: {'0-based' if uses_zero_based else '1-based'}")
+ else:
+ uses_zero_based = False
+ config._force_disable_zero_detection = False
+
+ # Store this for later use
+ config._uses_zero_based = uses_zero_based
+
+
+ rng = os.getenv("CHAPTER_RANGE", "")
+ start = None
+ end = None
+ if rng and re.match(r"^\d+\s*-\s*\d+$", rng):
+ start, end = map(int, rng.split("-", 1))
+
+ if config.DISABLE_ZERO_DETECTION:
+ print(f"📊 0-based detection disabled - using range as specified: {start}-{end}")
+ elif uses_zero_based:
+ print(f"📊 0-based novel detected")
+ print(f"📊 User range {start}-{end} will be used as-is (chapters are already adjusted)")
+ else:
+ print(f"📊 1-based novel detected")
+ print(f"📊 Using range as specified: {start}-{end}")
+
+ print("📊 Calculating total chunks needed...")
+ total_chunks_needed = 0
+ chunks_per_chapter = {}
+ chapters_to_process = 0
+
+ # When setting actual chapter numbers (in the main function)
+ for idx, c in enumerate(chapters):
+ chap_num = c["num"]
+ content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
+
+ # Extract the raw chapter number from the file
+ raw_num = FileUtilities.extract_actual_chapter_number(c, patterns=None, config=config)
+ #print(f"[DEBUG] Extracted raw_num={raw_num} from {c.get('original_basename', 'unknown')}")
+
+
+ # Apply the offset
+ offset = config.CHAPTER_NUMBER_OFFSET if hasattr(config, 'CHAPTER_NUMBER_OFFSET') else 0
+ raw_num += offset
+
+ # When toggle is disabled, use raw numbers without any 0-based adjustment
+ if config.DISABLE_ZERO_DETECTION:
+ c['actual_chapter_num'] = raw_num
+ # Store raw number for consistency
+ c['raw_chapter_num'] = raw_num
+ c['zero_adjusted'] = False
+ else:
+ # Store raw number
+ c['raw_chapter_num'] = raw_num
+ # Apply adjustment only if this is a 0-based novel
+ if uses_zero_based:
+ c['actual_chapter_num'] = raw_num + 1
+ c['zero_adjusted'] = True
+ else:
+ c['actual_chapter_num'] = raw_num
+ c['zero_adjusted'] = False
+
+ # Now we can safely use actual_num
+ actual_num = c['actual_chapter_num']
+
+
+ if start is not None:
+ if not (start <= c['actual_chapter_num'] <= end):
+ #print(f"[SKIP] Chapter {c['actual_chapter_num']} outside range {start}-{end}")
+ continue
+
+ needs_translation, skip_reason, _ = progress_manager.check_chapter_status(
+ idx, actual_num, content_hash, out
+ )
+
+ if not needs_translation:
+ chunks_per_chapter[idx] = 0
+ continue
+
+ chapters_to_process += 1
+
+ chapter_key = str(actual_num)
+ if chapter_key in progress_manager.prog["chapters"] and progress_manager.prog["chapters"][chapter_key].get("status") == "in_progress":
+ pass
+
+ # Calculate based on OUTPUT limit only
+ max_output_tokens = config.MAX_OUTPUT_TOKENS
+ safety_margin_output = 500
+
+ # Korean to English typically compresses to 0.7-0.9x
+ compression_factor = config.COMPRESSION_FACTOR
+ available_tokens = int((max_output_tokens - safety_margin_output) / compression_factor)
+
+ # Ensure minimum
+ available_tokens = max(available_tokens, 1000)
+
+ #print(f"📊 Chunk size: {available_tokens:,} tokens (based on {max_output_tokens:,} output limit, compression: {compression_factor})")
+
+ # For mixed content chapters, calculate on clean text
+ # For mixed content chapters, calculate on clean text
+ if c.get('has_images', False) and ContentProcessor.is_meaningful_text_content(c["body"]):
+ # Don't modify c["body"] at all during chunk calculation
+ # Just pass the body as-is, the chunking will be slightly off but that's OK
+ chunks = chapter_splitter.split_chapter(c["body"], available_tokens)
+ else:
+ chunks = chapter_splitter.split_chapter(c["body"], available_tokens)
+
+ chapter_key_str = content_hash
+ old_key_str = str(idx)
+
+ if chapter_key_str not in progress_manager.prog.get("chapter_chunks", {}) and old_key_str in progress_manager.prog.get("chapter_chunks", {}):
+ progress_manager.prog["chapter_chunks"][chapter_key_str] = progress_manager.prog["chapter_chunks"][old_key_str]
+ del progress_manager.prog["chapter_chunks"][old_key_str]
+ #print(f"[PROGRESS] Migrated chunks for chapter {actual_num} to new tracking system")
+
+ # Always count actual chunks - ignore "completed" tracking
+ chunks_per_chapter[idx] = len(chunks)
+ total_chunks_needed += chunks_per_chapter[idx]
+
+ terminology = "Sections" if is_text_file else "Chapters"
+ print(f"📊 Total chunks to translate: {total_chunks_needed}")
+ print(f"📚 {terminology} to process: {chapters_to_process}")
+
+ multi_chunk_chapters = [(idx, count) for idx, count in chunks_per_chapter.items() if count > 1]
+ if multi_chunk_chapters:
+ # Determine terminology based on file type
+ terminology = "Sections" if is_text_file else "Chapters"
+ print(f"📄 {terminology} requiring multiple chunks:")
+ for idx, chunk_count in multi_chunk_chapters:
+ chap = chapters[idx]
+ section_term = "Section" if is_text_file else "Chapter"
+ print(f" • {section_term} {idx+1} ({chap['title'][:30]}...): {chunk_count} chunks")
+
+ translation_start_time = time.time()
+ chunks_completed = 0
+ chapters_completed = 0
+
+ current_chunk_number = 0
+
+ if config.BATCH_TRANSLATION:
+ print(f"\n📦 PARALLEL TRANSLATION MODE ENABLED")
+ print(f"📦 Processing chapters with up to {config.BATCH_SIZE} concurrent API calls")
+
+ import concurrent.futures
+ from threading import Lock
+
+ progress_lock = Lock()
+
+ chapters_to_translate = []
+
+ # FIX: First pass to set actual chapter numbers for ALL chapters
+ # This ensures batch mode has the same chapter numbering as non-batch mode
+ print("📊 Setting chapter numbers...")
+ for idx, c in enumerate(chapters):
+ raw_num = FileUtilities.extract_actual_chapter_number(c, patterns=None, config=config)
+
+ # Apply offset if configured
+ offset = config.CHAPTER_NUMBER_OFFSET if hasattr(config, 'CHAPTER_NUMBER_OFFSET') else 0
+ raw_num += offset
+
+ if config.DISABLE_ZERO_DETECTION:
+ # Use raw numbers without adjustment
+ c['actual_chapter_num'] = raw_num
+ c['raw_chapter_num'] = raw_num
+ c['zero_adjusted'] = False
+ else:
+ # Store raw number
+ c['raw_chapter_num'] = raw_num
+ # Apply 0-based adjustment if detected
+ if uses_zero_based:
+ c['actual_chapter_num'] = raw_num + 1
+ c['zero_adjusted'] = True
+ else:
+ c['actual_chapter_num'] = raw_num
+ c['zero_adjusted'] = False
+
+ for idx, c in enumerate(chapters):
+ chap_num = c["num"]
+ content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
+
+ # Check if this is a pre-split text chunk with decimal number
+ if (is_text_file and c.get('is_chunk', False) and isinstance(c['num'], float)):
+ actual_num = c['num'] # Preserve the decimal for text files only
+ else:
+ actual_num = c.get('actual_chapter_num', c['num']) # Now this will exist!
+
+ # Skip chapters outside the range
+ if start is not None and not (start <= actual_num <= end):
+ continue
+
+ # Check if chapter needs translation
+ needs_translation, skip_reason, existing_file = progress_manager.check_chapter_status(
+ idx, actual_num, content_hash, out, c # Pass the chapter object
+ )
+ # Add explicit file check for supposedly completed chapters
+ if not needs_translation and existing_file:
+ file_path = os.path.join(out, existing_file)
+ if not os.path.exists(file_path):
+ print(f"⚠️ Output file missing for chapter {actual_num}: {existing_file}")
+ needs_translation = True
+ skip_reason = None
+ # Update status to file_missing
+ progress_manager.update(idx, actual_num, content_hash, None, status="file_missing")
+ progress_manager.save()
+
+ if not needs_translation:
+ # Modify skip_reason to use appropriate terminology
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+
+ # Replace "Chapter" with appropriate terminology in skip_reason
+ skip_reason_modified = skip_reason.replace("Chapter", terminology)
+ print(f"[SKIP] {skip_reason_modified}")
+ chapters_completed += 1
+ continue
+
+ # Check for empty or image-only chapters
+ has_images = c.get('has_images', False)
+ has_meaningful_text = ContentProcessor.is_meaningful_text_content(c["body"])
+ text_size = c.get('file_size', 0)
+
+ is_empty_chapter = (not has_images and text_size < 10)
+ is_image_only_chapter = (has_images and not has_meaningful_text)
+
+ # Handle empty chapters
+ if is_empty_chapter:
+ print(f"📄 Empty chapter {chap_num} - will process individually")
+
+ safe_title = make_safe_filename(c['title'], c['num'])
+
+ if isinstance(c['num'], float):
+ fname = FileUtilities.create_chapter_filename(c, c['num'])
+ else:
+ fname = FileUtilities.create_chapter_filename(c, c['num'])
+ with open(os.path.join(out, fname), 'w', encoding='utf-8') as f:
+ f.write(c["body"])
+ progress_manager.update(idx, actual_num, content_hash, fname, status="completed_empty")
+ progress_manager.save()
+ chapters_completed += 1
+ continue
+
+ # Add to chapters to translate
+ chapters_to_translate.append((idx, c))
+
+ print(f"📊 Found {len(chapters_to_translate)} chapters to translate in parallel")
+
+ # Continue with the rest of the existing batch processing code...
+ batch_processor = BatchTranslationProcessor(
+ config, client, base_msg, out, progress_lock,
+ progress_manager.save,
+ lambda idx, actual_num, content_hash, output_file=None, status="completed", **kwargs: progress_manager.update(idx, actual_num, content_hash, output_file, status, **kwargs),
+ check_stop,
+ image_translator,
+ is_text_file=is_text_file
+ )
+
+ total_to_process = len(chapters_to_translate)
+ processed = 0
+
+ # Apply conservative batching setting
+ batch_multiplier = 3 if os.getenv('CONSERVATIVE_BATCHING', '0') == '1' else 1
+ batch_group_size = config.BATCH_SIZE * batch_multiplier
+
+ if batch_multiplier > 1:
+ print(f"📦 Using conservative batching: {batch_group_size} chapters per group, {config.BATCH_SIZE} parallel")
+ else:
+ print(f"📦 Using direct batching (default): {batch_group_size} chapters per group, {config.BATCH_SIZE} parallel")
+
+ with concurrent.futures.ThreadPoolExecutor(max_workers=config.BATCH_SIZE) as executor:
+ for batch_start in range(0, total_to_process, batch_group_size):
+ if check_stop():
+ print("❌ Translation stopped during parallel processing")
+ executor.shutdown(wait=False)
+ return
+
+ batch_end = min(batch_start + batch_group_size, total_to_process)
+ current_batch = chapters_to_translate[batch_start:batch_end]
+
+ batch_number = (batch_start // batch_group_size) + 1
+ print(f"\n📦 Submitting batch {batch_number}: {len(current_batch)} chapters")
+
+ future_to_chapter = {
+ executor.submit(batch_processor.process_single_chapter, chapter_data): chapter_data
+ for chapter_data in current_batch
+ }
+
+ active_count = 0
+ completed_in_batch = 0
+ failed_in_batch = 0
+
+ for future in concurrent.futures.as_completed(future_to_chapter):
+ if check_stop():
+ print("❌ Translation stopped")
+ executor.shutdown(wait=False)
+ return
+
+ chapter_data = future_to_chapter[future]
+ idx, chapter = chapter_data
+
+ try:
+ success, chap_num = future.result()
+ if success:
+ completed_in_batch += 1
+ print(f"✅ Chapter {chap_num} done ({completed_in_batch + failed_in_batch}/{len(current_batch)} in batch)")
+ else:
+ failed_in_batch += 1
+ print(f"❌ Chapter {chap_num} failed ({completed_in_batch + failed_in_batch}/{len(current_batch)} in batch)")
+ except Exception as e:
+ failed_in_batch += 1
+ print(f"❌ Chapter thread error: {e}")
+
+ processed += 1
+
+ progress_percent = (processed / total_to_process) * 100
+ print(f"📊 Overall Progress: {processed}/{total_to_process} ({progress_percent:.1f}%)")
+
+ print(f"\n📦 Batch Summary:")
+ print(f" ✅ Successful: {completed_in_batch}")
+ print(f" ❌ Failed: {failed_in_batch}")
+
+ if batch_end < total_to_process:
+ print(f"⏳ Waiting {config.DELAY}s before next batch...")
+ time.sleep(config.DELAY)
+
+ chapters_completed = batch_processor.chapters_completed
+ chunks_completed = batch_processor.chunks_completed
+
+ print(f"\n🎉 Parallel translation complete!")
+ print(f" Total chapters processed: {processed}")
+
+ # Count qa_failed chapters correctly
+ qa_failed_count = 0
+ actual_successful = 0
+
+ for idx, c in enumerate(chapters):
+ # Get the chapter's actual number
+ if (is_text_file and c.get('is_chunk', False) and isinstance(c['num'], float)):
+ actual_num = c['num']
+ else:
+ actual_num = c.get('actual_chapter_num', c['num'])
+
+ # Check if this chapter was processed and has qa_failed status
+ content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
+
+ # Check if this chapter exists in progress
+ chapter_info = progress_manager.prog["chapters"].get(content_hash, {})
+ status = chapter_info.get("status")
+
+ if status == "qa_failed":
+ qa_failed_count += 1
+ elif status == "completed":
+ actual_successful += 1
+
+ # Correct the displayed counts
+ print(f" Successful: {actual_successful}")
+ if qa_failed_count > 0:
+ print(f"\n⚠️ {qa_failed_count} chapters failed due to content policy violations:")
+ qa_failed_chapters = []
+ for idx, c in enumerate(chapters):
+ if (is_text_file and c.get('is_chunk', False) and isinstance(c['num'], float)):
+ actual_num = c['num']
+ else:
+ actual_num = c.get('actual_chapter_num', c['num'])
+
+ content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
+ chapter_info = progress_manager.prog["chapters"].get(content_hash, {})
+ if chapter_info.get("status") == "qa_failed":
+ qa_failed_chapters.append(actual_num)
+
+ print(f" Failed chapters: {', '.join(map(str, sorted(qa_failed_chapters)))}")
+
+ # Stop translation completely after batch mode
+ print("\n📌 Batch translation completed.")
+
+ elif not config.BATCH_TRANSLATION:
+ translation_processor = TranslationProcessor(config, client, out, log_callback, check_stop, uses_zero_based, is_text_file)
+
+ if config.DUPLICATE_DETECTION_MODE == 'ai-hunter':
+ # Build the main config from environment variables and config object
+ main_config = {
+ 'duplicate_lookback_chapters': config.DUPLICATE_LOOKBACK_CHAPTERS,
+ 'duplicate_detection_mode': config.DUPLICATE_DETECTION_MODE,
+ }
+
+ # Check if AI Hunter config was passed via environment variable
+ ai_hunter_config_str = os.getenv('AI_HUNTER_CONFIG')
+ if ai_hunter_config_str:
+ try:
+ ai_hunter_config = json.loads(ai_hunter_config_str)
+ main_config['ai_hunter_config'] = ai_hunter_config
+ print("🤖 AI Hunter: Loaded configuration from environment")
+ except json.JSONDecodeError:
+ print("⚠️ AI Hunter: Failed to parse AI_HUNTER_CONFIG from environment")
+
+ # If no AI Hunter config in environment, try to load from file as fallback
+ if 'ai_hunter_config' not in main_config:
+ # Try multiple locations for config.json
+ config_paths = [
+ os.path.join(os.getcwd(), 'config.json'),
+ os.path.join(out, '..', 'config.json'),
+ ]
+
+ if getattr(sys, 'frozen', False):
+ config_paths.append(os.path.join(os.path.dirname(sys.executable), 'config.json'))
+ else:
+ script_dir = os.path.dirname(os.path.abspath(__file__))
+ config_paths.extend([
+ os.path.join(script_dir, 'config.json'),
+ os.path.join(os.path.dirname(script_dir), 'config.json')
+ ])
+
+ for config_path in config_paths:
+ if os.path.exists(config_path):
+ try:
+ with open(config_path, 'r', encoding='utf-8') as f:
+ file_config = json.load(f)
+ if 'ai_hunter_config' in file_config:
+ main_config['ai_hunter_config'] = file_config['ai_hunter_config']
+ print(f"🤖 AI Hunter: Loaded configuration from {config_path}")
+ break
+ except Exception as e:
+ print(f"⚠️ Failed to load config from {config_path}: {e}")
+
+ # Always create and inject the improved AI Hunter when ai-hunter mode is selected
+ ai_hunter = ImprovedAIHunterDetection(main_config)
+
+ # The TranslationProcessor class has a method that checks for duplicates
+ # We need to replace it with our enhanced AI Hunter
+
+ # Create a wrapper to match the expected signature
+ def enhanced_duplicate_check(self, result, idx, prog, out, actual_num=None):
+ # If actual_num is not provided, try to get it from progress
+ if actual_num is None:
+ # Look for the chapter being processed
+ for ch_key, ch_info in prog.get("chapters", {}).items():
+ if ch_info.get("chapter_idx") == idx:
+ actual_num = ch_info.get("actual_num", idx + 1)
+ break
+
+ # Fallback to idx+1 if not found
+ if actual_num is None:
+ actual_num = idx + 1
+
+ return ai_hunter.detect_duplicate_ai_hunter_enhanced(result, idx, prog, out, actual_num)
+
+ # Bind the enhanced method to the processor instance
+ translation_processor.check_duplicate_content = enhanced_duplicate_check.__get__(translation_processor, TranslationProcessor)
+
+ print("🤖 AI Hunter: Using enhanced detection with configurable thresholds")
+
+ # First pass: set actual chapter numbers respecting the config
+ for idx, c in enumerate(chapters):
+ raw_num = FileUtilities.extract_actual_chapter_number(c, patterns=None, config=config)
+ #print(f"[DEBUG] Extracted raw_num={raw_num} from {c.get('original_basename', 'unknown')}")
+
+
+ # Apply offset if configured
+ offset = config.CHAPTER_NUMBER_OFFSET if hasattr(config, 'CHAPTER_NUMBER_OFFSET') else 0
+ raw_num += offset
+
+ if config.DISABLE_ZERO_DETECTION:
+ # Use raw numbers without adjustment
+ c['actual_chapter_num'] = raw_num
+ c['raw_chapter_num'] = raw_num
+ c['zero_adjusted'] = False
+ else:
+ # Store raw number
+ c['raw_chapter_num'] = raw_num
+ # Apply 0-based adjustment if detected
+ if uses_zero_based:
+ c['actual_chapter_num'] = raw_num + 1
+ c['zero_adjusted'] = True
+ else:
+ c['actual_chapter_num'] = raw_num
+ c['zero_adjusted'] = False
+
+ # Second pass: process chapters
+ for idx, c in enumerate(chapters):
+ chap_num = c["num"]
+
+ # Check if this is a pre-split text chunk with decimal number
+ if (is_text_file and c.get('is_chunk', False) and isinstance(c['num'], float)):
+ actual_num = c['num'] # Preserve the decimal for text files only
+ else:
+ actual_num = c.get('actual_chapter_num', c['num'])
+ content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
+
+ if start is not None and not (start <= actual_num <= end):
+ #print(f"[SKIP] Chapter {actual_num} (file: {c.get('original_basename', 'unknown')}) outside range {start}-{end}")
+ continue
+
+ needs_translation, skip_reason, existing_file = progress_manager.check_chapter_status(
+ idx, actual_num, content_hash, out, c # Pass the chapter object
+ )
+ # Add explicit file check for supposedly completed chapters
+ if not needs_translation and existing_file:
+ file_path = os.path.join(out, existing_file)
+ if not os.path.exists(file_path):
+ print(f"⚠️ Output file missing for chapter {actual_num}: {existing_file}")
+ needs_translation = True
+ skip_reason = None
+ # Update status to file_missing
+ progress_manager.update(idx, actual_num, content_hash, None, status="file_missing")
+ progress_manager.save()
+ if not needs_translation:
+ # Modify skip_reason to use appropriate terminology
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+
+ # Replace "Chapter" with appropriate terminology in skip_reason
+ skip_reason_modified = skip_reason.replace("Chapter", terminology)
+ print(f"[SKIP] {skip_reason_modified}")
+ continue
+
+ chapter_position = f"{chapters_completed + 1}/{chapters_to_process}"
+
+ # Determine if this is a text file
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+
+ # Determine file reference based on type
+ if c.get('is_chunk', False):
+ file_ref = f"Section_{c['num']}"
+ else:
+ file_ref = c.get('original_basename', f'{terminology}_{actual_num}')
+
+ print(f"\n🔄 Processing #{idx+1}/{total_chapters} (Actual: {terminology} {actual_num}) ({chapter_position} to translate): {c['title']} [File: {file_ref}]")
+
+ chunk_context_manager.start_chapter(chap_num, c['title'])
+
+ has_images = c.get('has_images', False)
+ has_meaningful_text = ContentProcessor.is_meaningful_text_content(c["body"])
+ text_size = c.get('file_size', 0)
+
+ is_empty_chapter = (not has_images and text_size < 10)
+ is_image_only_chapter = (has_images and not has_meaningful_text)
+ is_mixed_content = (has_images and has_meaningful_text)
+ is_text_only = (not has_images and has_meaningful_text)
+
+ if is_empty_chapter:
+ print(f"📄 Empty chapter {actual_num} detected")
+
+ # Create filename for empty chapter
+ if isinstance(c['num'], float):
+ fname = FileUtilities.create_chapter_filename(c, c['num'])
+ else:
+ fname = FileUtilities.create_chapter_filename(c, actual_num)
+
+ # Save original content
+ with open(os.path.join(out, fname), 'w', encoding='utf-8') as f:
+ f.write(c["body"])
+
+ # Update progress tracking
+ progress_manager.update(idx, actual_num, content_hash, fname, status="completed_empty")
+ progress_manager.save()
+ chapters_completed += 1
+
+ # CRITICAL: Skip translation!
+ continue
+
+ elif is_image_only_chapter:
+ print(f"📸 Image-only chapter: {c.get('image_count', 0)} images")
+
+ translated_html = c["body"]
+ image_translations = {}
+
+ # Step 1: Process images if image translation is enabled
+ if image_translator and config.ENABLE_IMAGE_TRANSLATION:
+ print(f"🖼️ Translating {c.get('image_count', 0)} images...")
+ image_translator.set_current_chapter(chap_num)
+
+ translated_html, image_translations = process_chapter_images(
+ c["body"],
+ actual_num,
+ image_translator,
+ check_stop
+ )
+
+ if image_translations:
+ print(f"✅ Translated {len(image_translations)} images")
+
+ # Step 2: Check for headers/titles that need translation
+ from bs4 import BeautifulSoup
+ soup = BeautifulSoup(c["body"], 'html.parser')
+
+ # Look for headers
+ headers = soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'title'])
+
+ # If we have headers, we should translate them even in "image-only" chapters
+ if headers and any(h.get_text(strip=True) for h in headers):
+ print(f"📝 Found headers to translate in image-only chapter")
+
+ # Create a minimal HTML with just the headers for translation
+ headers_html = ""
+ for header in headers:
+ if header.get_text(strip=True):
+ headers_html += str(header) + "\n"
+
+ if headers_html:
+ print(f"📤 Translating chapter headers...")
+
+ # Send just the headers for translation
+ header_msgs = base_msg + [{"role": "user", "content": headers_html}]
+
+ # Use the standard filename
+ fname = FileUtilities.create_chapter_filename(c, actual_num)
+ client.set_output_filename(fname)
+
+ # Simple API call for headers
+ header_result, _ = client.send(
+ header_msgs,
+ temperature=config.TEMP,
+ max_tokens=config.MAX_OUTPUT_TOKENS
+ )
+
+ if header_result:
+ # Clean the result
+ header_result = re.sub(r"^```(?:html)?\s*\n?", "", header_result, count=1, flags=re.MULTILINE)
+ header_result = re.sub(r"\n?```\s*$", "", header_result, count=1, flags=re.MULTILINE)
+
+ # Parse both the translated headers and the original body
+ soup_headers = BeautifulSoup(header_result, 'html.parser')
+ soup_body = BeautifulSoup(translated_html, 'html.parser')
+
+ # Replace headers in the body with translated versions
+ translated_headers = soup_headers.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'title'])
+ original_headers = soup_body.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'title'])
+
+ # Match and replace headers
+ for orig, trans in zip(original_headers, translated_headers):
+ if trans and trans.get_text(strip=True):
+ orig.string = trans.get_text(strip=True)
+
+ translated_html = str(soup_body)
+ print(f"✅ Headers translated successfully")
+ status = "completed"
+ else:
+ print(f"⚠️ Failed to translate headers")
+ status = "completed_image_only"
+ else:
+ status = "completed_image_only"
+ else:
+ print(f"ℹ️ No headers found to translate")
+ status = "completed_image_only"
+
+ # Step 3: Save with correct filename
+ fname = FileUtilities.create_chapter_filename(c, actual_num)
+
+ with open(os.path.join(out, fname), 'w', encoding='utf-8') as f:
+ f.write(translated_html)
+
+ print(f"[Chapter {idx+1}/{total_chapters}] ✅ Saved image-only chapter")
+ progress_manager.update(idx, actual_num, content_hash, fname, status=status)
+ progress_manager.save()
+ chapters_completed += 1
+ continue
+
+ else:
+ # Set default text to translate
+ text_to_translate = c["body"]
+ image_translations = {}
+ if is_mixed_content and image_translator and config.ENABLE_IMAGE_TRANSLATION:
+ print(f"🖼️ Processing {c.get('image_count', 0)} images first...")
+
+ print(f"[DEBUG] Content before image processing (first 200 chars):")
+ print(c["body"][:200])
+ print(f"[DEBUG] Has h1 tags: {'' in c['body']}")
+ print(f"[DEBUG] Has h2 tags: {'
', '
', '', '
', '+ 'smarty-pants', # Smart quotes and dashes + 'tables', # Markdown tables + ]) + + # Post-process to ensure proper paragraph structure + if not '
' in html: + # If markdown2 didn't create paragraphs, wrap content + lines = html.split('\n') + processed_lines = [] + for line in lines: + line = line.strip() + if line and not line.startswith('<') and not line.endswith('>'): + processed_lines.append(f'
{line}
') + elif line: + processed_lines.append(line) + html = '\n'.join(processed_lines) + + return html + + except ImportError: + print("⚠️ markdown2 not available, using fallback HTML conversion") + + # Fallback: Manual markdown-to-HTML conversion + lines = plain_text.strip().split('\n') + html_parts = [] + in_code_block = False + code_block_content = [] + + for line in lines: + # Handle code blocks + if line.strip().startswith('```'): + if in_code_block: + # End code block + html_parts.append('' + '\n'.join(code_block_content) + '') + continue + + # Check for markdown headers + if line.startswith('#'): + match = re.match(r'^(#+)\s*(.+)$', line) + if match: + level = min(len(match.group(1)), 6) + header_text = match.group(2).strip() + html_parts.append(f'
{quote_text}') + continue + + # Check for lists + if re.match(r'^[*\-+]\s+', line): + list_text = re.sub(r'^[*\-+]\s+', '', line) + html_parts.append(f'
\1', line)
+
+ # Regular paragraph
+ html_parts.append(f'{line}
') + + # Post-process lists to wrap in ul/ol tags + final_html = [] + in_list = False + list_type = None + + for part in html_parts: + if part.startswith('' in c['body']}")
+ print(f"[DEBUG] Has h2 tags: {'' in c['body']}")
+
+ image_translator.set_current_chapter(chap_num)
+
+ # Store the original body before processing
+ original_body = c["body"]
+
+ # Calculate original chapter tokens before modification
+ original_chapter_tokens = chapter_splitter.count_tokens(original_body)
+
+ # Process images and get body with translations
+ body_with_images, image_translations = process_chapter_images(
+ c["body"],
+ actual_num,
+ image_translator,
+ check_stop
+ )
+
+ if image_translations:
+ print(f"✅ Translated {len(image_translations)} images")
+
+ # Store the body with images for later merging
+ c["body_with_images"] = c["body"]
+
+ # For chapters with only images and title, we still need to translate the title
+ # Extract clean text for translation from ORIGINAL body
+ from bs4 import BeautifulSoup
+ soup_clean = BeautifulSoup(original_body, 'html.parser')
+
+ # Remove images from the original to get pure text
+ for img in soup_clean.find_all('img'):
+ img.decompose()
+
+ # Set clean text for translation - use prettify() or str() on the full document
+ c["body"] = str(soup_clean) if soup_clean.body else original_body
+
+ # If there's no meaningful text content after removing images,
+ # the text translation will just translate the title, which is correct
+ print(f" 📝 Clean text for translation: {len(c['body'])} chars")
+
+ # Update text_size to reflect actual text to translate
+ text_size = len(c["body"])
+
+ # Recalculate the actual token count for clean text
+ actual_text_tokens = chapter_splitter.count_tokens(c["body"])
+ print(f" 📊 Actual text tokens: {actual_text_tokens} (was counting {original_chapter_tokens} with images)")
+ else:
+ print(f"ℹ️ No translatable text found in images")
+ # Keep original body if no image translations
+ c["body"] = original_body
+
+ print(f"📖 Translating text content ({text_size} characters)")
+ progress_manager.update(idx, actual_num, content_hash, output_file=None, status="in_progress")
+ progress_manager.save()
+
+ # Apply ignore filtering to the content before chunk splitting
+ batch_translate_active = os.getenv('BATCH_TRANSLATE_HEADERS', '0') == '1'
+ ignore_title_tag = os.getenv('IGNORE_TITLE', '0') == '1' and batch_translate_active
+ ignore_header_tags = os.getenv('IGNORE_HEADER', '0') == '1' and batch_translate_active
+
+ if (ignore_title_tag or ignore_header_tags) and c["body"]:
+ from bs4 import BeautifulSoup
+ content_soup = BeautifulSoup(c["body"], 'html.parser')
+
+ # Remove title tags if ignored
+ if ignore_title_tag:
+ for title_tag in content_soup.find_all('title'):
+ title_tag.decompose()
+
+ # Remove header tags if ignored
+ if ignore_header_tags:
+ for header_tag in content_soup.find_all(['h1', 'h2', 'h3']):
+ header_tag.decompose()
+
+ c["body"] = str(content_soup) # Update the chapter body
+
+ # Check if this chapter is already a chunk from text file splitting
+ if c.get('is_chunk', False):
+ # This is already a pre-split chunk, but still check if it needs further splitting
+ # Calculate based on OUTPUT limit only
+ max_output_tokens = config.MAX_OUTPUT_TOKENS
+ safety_margin_output = 500
+
+ # CJK to English typically compresses to 0.7-0.9x
+ compression_factor = config.COMPRESSION_FACTOR
+ available_tokens = int((max_output_tokens - safety_margin_output) / compression_factor)
+
+ # Ensure minimum
+ available_tokens = max(available_tokens, 1000)
+
+ print(f"📊 Chunk size: {available_tokens:,} tokens (based on {max_output_tokens:,} output limit, compression: {compression_factor})")
+
+ chapter_tokens = chapter_splitter.count_tokens(c["body"])
+
+ if chapter_tokens > available_tokens:
+ # Even pre-split chunks might need further splitting
+ chunks = chapter_splitter.split_chapter(c["body"], available_tokens)
+ print(f"📄 Section {c['num']} (pre-split from text file) needs further splitting into {len(chunks)} chunks")
+ else:
+ chunks = [(c["body"], 1, 1)]
+ print(f"📄 Section {c['num']} (pre-split from text file)")
+ else:
+ # Normal splitting logic for non-text files
+ # Calculate based on OUTPUT limit only
+ max_output_tokens = config.MAX_OUTPUT_TOKENS
+ safety_margin_output = 500
+
+ # CJK to English typically compresses to 0.7-0.9x
+ compression_factor = config.COMPRESSION_FACTOR
+ available_tokens = int((max_output_tokens - safety_margin_output) / compression_factor)
+
+ # Ensure minimum
+ available_tokens = max(available_tokens, 1000)
+
+ print(f"📊 Chunk size: {available_tokens:,} tokens (based on {max_output_tokens:,} output limit, compression: {compression_factor})")
+
+ chunks = chapter_splitter.split_chapter(c["body"], available_tokens)
+
+ # Use consistent terminology
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+ print(f"📄 {terminology} will be processed in {len(chunks)} chunk(s)")
+
+ # Recalculate tokens on the actual text to be translated
+ actual_chapter_tokens = chapter_splitter.count_tokens(c["body"])
+
+ if len(chunks) > 1:
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+ print(f" ℹ️ {terminology} size: {actual_chapter_tokens:,} tokens (limit: {available_tokens:,} tokens per chunk)")
+ else:
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+ print(f" ℹ️ {terminology} size: {actual_chapter_tokens:,} tokens (within limit of {available_tokens:,} tokens)")
+
+ chapter_key_str = str(idx)
+ if chapter_key_str not in progress_manager.prog["chapter_chunks"]:
+ progress_manager.prog["chapter_chunks"][chapter_key_str] = {
+ "total": len(chunks),
+ "completed": [],
+ "chunks": {}
+ }
+
+ progress_manager.prog["chapter_chunks"][chapter_key_str]["total"] = len(chunks)
+
+ translated_chunks = []
+
+ for chunk_html, chunk_idx, total_chunks in chunks:
+ chapter_key_str = content_hash
+ old_key_str = str(idx)
+
+ if chapter_key_str not in progress_manager.prog.get("chapter_chunks", {}) and old_key_str in progress_manager.prog.get("chapter_chunks", {}):
+ progress_manager.prog["chapter_chunks"][chapter_key_str] = progress_manager.prog["chapter_chunks"][old_key_str]
+ del progress_manager.prog["chapter_chunks"][old_key_str]
+ #print(f"[PROGRESS] Migrated chunks for chapter {chap_num} to new tracking system")
+
+ if chapter_key_str not in progress_manager.prog["chapter_chunks"]:
+ progress_manager.prog["chapter_chunks"][chapter_key_str] = {
+ "total": len(chunks),
+ "completed": [],
+ "chunks": {}
+ }
+
+ progress_manager.prog["chapter_chunks"][chapter_key_str]["total"] = len(chunks)
+
+ # Get chapter status to check for qa_failed
+ chapter_info = progress_manager.prog["chapters"].get(chapter_key_str, {})
+ chapter_status = chapter_info.get("status")
+
+ if chapter_status == "qa_failed":
+ # Force retranslation of qa_failed chapters
+ print(f" [RETRY] Chunk {chunk_idx}/{total_chunks} - retranslating due to QA failure")
+
+ if config.CONTEXTUAL and history_manager.will_reset_on_next_append(config.HIST_LIMIT):
+ print(f" 📌 History will reset after this chunk (current: {len(history_manager.load_history())//2}/{config.HIST_LIMIT} exchanges)")
+
+ if check_stop():
+ print(f"❌ Translation stopped during chapter {actual_num}, chunk {chunk_idx}")
+ return
+
+ current_chunk_number += 1
+
+ progress_percent = (current_chunk_number / total_chunks_needed) * 100 if total_chunks_needed > 0 else 0
+
+ if chunks_completed > 0:
+ elapsed_time = time.time() - translation_start_time
+ avg_time_per_chunk = elapsed_time / chunks_completed
+ remaining_chunks = total_chunks_needed - current_chunk_number + 1
+ eta_seconds = remaining_chunks * avg_time_per_chunk
+
+ eta_hours = int(eta_seconds // 3600)
+ eta_minutes = int((eta_seconds % 3600) // 60)
+ eta_str = f"{eta_hours}h {eta_minutes}m" if eta_hours > 0 else f"{eta_minutes}m"
+ else:
+ eta_str = "calculating..."
+
+ if total_chunks > 1:
+ print(f" 🔄 Translating chunk {chunk_idx}/{total_chunks} for #{idx+1} (Overall: {current_chunk_number}/{total_chunks_needed} - {progress_percent:.1f}% - ETA: {eta_str})")
+ print(f" ⏳ Chunk size: {len(chunk_html):,} characters (~{chapter_splitter.count_tokens(chunk_html):,} tokens)")
+ else:
+ # Determine terminology and file reference
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+
+ # Consistent file reference
+ if c.get('is_chunk', False):
+ file_ref = f"Section_{c['num']}"
+ else:
+ file_ref = c.get('original_basename', f'{terminology}_{actual_num}')
+
+ print(f" 📄 Translating {terminology.lower()} content (Overall: {current_chunk_number}/{total_chunks_needed} - {progress_percent:.1f}% - ETA: {eta_str}) [File: {file_ref}]")
+ print(f" 📊 {terminology} {actual_num} size: {len(chunk_html):,} characters (~{chapter_splitter.count_tokens(chunk_html):,} tokens)")
+
+ print(f" ℹ️ This may take 30-60 seconds. Stop will take effect after completion.")
+
+ if log_callback:
+ if hasattr(log_callback, '__self__') and hasattr(log_callback.__self__, 'append_chunk_progress'):
+ if total_chunks == 1:
+ # Determine terminology based on source type
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+
+ log_callback.__self__.append_chunk_progress(
+ 1, 1, "text",
+ f"{terminology} {actual_num}",
+ overall_current=current_chunk_number,
+ overall_total=total_chunks_needed,
+ extra_info=f"{len(chunk_html):,} chars"
+ )
+ else:
+ log_callback.__self__.append_chunk_progress(
+ chunk_idx,
+ total_chunks,
+ "text",
+ f"{terminology} {actual_num}",
+ overall_current=current_chunk_number,
+ overall_total=total_chunks_needed
+ )
+ else:
+ # Determine terminology based on source type
+ is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
+ terminology = "Section" if is_text_source else "Chapter"
+ terminology_lower = "section" if is_text_source else "chapter"
+
+ if total_chunks == 1:
+ log_callback(f"📄 Processing {terminology} {actual_num} ({chapters_completed + 1}/{chapters_to_process}) - {progress_percent:.1f}% complete")
+ else:
+ log_callback(f"📄 processing chunk {chunk_idx}/{total_chunks} for {terminology_lower} {actual_num} - {progress_percent:.1f}% complete")
+
+ # Get custom chunk prompt template from environment
+ chunk_prompt_template = os.getenv("TRANSLATION_CHUNK_PROMPT", "[PART {chunk_idx}/{total_chunks}]\n{chunk_html}")
+
+ if total_chunks > 1:
+ user_prompt = chunk_prompt_template.format(
+ chunk_idx=chunk_idx,
+ total_chunks=total_chunks,
+ chunk_html=chunk_html
+ )
+ else:
+ user_prompt = chunk_html
+
+ if config.CONTEXTUAL:
+ history = history_manager.load_history()
+ trimmed = history[-config.HIST_LIMIT*2:]
+ chunk_context = chunk_context_manager.get_context_messages(limit=2)
+ else:
+ history = [] # Set empty history when not contextual
+ trimmed = []
+ chunk_context = []
+
+ # Build the current system prompt from the original each time, and append the last summary block if present
+ current_system_content = original_system_prompt
+ if config.USE_ROLLING_SUMMARY and last_summary_block_text:
+ current_system_content = (
+ current_system_content
+ + "\n\n[Rolling Summary of Previous Chapter]\n"
+ + "(For AI: Use as context only; do not include in output)\n"
+ + last_summary_block_text
+ + "\n[End of Rolling Summary]"
+ )
+ current_base = [{"role": "system", "content": current_system_content}]
+ # If we have a prepared rolling summary from previous chapter, include it as a separate message (do NOT mutate system prompt)
+ summary_msgs_list = []
+ if config.USE_ROLLING_SUMMARY and last_summary_block_text:
+ summary_msgs_list = [{
+ "role": os.getenv("SUMMARY_ROLE", "user"),
+ "content": (
+ "CONTEXT ONLY - DO NOT INCLUDE IN TRANSLATION:\n"
+ "[MEMORY] Previous context summary:\n\n"
+ f"{last_summary_block_text}\n\n"
+ "[END MEMORY]\n"
+ "END OF CONTEXT - BEGIN ACTUAL CONTENT TO TRANSLATE:"
+ )
+ }]
+ msgs = current_base + summary_msgs_list + chunk_context + trimmed + [{"role": "user", "content": user_prompt}]
+
+ c['__index'] = idx
+ c['__progress'] = progress_manager.prog
+ c['history_manager'] = history_manager
+
+ result, finish_reason = translation_processor.translate_with_retry(
+ msgs, chunk_html, c, chunk_idx, total_chunks
+ )
+
+ if result is None:
+ progress_manager.update(idx, actual_num, content_hash, output_file=None, status="failed")
+ progress_manager.save()
+ continue
+
+ if config.REMOVE_AI_ARTIFACTS:
+ result = ContentProcessor.clean_ai_artifacts(result, True)
+
+ if config.EMERGENCY_RESTORE:
+ result = ContentProcessor.emergency_restore_paragraphs(result, chunk_html)
+
+ if config.REMOVE_AI_ARTIFACTS:
+ lines = result.split('\n')
+
+ json_line_count = 0
+ for i, line in enumerate(lines[:5]):
+ if line.strip() and any(pattern in line for pattern in [
+ '"role":', '"content":', '"messages":',
+ '{"role"', '{"content"', '[{', '}]'
+ ]):
+ json_line_count = i + 1
+ else:
+ break
+
+ if json_line_count > 0 and json_line_count < len(lines):
+ remaining = '\n'.join(lines[json_line_count:])
+ if remaining.strip() and len(remaining) > 100:
+ result = remaining
+ print(f"✂️ Removed {json_line_count} lines of JSON artifacts")
+
+ result = re.sub(r'\[PART \d+/\d+\]\s*', '', result, flags=re.IGNORECASE)
+
+ translated_chunks.append((result, chunk_idx, total_chunks))
+
+ chunk_context_manager.add_chunk(user_prompt, result, chunk_idx, total_chunks)
+
+ progress_manager.prog["chapter_chunks"][chapter_key_str]["completed"].append(chunk_idx)
+ progress_manager.prog["chapter_chunks"][chapter_key_str]["chunks"][str(chunk_idx)] = result
+ progress_manager.save()
+
+ chunks_completed += 1
+
+ will_reset = history_manager.will_reset_on_next_append(
+ config.HIST_LIMIT if config.CONTEXTUAL else 0,
+ config.TRANSLATION_HISTORY_ROLLING
+ )
+
+
+ history = history_manager.append_to_history(
+ user_prompt,
+ result,
+ config.HIST_LIMIT if config.CONTEXTUAL else 0,
+ reset_on_limit=True,
+ rolling_window=config.TRANSLATION_HISTORY_ROLLING
+ )
+
+ if chunk_idx < total_chunks:
+ # Handle float delays while checking for stop
+ full_seconds = int(config.DELAY)
+ fractional_second = config.DELAY - full_seconds
+
+ # Check stop signal every second for full seconds
+ for i in range(full_seconds):
+ if check_stop():
+ print("❌ Translation stopped during delay")
+ return
+ time.sleep(1)
+
+ # Handle the fractional part if any
+ if fractional_second > 0:
+ if check_stop():
+ print("❌ Translation stopped during delay")
+ return
+ time.sleep(fractional_second)
+
+ if check_stop():
+ print(f"❌ Translation stopped before saving chapter {actual_num}")
+ return
+
+ if len(translated_chunks) > 1:
+ print(f" 📎 Merging {len(translated_chunks)} chunks...")
+ translated_chunks.sort(key=lambda x: x[1])
+ merged_result = chapter_splitter.merge_translated_chunks(translated_chunks)
+ else:
+ merged_result = translated_chunks[0][0] if translated_chunks else ""
+
+ if config.CONTEXTUAL and len(translated_chunks) > 1:
+ user_summary, assistant_summary = chunk_context_manager.get_summary_for_history()
+
+ if user_summary and assistant_summary:
+ history_manager.append_to_history(
+ user_summary,
+ assistant_summary,
+ config.HIST_LIMIT,
+ reset_on_limit=False,
+ rolling_window=config.TRANSLATION_HISTORY_ROLLING
+ )
+ print(f" 📝 Added chapter summary to history")
+
+ chunk_context_manager.clear()
+
+ # For text file chunks, ensure we pass the decimal number
+ if is_text_file and c.get('is_chunk', False) and isinstance(c.get('num'), float):
+ fname = FileUtilities.create_chapter_filename(c, c['num']) # Use the decimal num directly
+ else:
+ fname = FileUtilities.create_chapter_filename(c, actual_num)
+
+ client.set_output_filename(fname)
+ cleaned = re.sub(r"^```(?:html)?\s*\n?", "", merged_result, count=1, flags=re.MULTILINE)
+ cleaned = re.sub(r"\n?```\s*$", "", cleaned, count=1, flags=re.MULTILINE)
+
+ cleaned = ContentProcessor.clean_ai_artifacts(cleaned, remove_artifacts=config.REMOVE_AI_ARTIFACTS)
+
+ if is_mixed_content and image_translations:
+ print(f"🔀 Merging {len(image_translations)} image translations with text...")
+ from bs4 import BeautifulSoup
+ # Parse the translated text (which has the translated title/header)
+ soup_translated = BeautifulSoup(cleaned, 'html.parser')
+
+ # For each image translation, insert it into the document
+ for img_path, translation_html in image_translations.items():
+ if translation_html and ' 0:
+ combined.write(f"\n\n{'='*50}\n\n")
+
+ # Write the original chapter title (without Part X/Y suffix)
+ original_title = chapter_data['title']
+ # Remove the (Part X/Y) suffix if present
+ if ' (Part ' in original_title:
+ original_title = original_title.split(' (Part ')[0]
+
+ combined.write(f"{original_title}\n\n")
+
+ # Add the chunk content
+ combined.write(content)
+
+ # Add spacing between chunks of the same chapter
+ if chunk_idx < total_chunks:
+ combined.write("\n\n")
+ else:
+ # This is a standalone chapter
+ current_main_chapter = chapter_data['num']
+
+ # Add separator if not first chapter
+ if i > 0:
+ combined.write(f"\n\n{'='*50}\n\n")
+
+ # Write the chapter title
+ combined.write(f"{chapter_data['title']}\n\n")
+
+ # Add the content
+ combined.write(content)
+
+ print(f" • Combined file with preserved sections: {combined_path}")
+
+ total_time = time.time() - translation_start_time
+ hours = int(total_time // 3600)
+ minutes = int((total_time % 3600) // 60)
+ seconds = int(total_time % 60)
+
+ print(f"\n⏱️ Total translation time: {hours}h {minutes}m {seconds}s")
+ print(f"📊 Chapters completed: {chapters_completed}")
+ print(f"✅ Text file translation complete!")
+
+ if log_callback:
+ log_callback(f"✅ Text file translation complete! Created {combined_path}")
+
+ except Exception as e:
+ print(f"❌ Error creating combined text file: {e}")
+ if log_callback:
+ log_callback(f"❌ Error creating combined text file: {e}")
+ else:
+ print("🔍 Checking for translated chapters...")
+ # Respect retain extension toggle: if enabled, don't look for response_ prefix
+ if should_retain_source_extension():
+ response_files = [f for f in os.listdir(out) if f.endswith('.html') and not f.startswith('chapter_')]
+ else:
+ response_files = [f for f in os.listdir(out) if f.startswith('response_') and f.endswith('.html')]
+ chapter_files = [f for f in os.listdir(out) if f.startswith('chapter_') and f.endswith('.html')]
+
+ if not response_files and chapter_files:
+ if should_retain_source_extension():
+ print(f"⚠️ No translated files found, but {len(chapter_files)} original chapters exist")
+ print("ℹ️ Retain-source-extension mode is ON: skipping placeholder creation and using original files for EPUB compilation.")
+ else:
+ print(f"⚠️ No translated files found, but {len(chapter_files)} original chapters exist")
+ print("📝 Creating placeholder response files for EPUB compilation...")
+
+ for chapter_file in chapter_files:
+ response_file = chapter_file.replace('chapter_', 'response_', 1)
+ src = os.path.join(out, chapter_file)
+ dst = os.path.join(out, response_file)
+
+ try:
+ with open(src, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ soup = BeautifulSoup(content, 'html.parser')
+ notice = soup.new_tag('p')
+ notice.string = "[Note: This chapter could not be translated - showing original content]"
+ notice['style'] = "color: red; font-style: italic;"
+
+ if soup.body:
+ soup.body.insert(0, notice)
+
+ with open(dst, 'w', encoding='utf-8') as f:
+ f.write(str(soup))
+
+ except Exception as e:
+ print(f"⚠️ Error processing {chapter_file}: {e}")
+ try:
+ shutil.copy2(src, dst)
+ except:
+ pass
+
+ print(f"✅ Created {len(chapter_files)} placeholder response files")
+ print("⚠️ Note: The EPUB will contain untranslated content")
+
+ print("📘 Building final EPUB…")
+ try:
+ from epub_converter import fallback_compile_epub
+ fallback_compile_epub(out, log_callback=log_callback)
+ print("✅ All done: your final EPUB is in", out)
+
+ total_time = time.time() - translation_start_time
+ hours = int(total_time // 3600)
+ minutes = int((total_time % 3600) // 60)
+ seconds = int(total_time % 60)
+
+ print(f"\n📊 Translation Statistics:")
+ print(f" • Total chunks processed: {chunks_completed}")
+ print(f" • Total time: {hours}h {minutes}m {seconds}s")
+ if chunks_completed > 0:
+ avg_time = total_time / chunks_completed
+ print(f" • Average time per chunk: {avg_time:.1f} seconds")
+
+ stats = progress_manager.get_stats(out)
+ print(f"\n📊 Progress Tracking Summary:")
+ print(f" • Total chapters tracked: {stats['total_tracked']}")
+ print(f" • Successfully completed: {stats['completed']}")
+ print(f" • Missing files: {stats['missing_files']}")
+ print(f" • In progress: {stats['in_progress']}")
+
+ except Exception as e:
+ print("❌ EPUB build failed:", e)
+
+ print("TRANSLATION_COMPLETE_SIGNAL")
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/ai_hunter_enhanced.py b/ai_hunter_enhanced.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d244296a815022b35c326963af2143fd26c0cc8
--- /dev/null
+++ b/ai_hunter_enhanced.py
@@ -0,0 +1,1385 @@
+# ai_hunter_enhanced.py
+# Combined AI Hunter configuration GUI and detection logic
+
+import tkinter as tk
+from tkinter import ttk
+import ttkbootstrap as tb
+import json
+import os
+import re
+import unicodedata
+from difflib import SequenceMatcher
+from collections import Counter
+
+class AIHunterConfigGUI:
+ """GUI for configuring AI Hunter detection parameters"""
+ def __init__(self, parent, config_dict, callback=None):
+ """
+ Initialize with reference to main config dictionary
+
+ Args:
+ parent: Parent window
+ config_dict: Reference to main translator config dictionary
+ callback: Function to call after saving
+ """
+ self.parent = parent
+ self.config = config_dict # Reference to main config
+ self.callback = callback
+ self.window = None
+
+ # Default AI Hunter settings structure
+ self.default_ai_hunter = {
+ 'enabled': True,
+ 'ai_hunter_max_workers': 1,
+ 'retry_attempts': 6,
+ 'disable_temperature_change': False,
+ 'sample_size': 3000,
+ 'thresholds': {
+ 'exact': 90,
+ 'text': 35,
+ 'semantic': 85,
+ 'structural': 85,
+ 'character': 90,
+ 'pattern': 80
+ },
+ 'weights': {
+ 'exact': 1.5,
+ 'text': 1.2,
+ 'semantic': 1.0,
+ 'structural': 1.0,
+ 'character': 0.8,
+ 'pattern': 0.8
+ },
+ 'detection_mode': 'weighted_average',
+ 'multi_method_requirements': {
+ 'methods_required': 3,
+ 'min_methods': ['semantic', 'structural']
+ },
+ 'preprocessing': {
+ 'remove_html_spacing': True,
+ 'normalize_unicode': True,
+ 'ignore_case': True,
+ 'remove_extra_whitespace': True
+ },
+ 'edge_filters': {
+ 'min_text_length': 500,
+ 'max_length_ratio': 1.3,
+ 'min_length_ratio': 0.7
+ },
+ 'language_detection': {
+ 'enabled': False,
+ 'target_language': 'english',
+ 'threshold_characters': 500,
+ 'languages': {
+ 'english': ['en'],
+ 'japanese': ['ja', 'jp'],
+ 'korean': ['ko', 'kr'],
+ 'chinese': ['zh', 'zh-cn', 'zh-tw'],
+ 'spanish': ['es'],
+ 'french': ['fr'],
+ 'german': ['de'],
+ 'russian': ['ru'],
+ 'arabic': ['ar'],
+ 'hindi': ['hi'],
+ 'portuguese': ['pt'],
+ 'italian': ['it'],
+ 'dutch': ['nl'],
+ 'thai': ['th'],
+ 'vietnamese': ['vi'],
+ 'turkish': ['tr'],
+ 'polish': ['pl'],
+ 'swedish': ['sv'],
+ 'danish': ['da'],
+ 'norwegian': ['no'],
+ 'finnish': ['fi']
+ }
+ }
+ }
+
+ # Initialize AI Hunter config in main config if not present
+ if 'ai_hunter_config' not in self.config:
+ self.config['ai_hunter_config'] = self.default_ai_hunter.copy()
+ else:
+ # Merge with defaults to ensure all keys exist
+ self.config['ai_hunter_config'] = self._merge_configs(
+ self.default_ai_hunter,
+ self.config['ai_hunter_config']
+ )
+
+ def _merge_configs(self, default, existing):
+ """Recursively merge existing config with defaults"""
+ result = default.copy()
+ for key, value in existing.items():
+ if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+ result[key] = self._merge_configs(result[key], value)
+ else:
+ result[key] = value
+ return result
+
+ def get_ai_config(self):
+ """Get AI Hunter configuration from main config"""
+ return self.config.get('ai_hunter_config', self.default_ai_hunter)
+
+ def show_ai_hunter_config(self):
+ """Display the AI Hunter configuration window with scrollbar using WindowManager"""
+ if self.window and self.window.winfo_exists():
+ self.window.lift()
+ return
+
+ # Import WindowManager if not already available
+ if not hasattr(self, 'wm'):
+ from translator_gui import WindowManager
+ import sys
+ import os
+ base_dir = getattr(sys, '_MEIPASS', os.path.dirname(os.path.abspath(__file__)))
+ self.wm = WindowManager(base_dir)
+
+ # Create scrollable dialog using WindowManager
+ dialog, scrollable_frame, canvas = self.wm.setup_scrollable(
+ self.parent,
+ "AI Hunter Configuration",
+ width=820,
+ height=None, # Will use default height
+ max_width_ratio=0.9,
+ max_height_ratio=0.85
+ )
+
+ self.window = dialog
+
+ # Create notebook inside scrollable frame
+ notebook = ttk.Notebook(scrollable_frame)
+ notebook.pack(fill='both', expand=True, padx=10, pady=10)
+
+ # Tab 1: Detection Thresholds
+ self.create_thresholds_tab(notebook)
+
+ # Tab 2: Detection Mode
+ self.create_mode_tab(notebook)
+
+ # Tab 3: Preprocessing
+ self.create_preprocessing_tab(notebook)
+
+ # Tab 4: Advanced Settings
+ self.create_advanced_tab(notebook)
+
+ # Buttons at the bottom (inside scrollable frame)
+ button_frame = tk.Frame(scrollable_frame)
+ button_frame.pack(fill='x', padx=10, pady=(10, 20))
+
+ tb.Button(button_frame, text="Save", command=self.apply_ai_hunter_settings,
+ bootstyle="success").pack(side='right', padx=5)
+ tb.Button(button_frame, text="Cancel", command=self.window.destroy,
+ bootstyle="secondary").pack(side='right')
+ tb.Button(button_frame, text="Reset to Defaults", command=self.reset_defaults,
+ bootstyle="warning").pack(side='left')
+
+ # Auto-resize and show
+ self.wm.auto_resize_dialog(dialog, canvas, max_width_ratio=0.9, max_height_ratio=1.1)
+
+ # Handle window close
+ dialog.protocol("WM_DELETE_WINDOW", lambda: [dialog._cleanup_scrolling(), dialog.destroy()])
+
+ def create_thresholds_tab(self, notebook):
+ """Create the thresholds configuration tab"""
+ frame = ttk.Frame(notebook)
+ notebook.add(frame, text="Detection Thresholds")
+
+ # Title
+ tk.Label(frame, text="Detection Method Thresholds",
+ font=('TkDefaultFont', 12, 'bold')).pack(pady=10)
+
+ tk.Label(frame, text="Higher values = fewer false positives (more strict)\n"
+ "Lower values = more false positives (more sensitive)",
+ font=('TkDefaultFont', 10), fg='gray').pack(pady=(0, 20))
+
+ # Threshold controls
+ self.threshold_vars = {}
+ threshold_frame = tk.Frame(frame)
+ threshold_frame.pack(fill='both', expand=True, padx=20)
+
+ descriptions = {
+ 'exact': 'Exact Text Match - Direct character-by-character comparison',
+ 'text': 'Smart Text Similarity - Intelligent text comparison with sampling',
+ 'semantic': 'Semantic Analysis - Character names, dialogue patterns, numbers',
+ 'structural': 'Structural Patterns - Paragraph structure, dialogue distribution',
+ 'character': 'Character Overlap - Common character names between chapters',
+ 'pattern': 'Pattern Analysis - Narrative flow and structure patterns'
+ }
+
+ ai_config = self.get_ai_config()
+
+ for method, desc in descriptions.items():
+ method_frame = tk.Frame(threshold_frame)
+ method_frame.pack(fill='x', pady=10)
+
+ # Method name and description
+ label_frame = tk.Frame(method_frame)
+ label_frame.pack(fill='x')
+
+ tk.Label(label_frame, text=f"{method.title()}:",
+ font=('TkDefaultFont', 10, 'bold')).pack(side='left')
+ tk.Label(label_frame, text=f" {desc}",
+ font=('TkDefaultFont', 9), fg='gray').pack(side='left', padx=(10, 0))
+
+ # Slider and value
+ slider_frame = tk.Frame(method_frame)
+ slider_frame.pack(fill='x', pady=(5, 0))
+
+ self.threshold_vars[method] = tk.IntVar(value=ai_config['thresholds'][method])
+
+ slider = tb.Scale(slider_frame, from_=10, to=100,
+ variable=self.threshold_vars[method],
+ bootstyle="info", length=400)
+ slider.pack(side='left', padx=(20, 10))
+
+ value_label = tk.Label(slider_frame, text="", width=4)
+ value_label.pack(side='left')
+
+ # Update label when slider changes
+ def update_label(val, label=value_label, var=self.threshold_vars[method]):
+ label.config(text=f"{int(var.get())}%")
+
+ self.threshold_vars[method].trace('w', lambda *args, f=update_label: f(None))
+ update_label(None)
+
+ # Weight configuration
+ tk.Label(frame, text="Method Weights (for weighted average mode)",
+ font=('TkDefaultFont', 11, 'bold')).pack(pady=(30, 10))
+
+ self.weight_vars = {}
+ weight_frame = tk.Frame(frame)
+ weight_frame.pack(fill='x', padx=20)
+
+ for method in descriptions.keys():
+ w_frame = tk.Frame(weight_frame)
+ w_frame.pack(fill='x', pady=5)
+
+ tk.Label(w_frame, text=f"{method.title()} weight:", width=20,
+ anchor='w').pack(side='left')
+
+ self.weight_vars[method] = tk.DoubleVar(value=ai_config['weights'][method])
+
+ tb.Spinbox(w_frame, from_=0.1, to=2.0, increment=0.1,
+ textvariable=self.weight_vars[method],
+ width=10).pack(side='left', padx=10)
+
+ def create_mode_tab(self, notebook):
+ """Create the detection mode configuration tab"""
+ frame = ttk.Frame(notebook)
+ notebook.add(frame, text="Detection Mode")
+
+ tk.Label(frame, text="Detection Mode Configuration",
+ font=('TkDefaultFont', 12, 'bold')).pack(pady=10)
+
+ # Detection mode selection
+ mode_frame = tk.LabelFrame(frame, text="Detection Mode", padx=20, pady=20)
+ mode_frame.pack(fill='x', padx=20, pady=10)
+
+ ai_config = self.get_ai_config()
+ self.mode_var = tk.StringVar(value=ai_config['detection_mode'])
+
+ modes = [
+ ('single_method', 'Single Method',
+ 'Flag as duplicate if ANY method exceeds its threshold\n(Most sensitive, most false positives)'),
+ ('multi_method', 'Multi-Method Agreement',
+ 'Require multiple methods to agree before flagging\n(Balanced approach)'),
+ ('weighted_average', 'Weighted Average',
+ 'Calculate weighted average of all methods\n(Most nuanced, least false positives)')
+ ]
+
+ for value, text, desc in modes:
+ rb_frame = tk.Frame(mode_frame)
+ rb_frame.pack(fill='x', pady=10)
+
+ tb.Radiobutton(rb_frame, text=text, variable=self.mode_var,
+ value=value, bootstyle="primary").pack(anchor='w')
+ tk.Label(rb_frame, text=desc, font=('TkDefaultFont', 9),
+ fg='gray').pack(anchor='w', padx=(25, 0))
+
+ # Multi-method configuration
+ multi_frame = tk.LabelFrame(frame, text="Multi-Method Settings", padx=20, pady=20)
+ multi_frame.pack(fill='x', padx=20, pady=10)
+
+ tk.Label(multi_frame, text="Number of methods required to agree:",
+ font=('TkDefaultFont', 10)).pack(anchor='w')
+
+ self.methods_required_var = tk.IntVar(
+ value=ai_config['multi_method_requirements']['methods_required'])
+
+ tb.Spinbox(multi_frame, from_=1, to=6, textvariable=self.methods_required_var,
+ width=10).pack(anchor='w', pady=5)
+
+ tk.Label(multi_frame, text="Required methods (at least one must be included):",
+ font=('TkDefaultFont', 10)).pack(anchor='w', pady=(10, 5))
+
+ self.required_method_vars = {}
+ for method in ['exact', 'text', 'semantic', 'structural', 'character', 'pattern']:
+ var = tk.BooleanVar(
+ value=method in ai_config['multi_method_requirements']['min_methods'])
+ self.required_method_vars[method] = var
+
+ tb.Checkbutton(multi_frame, text=method.title(), variable=var,
+ bootstyle="round-toggle").pack(anchor='w', padx=20)
+
+ def create_preprocessing_tab(self, notebook):
+ """Create the preprocessing configuration tab"""
+ frame = ttk.Frame(notebook)
+ notebook.add(frame, text="Preprocessing")
+
+ tk.Label(frame, text="Text Preprocessing Options",
+ font=('TkDefaultFont', 12, 'bold')).pack(pady=10)
+
+ tk.Label(frame, text="Configure how text is processed before comparison",
+ font=('TkDefaultFont', 10), fg='gray').pack(pady=(0, 20))
+
+ # Preprocessing options
+ prep_frame = tk.Frame(frame)
+ prep_frame.pack(fill='both', expand=True, padx=20)
+
+ self.prep_vars = {}
+ ai_config = self.get_ai_config()
+
+ options = [
+ ('remove_html_spacing', 'Remove HTML with spacing',
+ 'Replace HTML tags with spaces instead of removing completely'),
+ ('normalize_unicode', 'Normalize Unicode',
+ 'Normalize unicode characters (recommended)'),
+ ('ignore_case', 'Case-insensitive comparison',
+ 'Ignore character case when comparing'),
+ ('remove_extra_whitespace', 'Remove extra whitespace',
+ 'Collapse multiple spaces/newlines into single spaces')
+ ]
+
+ for key, text, desc in options:
+ var = tk.BooleanVar(value=ai_config['preprocessing'][key])
+ self.prep_vars[key] = var
+
+ opt_frame = tk.Frame(prep_frame)
+ opt_frame.pack(fill='x', pady=10)
+
+ tb.Checkbutton(opt_frame, text=text, variable=var,
+ bootstyle="round-toggle").pack(anchor='w')
+ tk.Label(opt_frame, text=desc, font=('TkDefaultFont', 9),
+ fg='gray').pack(anchor='w', padx=(25, 0))
+
+ def create_advanced_tab(self, notebook):
+ """Create the advanced settings tab"""
+ frame = ttk.Frame(notebook)
+ notebook.add(frame, text="Advanced")
+
+ tk.Label(frame, text="Advanced Settings",
+ font=('TkDefaultFont', 12, 'bold')).pack(pady=10)
+
+ # General settings
+ general_frame = tk.LabelFrame(frame, text="General", padx=20, pady=20)
+ general_frame.pack(fill='x', padx=20, pady=10)
+
+ ai_config = self.get_ai_config()
+
+ # Add separator for better organization
+ ttk.Separator(general_frame, orient='horizontal').pack(fill='x', pady=(0, 10))
+
+ # Sample size
+ ss_frame = tk.Frame(general_frame)
+ ss_frame.pack(fill='x', pady=5)
+
+ tk.Label(ss_frame, text="Sample size:", width=20, anchor='w').pack(side='left')
+ self.sample_size_var = tk.IntVar(value=ai_config['sample_size'])
+ tb.Spinbox(ss_frame, from_=1000, to=10000, increment=500,
+ textvariable=self.sample_size_var, width=10).pack(side='left', padx=10)
+ tk.Label(ss_frame, text="characters",
+ font=('TkDefaultFont', 9)).pack(side='left')
+
+ # AI Hunter Behavior Settings
+ tk.Label(general_frame, text="AI Hunter Behavior",
+ font=('TkDefaultFont', 10, 'bold')).pack(anchor='w', pady=(0, 5))
+
+ # Retry Attempts
+ retry_frame = tk.Frame(general_frame)
+ retry_frame.pack(fill='x', pady=5)
+
+ tk.Label(retry_frame, text="Retry attempts:", width=20, anchor='w').pack(side='left')
+ self.retry_attempts_var = tk.IntVar(value=ai_config.get('retry_attempts', 3))
+ tb.Spinbox(retry_frame, from_=1, to=10, textvariable=self.retry_attempts_var, width=10).pack(side='left', padx=10)
+ tk.Label(retry_frame, text="attempts", font=('TkDefaultFont', 9)).pack(side='left')
+
+ # Temperature Change Toggle
+ temp_frame = tk.Frame(general_frame)
+ temp_frame.pack(fill='x', pady=10)
+
+ self.disable_temp_change_var = tk.BooleanVar(value=ai_config.get('disable_temperature_change', False))
+ tb.Checkbutton(temp_frame, text="Disable temperature change behavior",
+ variable=self.disable_temp_change_var, bootstyle="round-toggle").pack(anchor='w')
+ tk.Label(temp_frame, text="Prevents AI Hunter from modifying temperature settings during retries",
+ font=('TkDefaultFont', 9), fg='gray').pack(anchor='w', padx=(25, 0))
+
+ # Edge filters
+ edge_frame = tk.LabelFrame(frame, text="Edge Case Filters", padx=20, pady=20)
+ edge_frame.pack(fill='x', padx=20, pady=10)
+
+ # Min text length
+ min_frame = tk.Frame(edge_frame)
+ min_frame.pack(fill='x', pady=5)
+
+ tk.Label(min_frame, text="Minimum text length:", width=20, anchor='w').pack(side='left')
+ self.min_length_var = tk.IntVar(value=ai_config['edge_filters']['min_text_length'])
+ tb.Spinbox(min_frame, from_=100, to=2000, increment=100,
+ textvariable=self.min_length_var, width=10).pack(side='left', padx=10)
+ tk.Label(min_frame, text="characters",
+ font=('TkDefaultFont', 9)).pack(side='left')
+
+ # Length ratios
+ ratio_frame = tk.Frame(edge_frame)
+ ratio_frame.pack(fill='x', pady=10)
+
+ tk.Label(ratio_frame, text="Length ratio limits:").pack(anchor='w')
+
+ r_frame = tk.Frame(ratio_frame)
+ r_frame.pack(fill='x', pady=5)
+
+ tk.Label(r_frame, text="Min ratio:", width=10, anchor='w').pack(side='left', padx=(20, 5))
+ self.min_ratio_var = tk.DoubleVar(value=ai_config['edge_filters']['min_length_ratio'])
+ tb.Spinbox(r_frame, from_=0.5, to=0.9, increment=0.1,
+ textvariable=self.min_ratio_var, width=8).pack(side='left')
+
+ tk.Label(r_frame, text="Max ratio:", width=10, anchor='w').pack(side='left', padx=(20, 5))
+ self.max_ratio_var = tk.DoubleVar(value=ai_config['edge_filters']['max_length_ratio'])
+ tb.Spinbox(r_frame, from_=1.1, to=2.0, increment=0.1,
+ textvariable=self.max_ratio_var, width=8).pack(side='left')
+
+ tk.Label(edge_frame, text="Chapters with vastly different lengths won't be compared",
+ font=('TkDefaultFont', 9), fg='gray').pack(anchor='w', padx=20)
+
+ # Language Detection
+ lang_frame = tk.LabelFrame(frame, text="Non-Target Language Detection", padx=20, pady=20)
+ lang_frame.pack(fill='x', padx=20, pady=10)
+
+ # Enable toggle
+ enable_frame = tk.Frame(lang_frame)
+ enable_frame.pack(fill='x', pady=5)
+
+ self.lang_enabled_var = tk.BooleanVar(value=ai_config['language_detection']['enabled'])
+ tb.Checkbutton(enable_frame, text="Enable non-target language detection",
+ variable=self.lang_enabled_var, bootstyle="round-toggle").pack(anchor='w')
+ tk.Label(enable_frame, text="Trigger retranslation when too much non-target language is detected",
+ font=('TkDefaultFont', 9), fg='gray').pack(anchor='w', padx=(25, 0))
+
+ # Target language selection
+ target_frame = tk.Frame(lang_frame)
+ target_frame.pack(fill='x', pady=10)
+
+ tk.Label(target_frame, text="Target language:", width=20, anchor='w').pack(side='left')
+ self.target_lang_var = tk.StringVar(value=ai_config['language_detection']['target_language'])
+
+ lang_options = list(ai_config['language_detection']['languages'].keys())
+ target_combo = ttk.Combobox(target_frame, textvariable=self.target_lang_var,
+ values=lang_options, state='readonly', width=15)
+ target_combo.pack(side='left', padx=10)
+
+ tk.Label(target_frame, text="Language that should be in the translation",
+ font=('TkDefaultFont', 9), fg='gray').pack(side='left', padx=(10, 0))
+
+ # Threshold setting
+ thresh_frame = tk.Frame(lang_frame)
+ thresh_frame.pack(fill='x', pady=5)
+
+ tk.Label(thresh_frame, text="Character threshold:", width=20, anchor='w').pack(side='left')
+ self.lang_threshold_var = tk.IntVar(value=ai_config['language_detection']['threshold_characters'])
+ tb.Spinbox(thresh_frame, from_=100, to=2000, increment=50,
+ textvariable=self.lang_threshold_var, width=10).pack(side='left', padx=10)
+ tk.Label(thresh_frame, text="non-target language characters to trigger retranslation",
+ font=('TkDefaultFont', 9), fg='gray').pack(side='left')
+
+ def apply_ai_hunter_settings(self):
+ """Apply AI Hunter settings to the main config"""
+ ai_config = self.get_ai_config()
+
+ # Update from GUI variables
+ for method, var in self.threshold_vars.items():
+ ai_config['thresholds'][method] = var.get()
+
+ for method, var in self.weight_vars.items():
+ ai_config['weights'][method] = var.get()
+
+ ai_config['detection_mode'] = self.mode_var.get()
+ ai_config['multi_method_requirements']['methods_required'] = self.methods_required_var.get()
+
+ min_methods = [method for method, var in self.required_method_vars.items() if var.get()]
+ ai_config['multi_method_requirements']['min_methods'] = min_methods
+
+ for key, var in self.prep_vars.items():
+ ai_config['preprocessing'][key] = var.get()
+
+ ai_config['sample_size'] = self.sample_size_var.get()
+
+ ai_config['edge_filters']['min_text_length'] = self.min_length_var.get()
+ ai_config['edge_filters']['min_length_ratio'] = self.min_ratio_var.get()
+ ai_config['edge_filters']['max_length_ratio'] = self.max_ratio_var.get()
+
+ # Language detection settings
+ ai_config['language_detection']['enabled'] = self.lang_enabled_var.get()
+ ai_config['language_detection']['target_language'] = self.target_lang_var.get()
+ ai_config['language_detection']['threshold_characters'] = self.lang_threshold_var.get()
+
+ # Update retry attempts and temperature change settings
+ ai_config['retry_attempts'] = self.retry_attempts_var.get()
+ ai_config['disable_temperature_change'] = self.disable_temp_change_var.get()
+
+ # Update main config
+ self.config['ai_hunter_config'] = ai_config
+
+ # Call callback if provided (this should trigger main save_configuration)
+ if self.callback:
+ self.callback()
+
+ self.window.destroy()
+
+ def reset_defaults(self):
+ """Reset all values to defaults"""
+ import tkinter.messagebox as messagebox
+ result = messagebox.askyesno("Reset to Defaults",
+ "Are you sure you want to reset all settings to defaults?")
+ if result:
+ self.config['ai_hunter_config'] = self.default_ai_hunter.copy()
+ self.window.destroy()
+ self.show_ai_hunter_config() # Reopen with default values
+
+
+class ImprovedAIHunterDetection:
+ """Improved AI Hunter detection methods for TranslateKRtoEN"""
+
+ def __init__(self, main_config):
+ """
+ Initialize with reference to main config
+
+ Args:
+ main_config: Reference to main translator config dictionary
+ """
+ self.main_config = main_config
+
+ # Default AI Hunter settings
+ self.default_ai_hunter = {
+ 'enabled': True,
+ 'lookback_chapters': 5,
+ 'retry_attempts': 3,
+ 'disable_temperature_change': False,
+ 'sample_size': 3000,
+ 'thresholds': {
+ 'exact': 90,
+ 'text': 85,
+ 'semantic': 85,
+ 'structural': 85,
+ 'character': 80,
+ 'pattern': 80
+ },
+ 'weights': {
+ 'exact': 1.5,
+ 'text': 1.2,
+ 'semantic': 1.0,
+ 'structural': 1.0,
+ 'character': 0.8,
+ 'pattern': 0.8
+ },
+ 'detection_mode': 'multi_method',
+ 'multi_method_requirements': {
+ 'methods_required': 2,
+ 'min_methods': ['semantic', 'structural']
+ },
+ 'preprocessing': {
+ 'remove_html_spacing': True,
+ 'normalize_unicode': True,
+ 'ignore_case': True,
+ 'remove_extra_whitespace': True
+ },
+ 'edge_filters': {
+ 'min_text_length': 500,
+ 'max_length_ratio': 1.3,
+ 'min_length_ratio': 0.7
+ },
+ 'language_detection': {
+ 'enabled': False,
+ 'target_language': 'english',
+ 'threshold_characters': 500,
+ 'languages': {
+ 'english': ['en'],
+ 'japanese': ['ja', 'jp'],
+ 'korean': ['ko', 'kr'],
+ 'chinese': ['zh', 'zh-cn', 'zh-tw'],
+ 'spanish': ['es'],
+ 'french': ['fr'],
+ 'german': ['de'],
+ 'russian': ['ru'],
+ 'arabic': ['ar'],
+ 'hindi': ['hi'],
+ 'portuguese': ['pt'],
+ 'italian': ['it'],
+ 'dutch': ['nl'],
+ 'thai': ['th'],
+ 'vietnamese': ['vi'],
+ 'turkish': ['tr'],
+ 'polish': ['pl'],
+ 'swedish': ['sv'],
+ 'danish': ['da'],
+ 'norwegian': ['no'],
+ 'finnish': ['fi']
+ }
+ }
+ }
+
+ def get_ai_config(self):
+ """Get AI Hunter configuration from main config"""
+ return self.main_config.get('ai_hunter_config', self.default_ai_hunter)
+
+ def detect_duplicate_ai_hunter_enhanced(self, result, idx, prog, out, current_chapter_num=None):
+ """Enhanced AI Hunter duplicate detection with configurable parameters"""
+ try:
+ print(f"\n ========== AI HUNTER DEBUG START ==========")
+ print(f" 📍 Current chapter index: {idx}")
+ if current_chapter_num:
+ print(f" 📖 Current chapter number: {current_chapter_num}")
+
+ # Get configuration
+ config = self.get_ai_config()
+
+ if not config.get('enabled', True):
+ print(f" ⚠️ AI Hunter is disabled")
+ print(f" ========== AI HUNTER DEBUG END ==========\n")
+ return False, 0
+
+ # Preprocess text
+ result_clean = self._preprocess_text(result, config['preprocessing'])
+ print(f" 📄 Text length after preprocessing: {len(result_clean)} chars")
+
+ # Check for non-target language detection
+ if config['language_detection']['enabled']:
+ non_target_detected, non_target_count = self._check_non_target_language(
+ result_clean, config['language_detection']
+ )
+ if non_target_detected:
+ print(f"\n 🌐 NON-TARGET LANGUAGE DETECTED!")
+ print(f" Non-target characters found: {non_target_count}")
+ print(f" Threshold: {config['language_detection']['threshold_characters']}")
+ print(f" Target language: {config['language_detection']['target_language']}")
+ print(f" ========== AI HUNTER DEBUG END ==========\n")
+ return True, 100 # High confidence for language detection
+
+ # Check edge cases
+ if len(result_clean) < config['edge_filters']['min_text_length']:
+ print(f" ⚠️ Text too short ({len(result_clean)} < {config['edge_filters']['min_text_length']})")
+ print(f" ========== AI HUNTER DEBUG END ==========\n")
+ return False, 0
+
+ # Extract features
+ print(f" 🔬 Extracting text features...")
+ result_features = self._extract_text_features(result_clean)
+
+ # Get lookback from main config, then fall back to env var if not found
+ lookback = self.main_config.get('duplicate_lookback_chapters',
+ int(os.getenv('DUPLICATE_LOOKBACK_CHAPTERS', '5')))
+
+ # Log configuration
+ print(f"\n 🔧 Configuration:")
+ print(f" Detection mode: {config['detection_mode']}")
+ print(f" Lookback chapters: {lookback}")
+ print(f" Sample size: {config['sample_size']}")
+
+ # FIX: Get all completed chapters sorted by actual chapter number
+ completed_chapters = []
+ for chapter_key, chapter_info in prog["chapters"].items():
+ if chapter_info.get("status") == "completed" and chapter_info.get("output_file"):
+ # Handle both numeric and hash-based chapter keys
+ try:
+ # Get actual_num from progress (this is the real chapter number)
+ chapter_num = chapter_info.get("actual_num")
+ if chapter_num is None:
+ # Try chapter_num as fallback
+ chapter_num = chapter_info.get("chapter_num")
+ if chapter_num is None:
+ # Skip chapters without valid numbers
+ print(f" ⚠️ No chapter number found for key {chapter_key}, skipping")
+ continue
+
+ completed_chapters.append({
+ 'key': chapter_key,
+ 'num': chapter_num,
+ 'file': chapter_info.get("output_file"),
+ 'ai_features': chapter_info.get("ai_features")
+ })
+ except Exception as e:
+ print(f" ⚠️ Error processing chapter {chapter_key}: {e}")
+ continue
+
+ # Sort by actual chapter number
+ completed_chapters.sort(key=lambda x: x['num'])
+
+ # If no current chapter number provided, try to infer it
+ if current_chapter_num is None:
+ # The current chapter should be passed in, but if not, we need to find it
+ # Since we're using content hash keys, we can't use idx directly
+ print(f" ⚠️ No current chapter number provided")
+ print(f" 📊 Current index: {idx}")
+
+ # The current chapter number should have been passed from the wrapper
+ # If it wasn't, we have a problem
+ print(f" ❌ ERROR: Current chapter number not provided to AI Hunter!")
+ print(f" ❌ This indicates the wrapper function is not passing the chapter number correctly")
+
+ # Emergency: just use a high number so we don't compare against anything
+ current_chapter_num = 999999
+ print(f" ⚠️ Using index-based chapter number: {current_chapter_num}")
+
+ print(f"\n 📚 Found {len(completed_chapters)} completed chapters in progress")
+ if completed_chapters:
+ chapter_nums = [ch['num'] for ch in completed_chapters]
+ print(f" 📊 Chapter numbers in progress: {sorted(chapter_nums)[:10]}{'...' if len(chapter_nums) > 10 else ''}")
+ print(f" 🎯 Current chapter number: {current_chapter_num}")
+ print(f" 🔍 Will check against last {lookback} chapters before chapter {current_chapter_num}")
+
+ # Check previous chapters
+ all_similarities = []
+ highest_similarity = 0.0
+ detected_method = None
+ detected_chapter = None
+
+ # FIX: Look at chapters by actual number, not index
+ chapters_checked = 0
+ for completed_chapter in reversed(completed_chapters):
+ # Only check chapters that come before the current one
+ if completed_chapter['num'] >= current_chapter_num:
+ continue
+
+ # Only check up to lookback number of chapters
+ if chapters_checked >= lookback:
+ break
+
+ chapters_checked += 1
+
+ print(f"\n 📝 Checking against chapter {completed_chapter['num']}...")
+
+ # Get previous chapter features
+ prev_features = completed_chapter.get('ai_features')
+ prev_clean = None
+
+ # Try to get cached features first
+ if prev_features:
+ print(f" ✅ Using cached features")
+ else:
+ # Read and extract features
+ prev_path = os.path.join(out, completed_chapter['file'])
+
+ if os.path.exists(prev_path):
+ try:
+ with open(prev_path, 'r', encoding='utf-8') as f:
+ prev_content = f.read()
+ prev_clean = self._preprocess_text(prev_content, config['preprocessing'])
+
+ # Check length ratio
+ len_ratio = len(result_clean) / max(1, len(prev_clean))
+ if (len_ratio < config['edge_filters']['min_length_ratio'] or
+ len_ratio > config['edge_filters']['max_length_ratio']):
+ print(f" ⚠️ Length ratio out of bounds: {len_ratio:.2f}")
+ continue
+
+ prev_features = self._extract_text_features(prev_clean)
+ print(f" 📄 Extracted features from file")
+ except Exception as e:
+ print(f" ❌ Failed to read file: {e}")
+ continue
+ else:
+ print(f" ❌ File not found: {prev_path}")
+ continue
+
+ # Calculate similarities
+ print(f" 🔍 Calculating similarities...")
+ similarities = self._calculate_all_similarities(
+ result_clean, result_features,
+ prev_clean, prev_features, config
+ )
+
+ # Store for reporting
+ all_similarities.append({
+ 'chapter': completed_chapter['num'],
+ 'similarities': similarities
+ })
+
+ # Log similarity scores
+ for method, score in similarities.items():
+ if score > 0:
+ print(f" {method}: {int(score*100)}%")
+
+ # Check if duplicate based on configured mode
+ is_duplicate, confidence, methods_triggered = self._evaluate_duplicate(
+ similarities, config
+ )
+
+ if is_duplicate:
+ print(f"\n 🚨 DUPLICATE DETECTED!")
+ print(f" Detection mode: {config['detection_mode']}")
+ print(f" Confidence: {int(confidence*100)}%")
+ print(f" Triggered methods: {', '.join(methods_triggered)}")
+ print(f" Match with: Chapter {completed_chapter['num']}")
+ print(f" ========== AI HUNTER DEBUG END ==========\n")
+ return True, int(confidence * 100)
+
+ # Track highest for reporting
+ for method, sim in similarities.items():
+ if sim > highest_similarity:
+ highest_similarity = sim
+ detected_method = method
+ detected_chapter = completed_chapter['num']
+
+ # No duplicate found
+ print(f"\n ✅ No duplicate found")
+ if detected_method:
+ print(f" Highest similarity: {int(highest_similarity*100)}% via {detected_method}")
+ print(f" Closest match: Chapter {detected_chapter}")
+
+ # Show top 3 closest matches
+ if all_similarities:
+ print(f"\n 📊 Top 3 closest matches:")
+ sorted_chapters = sorted(all_similarities,
+ key=lambda x: self._get_chapter_score(x['similarities'], config),
+ reverse=True)[:3]
+ for i, chapter_data in enumerate(sorted_chapters, 1):
+ score = self._get_chapter_score(chapter_data['similarities'], config)
+ print(f" {i}. Chapter {chapter_data['chapter']}: {int(score*100)}%")
+
+ print(f" ========== AI HUNTER DEBUG END ==========\n")
+ return False, 0
+
+ except Exception as e:
+ print(f" ❌ AI Hunter detection failed with error: {e}")
+ import traceback
+ print(f" {traceback.format_exc()}")
+ print(f" ========== AI HUNTER DEBUG END ==========\n")
+ return False, 0
+
+ def _preprocess_text(self, text, prep_config):
+ """Preprocess text according to configuration"""
+ # Remove HTML
+ if prep_config.get('remove_html_spacing', True):
+ text = re.sub(r'<[^>]+>', ' ', text)
+ else:
+ text = re.sub(r'<[^>]+>', '', text)
+
+ # Normalize unicode
+ if prep_config.get('normalize_unicode', True):
+ text = unicodedata.normalize('NFKD', text)
+
+ # Remove extra whitespace
+ if prep_config.get('remove_extra_whitespace', True):
+ text = re.sub(r'\s+', ' ', text)
+ text = re.sub(r'\n\s*\n', '\n\n', text)
+
+ text = text.strip()
+
+ # Convert to lowercase if case-insensitive
+ if prep_config.get('ignore_case', True):
+ text = text.lower()
+
+ return text
+
+ def _calculate_all_similarities(self, result_clean, result_features,
+ prev_clean, prev_features, config):
+ """Calculate all similarity metrics"""
+ similarities = {}
+
+ # Method 1: Exact content match
+ if prev_clean is not None:
+ sample_size = min(config['sample_size'], len(result_clean), len(prev_clean))
+ exact_sim = self._calculate_exact_similarity(
+ result_clean[:sample_size],
+ prev_clean[:sample_size]
+ )
+ similarities['exact'] = exact_sim
+
+ # Method 2: Smart text similarity
+ text_sim = self._calculate_smart_similarity(
+ result_clean, prev_clean, config['sample_size']
+ )
+ similarities['text'] = text_sim
+ else:
+ similarities['exact'] = 0.0
+ similarities['text'] = 0.0
+
+ # Method 3: Semantic fingerprint
+ semantic_sim = self._calculate_semantic_similarity(
+ result_features.get('semantic', {}),
+ prev_features.get('semantic', {})
+ )
+ similarities['semantic'] = semantic_sim
+
+ # Method 4: Structural signature
+ structural_sim = self._calculate_structural_similarity(
+ result_features.get('structural', {}),
+ prev_features.get('structural', {})
+ )
+ similarities['structural'] = structural_sim
+
+ # Method 5: Character analysis
+ char_sim = self._calculate_character_similarity(
+ result_features.get('characters', []),
+ prev_features.get('characters', [])
+ )
+ similarities['character'] = char_sim
+
+ # Method 6: Pattern analysis
+ pattern_sim = self._calculate_pattern_similarity(
+ result_features.get('patterns', {}),
+ prev_features.get('patterns', {})
+ )
+ similarities['pattern'] = pattern_sim
+
+ return similarities
+
+ def _evaluate_duplicate(self, similarities, config):
+ """Evaluate if similarities indicate a duplicate based on detection mode"""
+ mode = config['detection_mode']
+ thresholds = {k: v/100.0 for k, v in config['thresholds'].items()}
+
+ if mode == 'single_method':
+ # Any method exceeding threshold
+ for method, sim in similarities.items():
+ if sim >= thresholds.get(method, 0.85):
+ return True, sim, [method]
+ return False, 0, []
+
+ elif mode == 'multi_method':
+ # Multiple methods must agree
+ triggered_methods = []
+ for method, sim in similarities.items():
+ if sim >= thresholds.get(method, 0.85):
+ triggered_methods.append(method)
+
+ # Check if enough methods triggered
+ required = config.get('multi_method_requirements', {}).get('methods_required', 2)
+ min_methods = config.get('multi_method_requirements', {}).get('min_methods', [])
+
+ if len(triggered_methods) >= required:
+ # Check if at least one required method is included
+ if not min_methods or any(m in triggered_methods for m in min_methods):
+ # Calculate average confidence of triggered methods
+ confidence = sum(similarities[m] for m in triggered_methods) / len(triggered_methods)
+ return True, confidence, triggered_methods
+
+ return False, 0, []
+
+ elif mode == 'weighted_average':
+ # Calculate weighted average
+ weights = config.get('weights', {})
+ total_weight = sum(weights.get(m, 1.0) for m in similarities)
+ weighted_sum = sum(similarities[m] * weights.get(m, 1.0) for m in similarities)
+ weighted_avg = weighted_sum / total_weight if total_weight > 0 else 0
+
+ # Check if weighted average exceeds average threshold
+ avg_threshold = sum(thresholds.values()) / len(thresholds) if thresholds else 0.85
+
+ if weighted_avg >= avg_threshold:
+ # Find which methods contributed most
+ triggered = [m for m, sim in similarities.items()
+ if sim >= thresholds.get(m, 0.85)]
+ return True, weighted_avg, triggered
+
+ return False, 0, []
+
+ return False, 0, []
+
+ def _get_chapter_score(self, similarities, config):
+ """Calculate overall score for a chapter comparison"""
+ if config['detection_mode'] == 'weighted_average':
+ weights = config.get('weights', {})
+ total_weight = sum(weights.get(m, 1.0) for m in similarities)
+ return sum(similarities.get(m, 0) * weights.get(m, 1.0) for m in similarities) / total_weight if total_weight > 0 else 0
+ else:
+ return max(similarities.values()) if similarities else 0
+
+ def _extract_text_features(self, text):
+ """Extract multiple features from text for AI Hunter analysis"""
+ features = {
+ 'semantic': {},
+ 'structural': {},
+ 'characters': [],
+ 'patterns': {}
+ }
+
+ # Semantic fingerprint
+ lines = text.split('\n')
+
+ # Character extraction (names that appear 3+ times)
+ words = re.findall(r'\b[A-Z][a-z]+\b', text)
+ word_freq = Counter(words)
+ features['characters'] = [name for name, count in word_freq.items()
+ if count >= 3 and name not in {
+ 'The', 'A', 'An', 'In', 'On', 'At', 'To',
+ 'From', 'With', 'By', 'For', 'Of', 'As',
+ 'But', 'And', 'Or', 'He', 'She', 'It',
+ 'They', 'We', 'You', 'What', 'When', 'Where',
+ 'Who', 'Why', 'How', 'That', 'This', 'These'
+ }]
+
+ # Dialogue patterns
+ dialogue_patterns = re.findall(r'"([^"]+)"', text)
+ features['semantic']['dialogue_count'] = len(dialogue_patterns)
+ features['semantic']['dialogue_lengths'] = [len(d) for d in dialogue_patterns[:10]]
+
+ # Speaker patterns
+ speaker_patterns = re.findall(r'(\w+)\s+(?:said|asked|replied|shouted|whispered)', text.lower())
+ features['semantic']['speakers'] = list(set(speaker_patterns[:20]))
+
+ # Number extraction
+ numbers = re.findall(r'\b\d+\b', text)
+ features['patterns']['numbers'] = numbers[:20]
+
+ # Structural signature
+ para_lengths = []
+ dialogue_count = 0
+ for para in text.split('\n\n'):
+ if para.strip():
+ para_lengths.append(len(para))
+ if '"' in para:
+ dialogue_count += 1
+
+ features['structural']['para_count'] = len(para_lengths)
+ features['structural']['avg_para_length'] = sum(para_lengths) / max(1, len(para_lengths))
+ features['structural']['dialogue_ratio'] = dialogue_count / max(1, len(para_lengths))
+
+ # Create structural pattern string
+ pattern = []
+ for para in text.split('\n\n')[:20]: # First 20 paragraphs
+ if para.strip():
+ if '"' in para:
+ pattern.append('D') # Dialogue
+ elif len(para) > 300:
+ pattern.append('L') # Long
+ elif len(para) < 100:
+ pattern.append('S') # Short
+ else:
+ pattern.append('M') # Medium
+ features['structural']['pattern'] = ''.join(pattern)
+
+ # Action density
+ action_verbs = len(re.findall(r'\b\w+ed\b', text))
+ features['semantic']['action_density'] = action_verbs / max(1, len(text.split()))
+
+ # Text length
+ features['semantic']['text_length'] = len(text)
+
+ return features
+
+ def _calculate_exact_similarity(self, text1, text2):
+ """Calculate exact text similarity"""
+ return SequenceMatcher(None, text1, text2).ratio()
+
+ def _calculate_smart_similarity(self, text1, text2, sample_size):
+ """Smart similarity with configurable sample size"""
+ if len(text1) > sample_size * 3 and len(text2) > sample_size * 3:
+ # Use multiple samples
+ samples1 = [
+ text1[:sample_size],
+ text1[len(text1)//2 - sample_size//2:len(text1)//2 + sample_size//2],
+ text1[-sample_size:]
+ ]
+ samples2 = [
+ text2[:sample_size],
+ text2[len(text2)//2 - sample_size//2:len(text2)//2 + sample_size//2],
+ text2[-sample_size:]
+ ]
+ similarities = [SequenceMatcher(None, s1, s2).ratio()
+ for s1, s2 in zip(samples1, samples2)]
+ return sum(similarities) / len(similarities)
+ else:
+ # Use full text up to sample size
+ return SequenceMatcher(None, text1[:sample_size], text2[:sample_size]).ratio()
+
+ def _calculate_semantic_similarity(self, sem1, sem2):
+ """Calculate semantic fingerprint similarity"""
+ score = 0.0
+ weights = 0.0
+
+ # Compare dialogue counts
+ if 'dialogue_count' in sem1 and 'dialogue_count' in sem2:
+ weights += 0.3
+ if sem1['dialogue_count'] > 0 or sem2['dialogue_count'] > 0:
+ ratio = min(sem1['dialogue_count'], sem2['dialogue_count']) / \
+ max(1, max(sem1['dialogue_count'], sem2['dialogue_count']))
+ score += ratio * 0.3
+
+ # Compare speakers
+ if 'speakers' in sem1 and 'speakers' in sem2:
+ weights += 0.4
+ if sem1['speakers'] and sem2['speakers']:
+ overlap = len(set(sem1['speakers']) & set(sem2['speakers']))
+ total = len(set(sem1['speakers']) | set(sem2['speakers']))
+ score += (overlap / max(1, total)) * 0.4
+ elif not sem1['speakers'] and not sem2['speakers']:
+ score += 0.4 # Both have no speakers
+
+ # Compare dialogue lengths pattern
+ if 'dialogue_lengths' in sem1 and 'dialogue_lengths' in sem2:
+ weights += 0.2
+ if sem1['dialogue_lengths'] and sem2['dialogue_lengths']:
+ len1 = sem1['dialogue_lengths'][:10]
+ len2 = sem2['dialogue_lengths'][:10]
+ if len1 and len2:
+ avg1 = sum(len1) / len(len1)
+ avg2 = sum(len2) / len(len2)
+ ratio = min(avg1, avg2) / max(1, max(avg1, avg2))
+ score += ratio * 0.2
+ elif not sem1['dialogue_lengths'] and not sem2['dialogue_lengths']:
+ score += 0.2 # Both have no dialogue
+
+ # Action density
+ if 'action_density' in sem1 and 'action_density' in sem2:
+ weights += 0.1
+ act_sim = 1 - abs(sem1['action_density'] - sem2['action_density'])
+ score += act_sim * 0.1
+
+ return score / max(0.1, weights)
+
+ def _calculate_structural_similarity(self, struct1, struct2):
+ """Calculate structural signature similarity"""
+ score = 0.0
+
+ # Compare paragraph patterns
+ if 'pattern' in struct1 and 'pattern' in struct2:
+ pattern_sim = SequenceMatcher(None, struct1['pattern'], struct2['pattern']).ratio()
+ score += pattern_sim * 0.5
+
+ # Compare paragraph statistics
+ if all(k in struct1 for k in ['para_count', 'avg_para_length', 'dialogue_ratio']) and \
+ all(k in struct2 for k in ['para_count', 'avg_para_length', 'dialogue_ratio']):
+
+ # Paragraph count ratio
+ para_ratio = min(struct1['para_count'], struct2['para_count']) / \
+ max(1, max(struct1['para_count'], struct2['para_count']))
+ score += para_ratio * 0.2
+
+ # Average length ratio
+ avg_ratio = min(struct1['avg_para_length'], struct2['avg_para_length']) / \
+ max(1, max(struct1['avg_para_length'], struct2['avg_para_length']))
+ score += avg_ratio * 0.15
+
+ # Dialogue ratio similarity
+ dialogue_diff = abs(struct1['dialogue_ratio'] - struct2['dialogue_ratio'])
+ score += (1 - min(1, dialogue_diff)) * 0.15
+
+ return score
+
+ def _calculate_character_similarity(self, chars1, chars2):
+ """Calculate character overlap similarity"""
+ if not chars1 or not chars2:
+ return 0.0
+
+ # Convert to sets
+ set1 = set(chars1)
+ set2 = set(chars2)
+
+ # If no overlap at all, return 0
+ intersection = set1 & set2
+ if not intersection:
+ return 0.0
+
+ # Calculate Jaccard index (intersection over union)
+ union = set1 | set2
+ jaccard = len(intersection) / len(union)
+
+ # Also consider the proportion of matching characters relative to each set
+ # This prevents small overlaps from scoring too high
+ overlap1 = len(intersection) / len(set1)
+ overlap2 = len(intersection) / len(set2)
+
+ # Take the minimum overlap to be more conservative
+ min_overlap = min(overlap1, overlap2)
+
+ # Combine jaccard and overlap scores
+ # Jaccard penalizes when sets are very different sizes
+ # Min overlap ensures both texts share a significant portion of characters
+ score = (jaccard + min_overlap) / 2
+
+ return score
+
+ def _calculate_pattern_similarity(self, pat1, pat2):
+ """Calculate pattern similarity (numbers, etc.)"""
+ score = 0.0
+
+ # Number overlap
+ if 'numbers' in pat1 and 'numbers' in pat2:
+ nums1 = set(pat1['numbers'])
+ nums2 = set(pat2['numbers'])
+
+ if nums1 or nums2:
+ overlap = len(nums1 & nums2)
+ total = len(nums1 | nums2)
+ score = overlap / max(1, total)
+ else:
+ score = 1.0 # Both have no numbers
+
+ return score
+
+ def _check_non_target_language(self, text, lang_config):
+ """Check if text contains too much non-target language"""
+ target_language = lang_config['target_language'].lower()
+ threshold = lang_config['threshold_characters']
+
+ # Character ranges for different languages
+ language_ranges = {
+ 'english': [ # Latin script + basic symbols
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ (0x2000, 0x206F), # General Punctuation
+ (0x20A0, 0x20CF), # Currency Symbols
+ (0xFF00, 0xFFEF), # Halfwidth and Fullwidth Forms
+ ],
+ 'japanese': [
+ (0x3040, 0x309F), # Hiragana
+ (0x30A0, 0x30FF), # Katakana
+ (0x4E00, 0x9FAF), # CJK Unified Ideographs
+ (0x3400, 0x4DBF), # CJK Extension A
+ (0xFF66, 0xFF9F), # Halfwidth Katakana
+ ],
+ 'korean': [
+ (0xAC00, 0xD7AF), # Hangul Syllables
+ (0x1100, 0x11FF), # Hangul Jamo
+ (0x3130, 0x318F), # Hangul Compatibility Jamo
+ (0xA960, 0xA97F), # Hangul Jamo Extended-A
+ (0xD7B0, 0xD7FF), # Hangul Jamo Extended-B
+ ],
+ 'chinese': [
+ (0x4E00, 0x9FAF), # CJK Unified Ideographs
+ (0x3400, 0x4DBF), # CJK Extension A
+ (0x20000, 0x2A6DF), # CJK Extension B
+ (0x2A700, 0x2B73F), # CJK Extension C
+ (0x2B740, 0x2B81F), # CJK Extension D
+ (0x3000, 0x303F), # CJK Symbols and Punctuation
+ ],
+ 'arabic': [
+ (0x0600, 0x06FF), # Arabic
+ (0x0750, 0x077F), # Arabic Supplement
+ (0x08A0, 0x08FF), # Arabic Extended-A
+ (0xFB50, 0xFDFF), # Arabic Presentation Forms-A
+ (0xFE70, 0xFEFF), # Arabic Presentation Forms-B
+ ],
+ 'russian': [
+ (0x0400, 0x04FF), # Cyrillic
+ (0x0500, 0x052F), # Cyrillic Supplement
+ (0x2DE0, 0x2DFF), # Cyrillic Extended-A
+ (0xA640, 0xA69F), # Cyrillic Extended-B
+ ],
+ 'thai': [
+ (0x0E00, 0x0E7F), # Thai
+ ],
+ 'hindi': [
+ (0x0900, 0x097F), # Devanagari
+ (0xA8E0, 0xA8FF), # Devanagari Extended
+ ],
+ 'spanish': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'french': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'german': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'portuguese': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'italian': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'dutch': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'vietnamese': [
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ (0x1EA0, 0x1EFF), # Latin Extended Additional (Vietnamese)
+ ],
+ 'turkish': [
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'polish': [
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'swedish': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'danish': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'norwegian': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ 'finnish': [ # Same as English (Latin script)
+ (0x0000, 0x007F), # Basic Latin
+ (0x0080, 0x00FF), # Latin-1 Supplement
+ (0x0100, 0x017F), # Latin Extended-A
+ (0x0180, 0x024F), # Latin Extended-B
+ ],
+ }
+
+ # Get target language ranges
+ target_ranges = language_ranges.get(target_language, language_ranges['english'])
+
+ # Count characters that are NOT in target language ranges
+ non_target_count = 0
+ total_letters = 0
+
+ for char in text:
+ # Skip whitespace, punctuation, and numbers for counting
+ if char.isspace() or char.isdigit():
+ continue
+
+ # Count as letter character
+ total_letters += 1
+
+ # Check if character is in any target language range
+ char_code = ord(char)
+ is_target_char = any(start <= char_code <= end for start, end in target_ranges)
+
+ if not is_target_char:
+ non_target_count += 1
+
+ # Debug logging
+ if non_target_count > 0:
+ print(f" 🌐 Language detection: {non_target_count}/{total_letters} non-target chars ({target_language})")
+
+ # Return True if non-target character count exceeds threshold
+ return non_target_count >= threshold, non_target_count
diff --git a/api_key_encryption.py b/api_key_encryption.py
new file mode 100644
index 0000000000000000000000000000000000000000..42560e2d917bfbf4b890d1cc400b871f58bbbca8
--- /dev/null
+++ b/api_key_encryption.py
@@ -0,0 +1,244 @@
+"""
+Simple API Key Encryption Module for Glossarion
+Encrypts only specific API key fields including multi-key support
+"""
+
+import os
+import json
+import base64
+from cryptography.fernet import Fernet
+from pathlib import Path
+
+
+class APIKeyEncryption:
+ """Simple encryption handler for API keys"""
+
+ def __init__(self):
+ self.key_file = Path('.glossarion_key')
+ self.cipher = self._get_or_create_cipher()
+
+ # Define which fields to encrypt
+ self.api_key_fields = [
+ 'api_key',
+ 'replicate_api_key',
+ # Add more field names here if needed
+ ]
+
+ def _get_or_create_cipher(self):
+ """Get existing cipher or create new one"""
+ if self.key_file.exists():
+ try:
+ key = self.key_file.read_bytes()
+ return Fernet(key)
+ except:
+ pass
+
+ # Generate new key
+ key = Fernet.generate_key()
+ self.key_file.write_bytes(key)
+
+ # Hide file on Windows
+ if os.name == 'nt':
+ import ctypes
+ ctypes.windll.kernel32.SetFileAttributesW(str(self.key_file), 2)
+ else:
+ # Restrict permissions on Unix
+ os.chmod(self.key_file, 0o600)
+
+ return Fernet(key)
+
+ def encrypt_value(self, value):
+ """Encrypt a single value"""
+ try:
+ encrypted = self.cipher.encrypt(value.encode())
+ return f"ENC:{base64.b64encode(encrypted).decode()}"
+ except:
+ return value
+
+ def decrypt_value(self, value):
+ """Decrypt a single value"""
+ if not isinstance(value, str) or not value.startswith('ENC:'):
+ return value
+
+ try:
+ encrypted_data = base64.b64decode(value[4:])
+ return self.cipher.decrypt(encrypted_data).decode()
+ except:
+ return value
+
+ def encrypt_multi_keys(self, multi_keys):
+ """Encrypt API keys in multi_api_keys array"""
+ if not isinstance(multi_keys, list):
+ return multi_keys
+
+ encrypted_keys = []
+ for key_entry in multi_keys:
+ if isinstance(key_entry, dict):
+ encrypted_entry = key_entry.copy()
+ # Encrypt the api_key field in each entry
+ if 'api_key' in encrypted_entry and encrypted_entry['api_key']:
+ value = encrypted_entry['api_key']
+ if isinstance(value, str) and not value.startswith('ENC:'):
+ encrypted_entry['api_key'] = self.encrypt_value(value)
+ encrypted_keys.append(encrypted_entry)
+ else:
+ encrypted_keys.append(key_entry)
+
+ return encrypted_keys
+
+ def decrypt_multi_keys(self, multi_keys):
+ """Decrypt API keys in multi_api_keys array"""
+ if not isinstance(multi_keys, list):
+ return multi_keys
+
+ decrypted_keys = []
+ for key_entry in multi_keys:
+ if isinstance(key_entry, dict):
+ decrypted_entry = key_entry.copy()
+ # Decrypt the api_key field in each entry
+ if 'api_key' in decrypted_entry and decrypted_entry['api_key']:
+ decrypted_entry['api_key'] = self.decrypt_value(decrypted_entry['api_key'])
+ decrypted_keys.append(decrypted_entry)
+ else:
+ decrypted_keys.append(key_entry)
+
+ return decrypted_keys
+
+ def encrypt_config(self, config):
+ """Encrypt specific API key fields including multi-key support"""
+ encrypted = config.copy()
+
+ # Encrypt regular API key fields
+ for field in self.api_key_fields:
+ if field in encrypted and encrypted[field]:
+ value = encrypted[field]
+ # Only encrypt if not already encrypted
+ if isinstance(value, str) and not value.startswith('ENC:'):
+ encrypted[field] = self.encrypt_value(value)
+
+ # Encrypt multi_api_keys if present
+ if 'multi_api_keys' in encrypted:
+ encrypted['multi_api_keys'] = self.encrypt_multi_keys(encrypted['multi_api_keys'])
+
+ # Encrypt fallback_keys if present
+ if 'fallback_keys' in encrypted:
+ encrypted['fallback_keys'] = self.encrypt_multi_keys(encrypted['fallback_keys'])
+
+ return encrypted
+
+ def decrypt_config(self, config):
+ """Decrypt specific API key fields including multi-key support"""
+ decrypted = config.copy()
+
+ # Decrypt regular API key fields
+ for field in self.api_key_fields:
+ if field in decrypted and decrypted[field]:
+ decrypted[field] = self.decrypt_value(decrypted[field])
+
+ # Decrypt multi_api_keys if present
+ if 'multi_api_keys' in decrypted:
+ decrypted['multi_api_keys'] = self.decrypt_multi_keys(decrypted['multi_api_keys'])
+
+ # Decrypt fallback_keys if present
+ if 'fallback_keys' in decrypted:
+ decrypted['fallback_keys'] = self.decrypt_multi_keys(decrypted['fallback_keys'])
+
+ return decrypted
+
+# Simple interface functions
+_handler = None
+
+def get_handler():
+ global _handler
+ if _handler is None:
+ _handler = APIKeyEncryption()
+ return _handler
+
+def encrypt_config(config):
+ """Encrypt API keys in config"""
+ return get_handler().encrypt_config(config)
+
+def decrypt_config(config):
+ """Decrypt API keys in config"""
+ return get_handler().decrypt_config(config)
+
+def migrate_config_file(config_file='config.json'):
+ """Migrate existing config to encrypted format"""
+ try:
+ # Read config
+ with open(config_file, 'r', encoding='utf-8') as f:
+ config = json.load(f)
+
+ # Check if already encrypted
+ handler = get_handler()
+ needs_encryption = False
+
+ # Check regular API key fields
+ for field in handler.api_key_fields:
+ if field in config and config[field]:
+ if isinstance(config[field], str) and not config[field].startswith('ENC:'):
+ needs_encryption = True
+ break
+
+ # Check multi_api_keys
+ if 'multi_api_keys' in config and isinstance(config['multi_api_keys'], list):
+ for key_entry in config['multi_api_keys']:
+ if isinstance(key_entry, dict) and 'api_key' in key_entry:
+ if key_entry['api_key'] and not key_entry['api_key'].startswith('ENC:'):
+ needs_encryption = True
+ break
+
+ # Check fallback_keys
+ if 'fallback_keys' in config and isinstance(config['fallback_keys'], list):
+ for key_entry in config['fallback_keys']:
+ if isinstance(key_entry, dict) and 'api_key' in key_entry:
+ if key_entry['api_key'] and not key_entry['api_key'].startswith('ENC:'):
+ needs_encryption = True
+ break
+
+ if not needs_encryption:
+ print("Config already encrypted or no API keys found.")
+ return True
+
+ # Backup
+ backup_file = f"{config_file}.backup"
+ with open(backup_file, 'w', encoding='utf-8') as f:
+ json.dump(config, f, ensure_ascii=False, indent=2)
+ print(f"Created backup: {backup_file}")
+
+ # Encrypt
+ encrypted = encrypt_config(config)
+
+ # Save
+ with open(config_file, 'w', encoding='utf-8') as f:
+ json.dump(encrypted, f, ensure_ascii=False, indent=2)
+
+ print("✅ Successfully encrypted API keys!")
+
+ # Show summary
+ if 'multi_api_keys' in config:
+ print(f" - Encrypted {len(config['multi_api_keys'])} multi-key entries")
+
+ if 'fallback_keys' in config:
+ print(f" - Encrypted {len(config['fallback_keys'])} fallback-key entries")
+
+ return True
+
+ except Exception as e:
+ print(f"❌ Error: {e}")
+ return False
+
+
+if __name__ == "__main__":
+ # Simple migration script
+ import sys
+
+ config_file = 'config.json'
+ if len(sys.argv) > 1:
+ config_file = sys.argv[1]
+
+ if os.path.exists(config_file):
+ print(f"Encrypting API keys in {config_file}...")
+ migrate_config_file(config_file)
+ else:
+ print(f"Config file not found: {config_file}")
diff --git a/async_api_processor.py b/async_api_processor.py
new file mode 100644
index 0000000000000000000000000000000000000000..6da743c76e0025765e266118518a3c8f74de3bf2
--- /dev/null
+++ b/async_api_processor.py
@@ -0,0 +1,3431 @@
+# async_api_processor.py
+"""
+Asynchronous API Processing for Glossarion
+Implements batch API processing with 50% discount from supported providers.
+This is SEPARATE from the existing batch processing (parallel API calls).
+
+Supported Providers with Async/Batch APIs (50% discount):
+- Gemini (Batch API)
+- Anthropic (Message Batches API)
+- OpenAI (Batch API)
+- Mistral (Batch API)
+- Amazon Bedrock (Batch Inference)
+- Groq (Batch API)
+
+Providers without Async APIs:
+- DeepSeek (no batch API)
+- Cohere (only batch embeddings, not completions)
+"""
+
+import os
+import sys
+import re
+from bs4 import BeautifulSoup
+import ebooklib
+from ebooklib import epub
+import json
+import time
+import threading
+import logging
+import hashlib
+import traceback
+from datetime import datetime, timedelta
+from typing import Dict, List, Optional, Tuple, Any
+import tkinter as tk
+from tkinter import ttk, messagebox
+import ttkbootstrap as tb
+from dataclasses import dataclass, asdict
+from enum import Enum
+import requests
+import uuid
+from pathlib import Path
+
+try:
+ import tiktoken
+except ImportError:
+ tiktoken = None
+
+# For TXT file processing
+try:
+ from txt_processor import TextFileProcessor
+except ImportError:
+ TextFileProcessor = None
+ print("txt_processor not available - TXT file support disabled")
+# For provider-specific implementations
+try:
+ import google.generativeai as genai
+ HAS_GEMINI = True
+except ImportError:
+ HAS_GEMINI = False
+
+try:
+ import anthropic
+ HAS_ANTHROPIC = True
+except ImportError:
+ HAS_ANTHROPIC = False
+
+try:
+ import openai
+ HAS_OPENAI = True
+except ImportError:
+ HAS_OPENAI = False
+
+logger = logging.getLogger(__name__)
+
+class AsyncAPIStatus(Enum):
+ """Status states for async API jobs"""
+ PENDING = "pending"
+ PROCESSING = "processing"
+ COMPLETED = "completed"
+ FAILED = "failed"
+ CANCELLED = "cancelled"
+ EXPIRED = "expired"
+
+@dataclass
+class AsyncJobInfo:
+ """Information about an async API job"""
+ job_id: str
+ provider: str
+ model: str
+ status: AsyncAPIStatus
+ created_at: datetime
+ updated_at: datetime
+ total_requests: int
+ completed_requests: int = 0
+ failed_requests: int = 0
+ cost_estimate: float = 0.0
+ input_file: Optional[str] = None
+ output_file: Optional[str] = None
+ error_message: Optional[str] = None
+ metadata: Dict[str, Any] = None
+
+ def to_dict(self) -> Dict[str, Any]:
+ """Convert to dictionary for JSON serialization"""
+ data = asdict(self)
+ data['status'] = self.status.value
+ data['created_at'] = self.created_at.isoformat()
+ data['updated_at'] = self.updated_at.isoformat()
+ return data
+
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> 'AsyncJobInfo':
+ """Create from dictionary"""
+ data['status'] = AsyncAPIStatus(data['status'])
+ data['created_at'] = datetime.fromisoformat(data['created_at'])
+ data['updated_at'] = datetime.fromisoformat(data['updated_at'])
+ if data.get('metadata') is None:
+ data['metadata'] = {}
+ return cls(**data)
+
+class AsyncAPIProcessor:
+ """Handles asynchronous batch API processing for supported providers"""
+
+ # Provider configurations
+ PROVIDER_CONFIGS = {
+ 'gemini': {
+ 'batch_endpoint': 'native_sdk', # Uses native SDK instead of REST
+ 'status_endpoint': 'native_sdk',
+ 'max_requests_per_batch': 10000,
+ 'supports_chunking': False,
+ 'discount': 0.5,
+ 'available': True # Now available!
+ },
+ 'anthropic': {
+ 'batch_endpoint': 'https://api.anthropic.com/v1/messages/batches',
+ 'status_endpoint': 'https://api.anthropic.com/v1/messages/batches/{job_id}',
+ 'max_requests_per_batch': 10000,
+ 'supports_chunking': False,
+ 'discount': 0.5
+ },
+ 'openai': {
+ 'batch_endpoint': 'https://api.openai.com/v1/batches',
+ 'status_endpoint': 'https://api.openai.com/v1/batches/{job_id}',
+ 'cancel_endpoint': 'https://api.openai.com/v1/batches/{job_id}/cancel',
+ 'max_requests_per_batch': 50000,
+ 'supports_chunking': False,
+ 'discount': 0.5
+ },
+ 'mistral': {
+ 'batch_endpoint': 'https://api.mistral.ai/v1/batch/jobs',
+ 'status_endpoint': 'https://api.mistral.ai/v1/batch/jobs/{job_id}',
+ 'max_requests_per_batch': 10000,
+ 'supports_chunking': False,
+ 'discount': 0.5
+ },
+ 'bedrock': {
+ 'batch_endpoint': 'batch-inference', # AWS SDK specific
+ 'max_requests_per_batch': 10000,
+ 'supports_chunking': False,
+ 'discount': 0.5
+ },
+ 'groq': {
+ 'batch_endpoint': 'https://api.groq.com/openai/v1/batch',
+ 'status_endpoint': 'https://api.groq.com/openai/v1/batch/{job_id}',
+ 'max_requests_per_batch': 1000,
+ 'supports_chunking': False,
+ 'discount': 0.5
+ }
+ }
+
+ def __init__(self, gui_instance):
+ """Initialize the async processor
+
+ Args:
+ gui_instance: Reference to TranslatorGUI instance
+ """
+ self.gui = gui_instance
+ self.jobs_file = os.path.join(os.path.dirname(__file__), 'async_jobs.json')
+ self.jobs: Dict[str, AsyncJobInfo] = {}
+ self.stop_flag = threading.Event()
+ self.processing_thread = None
+ self._load_jobs()
+
+ def _load_jobs(self):
+ """Load saved async jobs from file"""
+ try:
+ if os.path.exists(self.jobs_file):
+ with open(self.jobs_file, 'r', encoding='utf-8') as f:
+ data = json.load(f)
+ for job_id, job_data in data.items():
+ try:
+ self.jobs[job_id] = AsyncJobInfo.from_dict(job_data)
+ except Exception as e:
+ print(f"Failed to load job {job_id}: {e}")
+ except Exception as e:
+ print(f"Failed to load async jobs: {e}")
+
+ def _save_jobs(self):
+ """Save async jobs to file"""
+ try:
+ data = {job_id: job.to_dict() for job_id, job in self.jobs.items()}
+ with open(self.jobs_file, 'w', encoding='utf-8') as f:
+ json.dump(data, f, indent=2)
+ except Exception as e:
+ print(f"Failed to save async jobs: {e}")
+
+ def get_provider_from_model(self, model: str) -> Optional[str]:
+ """Determine provider from model name"""
+ model_lower = model.lower()
+
+ # Check prefixes
+ if model_lower.startswith(('gpt', 'o1', 'o3', 'o4')):
+ return 'openai'
+ elif model_lower.startswith('gemini'):
+ return 'gemini'
+ elif model_lower.startswith(('claude', 'sonnet', 'opus', 'haiku')):
+ return 'anthropic'
+ elif model_lower.startswith(('mistral', 'mixtral', 'codestral')):
+ return 'mistral'
+ elif model_lower.startswith('groq'):
+ return 'groq'
+ elif model_lower.startswith('bedrock'):
+ return 'bedrock'
+
+ # Check for aggregator prefixes that might support async
+ if model_lower.startswith(('eh/', 'electronhub/', 'electron/')):
+ # Extract actual model after prefix
+ actual_model = model.split('/', 1)[1] if '/' in model else model
+ return self.get_provider_from_model(actual_model)
+
+ return None
+
+ def supports_async(self, model: str) -> bool:
+ """Check if model supports async processing"""
+ provider = self.get_provider_from_model(model)
+ return provider in self.PROVIDER_CONFIGS
+
+ def estimate_cost(self, num_chapters: int, avg_tokens_per_chapter: int, model: str, compression_factor: float = 1.0) -> Tuple[float, float]:
+ """Estimate costs for async vs regular processing
+
+ Returns:
+ Tuple of (async_cost, regular_cost)
+ """
+ provider = self.get_provider_from_model(model)
+ if not provider:
+ return (0.0, 0.0)
+
+ # UPDATED PRICING AS OF JULY 2025
+ # Prices are (input_price, output_price) per 1M tokens
+ token_prices = {
+ 'openai': {
+ # GPT-4.1 Series (Latest - June 2024 knowledge)
+ 'gpt-4.1': (2.0, 8.0),
+ 'gpt-4.1-mini': (0.4, 1.6),
+ 'gpt-4.1-nano': (0.1, 0.4),
+
+ # GPT-4.5 Preview
+ 'gpt-4.5-preview': (75.0, 150.0),
+
+ # GPT-4o Series
+ 'gpt-4o': (2.5, 10.0),
+ 'gpt-4o-mini': (0.15, 0.6),
+ 'gpt-4o-audio': (2.5, 10.0),
+ 'gpt-4o-audio-preview': (2.5, 10.0),
+ 'gpt-4o-realtime': (5.0, 20.0),
+ 'gpt-4o-realtime-preview': (5.0, 20.0),
+ 'gpt-4o-mini-audio': (0.15, 0.6),
+ 'gpt-4o-mini-audio-preview': (0.15, 0.6),
+ 'gpt-4o-mini-realtime': (0.6, 2.4),
+ 'gpt-4o-mini-realtime-preview': (0.6, 2.4),
+
+ # GPT-4 Legacy
+ 'gpt-4': (30.0, 60.0),
+ 'gpt-4-turbo': (10.0, 30.0),
+ 'gpt-4-32k': (60.0, 120.0),
+ 'gpt-4-0613': (30.0, 60.0),
+ 'gpt-4-0314': (30.0, 60.0),
+
+ # GPT-3.5
+ 'gpt-3.5-turbo': (0.5, 1.5),
+ 'gpt-3.5-turbo-instruct': (1.5, 2.0),
+ 'gpt-3.5-turbo-16k': (3.0, 4.0),
+ 'gpt-3.5-turbo-0125': (0.5, 1.5),
+
+ # O-series Reasoning Models (NOT batch compatible usually)
+ 'o1': (15.0, 60.0),
+ 'o1-pro': (150.0, 600.0),
+ 'o1-mini': (1.1, 4.4),
+ 'o3': (1.0, 4.0),
+ 'o3-pro': (20.0, 80.0),
+ 'o3-deep-research': (10.0, 40.0),
+ 'o3-mini': (1.1, 4.4),
+ 'o4-mini': (1.1, 4.4),
+ 'o4-mini-deep-research': (2.0, 8.0),
+
+ # Special models
+ 'chatgpt-4o-latest': (5.0, 15.0),
+ 'computer-use-preview': (3.0, 12.0),
+ 'gpt-4o-search-preview': (2.5, 10.0),
+ 'gpt-4o-mini-search-preview': (0.15, 0.6),
+ 'codex-mini-latest': (1.5, 6.0),
+
+ # Small models
+ 'davinci-002': (2.0, 2.0),
+ 'babbage-002': (0.4, 0.4),
+
+ 'default': (2.5, 10.0)
+ },
+ 'anthropic': {
+ # Claude 4 Series (Latest)
+ 'claude-4-opus': (3.0, 15.0),
+ 'claude-opus-4': (3.0, 15.0),
+ 'claude-4-sonnet': (3.0, 15.0),
+ 'claude-sonnet-4': (3.0, 15.0),
+
+ # Claude 3.5 Series
+ 'claude-3.5-sonnet': (3.0, 15.0),
+ 'claude-3.5-opus': (15.0, 75.0),
+ 'claude-3.5-haiku': (0.25, 1.25),
+
+ # Claude 3 Series
+ 'claude-3-opus': (15.0, 75.0),
+ 'claude-3-sonnet': (3.0, 15.0),
+ 'claude-3-haiku': (0.25, 1.25),
+
+ # Legacy
+ 'claude-2.1': (8.0, 24.0),
+ 'claude-2': (8.0, 24.0),
+ 'claude-instant': (0.8, 2.4),
+
+ 'default': (3.0, 15.0)
+ },
+ 'gemini': {
+ # Gemini 2.5 Series (Latest)
+ 'gemini-2.5-pro': (1.25, 10.0), # ≤200k tokens
+ 'gemini-2.5-flash': (0.3, 2.5),
+ 'gemini-2.5-flash-lite': (0.1, 0.4),
+ 'gemini-2.5-flash-lite-preview': (0.1, 0.4),
+ 'gemini-2.5-flash-lite-preview-06-17': (0.1, 0.4),
+ 'gemini-2.5-flash-native-audio': (0.5, 12.0), # Audio output
+ 'gemini-2.5-flash-preview-native-audio-dialog': (0.5, 12.0),
+ 'gemini-2.5-flash-exp-native-audio-thinking-dialog': (0.5, 12.0),
+ 'gemini-2.5-flash-preview-tts': (0.5, 10.0),
+ 'gemini-2.5-pro-preview-tts': (1.0, 20.0),
+
+ # Gemini 2.0 Series
+ 'gemini-2.0-flash': (0.1, 0.4),
+ 'gemini-2.0-flash-lite': (0.075, 0.3),
+ 'gemini-2.0-flash-live': (0.35, 1.5),
+ 'gemini-2.0-flash-live-001': (0.35, 1.5),
+ 'gemini-live-2.5-flash-preview': (0.35, 1.5),
+
+ # Gemini 1.5 Series
+ 'gemini-1.5-flash': (0.075, 0.3), # ≤128k tokens
+ 'gemini-1.5-flash-8b': (0.0375, 0.15),
+ 'gemini-1.5-pro': (1.25, 5.0),
+
+ # Legacy/Deprecated
+ 'gemini-1.0-pro': (0.5, 1.5),
+ 'gemini-pro': (0.5, 1.5),
+
+ # Experimental
+ 'gemini-exp': (1.25, 5.0),
+
+ 'default': (0.3, 2.5)
+ },
+ 'mistral': {
+ 'mistral-large': (3.0, 9.0),
+ 'mistral-large-2': (3.0, 9.0),
+ 'mistral-medium': (0.4, 2.0),
+ 'mistral-medium-3': (0.4, 2.0),
+ 'mistral-small': (1.0, 3.0),
+ 'mistral-small-v24.09': (1.0, 3.0),
+ 'mistral-nemo': (0.3, 0.3),
+ 'mixtral-8x7b': (0.24, 0.24),
+ 'mixtral-8x22b': (1.0, 3.0),
+ 'codestral': (0.1, 0.3),
+ 'ministral': (0.1, 0.3),
+ 'default': (0.4, 2.0)
+ },
+ 'groq': {
+ 'llama-4-scout': (0.11, 0.34), # Official pricing
+ 'llama-4-maverick': (0.5, 0.77), # Official pricing
+ 'llama-3.1-405b': (2.5, 2.5),
+ 'llama-3.1-70b': (0.59, 0.79),
+ 'llama-3.1-8b': (0.05, 0.1),
+ 'llama-3-70b': (0.59, 0.79),
+ 'llama-3-8b': (0.05, 0.1),
+ 'mixtral-8x7b': (0.24, 0.24),
+ 'gemma-7b': (0.07, 0.07),
+ 'gemma2-9b': (0.1, 0.1),
+ 'default': (0.3, 0.3)
+ },
+ 'deepseek': {
+ 'deepseek-v3': (0.27, 1.09), # Regular price
+ 'deepseek-v3-promo': (0.14, 0.27), # Promo until Feb 8
+ 'deepseek-chat': (0.27, 1.09),
+ 'deepseek-r1': (0.27, 1.09),
+ 'deepseek-reasoner': (0.27, 1.09),
+ 'deepseek-coder': (0.14, 0.14),
+ 'default': (0.27, 1.09)
+ },
+ 'cohere': {
+ 'command-a': (2.5, 10.0),
+ 'command-r-plus': (2.5, 10.0),
+ 'command-r+': (2.5, 10.0),
+ 'command-r': (0.15, 0.6),
+ 'command-r7b': (0.0375, 0.15),
+ 'command': (1.0, 3.0),
+ 'default': (0.5, 2.0)
+ }
+ }
+
+ provider_prices = token_prices.get(provider, {'default': (2.5, 10.0)})
+
+ # Find the right price for this model
+ price_tuple = provider_prices.get('default', (2.5, 10.0))
+ model_lower = model.lower()
+
+ # Try exact match first
+ if model_lower in provider_prices:
+ price_tuple = provider_prices[model_lower]
+ else:
+ # Try prefix matching
+ for model_key, price in provider_prices.items():
+ if model_key == 'default':
+ continue
+ # Remove version numbers for matching
+ model_key_clean = model_key.replace('-', '').replace('.', '')
+ model_lower_clean = model_lower.replace('-', '').replace('.', '')
+
+ if (model_lower.startswith(model_key) or
+ model_lower_clean.startswith(model_key_clean) or
+ model_key in model_lower):
+ price_tuple = price
+ break
+
+ # Calculate weighted average price based on compression_factor
+ input_price, output_price = price_tuple
+ input_ratio = 1 / (1 + compression_factor)
+ output_ratio = compression_factor / (1 + compression_factor)
+ price_per_million = (input_ratio * input_price) + (output_ratio * output_price)
+
+ # Calculate total tokens
+ # For translation: output is typically 1.2-1.5x input length
+ output_multiplier = compression_factor # Conservative estimate
+ total_tokens_per_chapter = avg_tokens_per_chapter * (1 + output_multiplier)
+ total_tokens = num_chapters * total_tokens_per_chapter
+
+ # Convert to cost
+ regular_cost = (total_tokens / 1_000_000) * price_per_million
+
+ # Batch API discount (50% off)
+ discount = self.PROVIDER_CONFIGS.get(provider, {}).get('discount', 0.5)
+ async_cost = regular_cost * discount
+
+ # Log for debugging
+ logger.info(f"Cost calculation for {model}:")
+ logger.info(f" Provider: {provider}")
+ logger.info(f" Input price: ${input_price:.4f}/1M tokens")
+ logger.info(f" Output price: ${output_price:.4f}/1M tokens")
+ logger.info(f" Compression factor: {compression_factor}")
+ logger.info(f" Weighted avg price: ${price_per_million:.4f}/1M tokens")
+ logger.info(f" Chapters: {num_chapters}")
+ logger.info(f" Avg input tokens/chapter: {avg_tokens_per_chapter:,}")
+ logger.info(f" Total tokens (input+output): {total_tokens:,}")
+ logger.info(f" Regular cost: ${regular_cost:.4f}")
+ logger.info(f" Async cost (50% off): ${async_cost:.4f}")
+
+ return (async_cost, regular_cost)
+
+ def prepare_batch_request(self, chapters: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+ """Prepare batch request for provider
+
+ Args:
+ chapters: List of chapter data with prompts
+ model: Model name
+
+ Returns:
+ Provider-specific batch request format
+ """
+ provider = self.get_provider_from_model(model)
+
+ if provider == 'openai':
+ return self._prepare_openai_batch(chapters, model)
+ elif provider == 'anthropic':
+ return self._prepare_anthropic_batch(chapters, model)
+ elif provider == 'gemini':
+ return self._prepare_gemini_batch(chapters, model)
+ elif provider == 'mistral':
+ return self._prepare_mistral_batch(chapters, model)
+ elif provider == 'groq':
+ return self._prepare_groq_batch(chapters, model)
+ else:
+ raise ValueError(f"Unsupported provider for async: {provider}")
+
+ def _prepare_openai_batch(self, chapters: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+ """Prepare OpenAI batch format"""
+
+ # CRITICAL: Map to exact supported model names
+ supported_batch_models = {
+ # Current models (as of July 2025)
+ 'gpt-4o': 'gpt-4o',
+ 'gpt-4o-mini': 'gpt-4o-mini',
+ 'gpt-4-turbo': 'gpt-4-turbo',
+ 'gpt-4-turbo-preview': 'gpt-4-turbo',
+ 'gpt-3.5-turbo': 'gpt-3.5-turbo',
+ 'gpt-3.5': 'gpt-3.5-turbo',
+
+ # New GPT-4.1 models (if available in your region)
+ 'gpt-4.1': 'gpt-4.1',
+ 'gpt-4.1-mini': 'gpt-4.1-mini',
+ 'gpt-4o-nano': 'gpt-4o-nano',
+
+ # Legacy models (may still work)
+ 'gpt-4': 'gpt-4',
+ 'gpt-4-0613': 'gpt-4-0613',
+ 'gpt-4-0314': 'gpt-4-0314',
+ }
+
+ # Check if model is supported
+ model_lower = model.lower()
+ actual_model = None
+
+ for key, value in supported_batch_models.items():
+ if model_lower == key.lower() or model_lower.startswith(key.lower()):
+ actual_model = value
+ break
+
+ if not actual_model:
+ print(f"Model '{model}' is not supported for batch processing!")
+ print(f"Supported models: {list(supported_batch_models.values())}")
+ raise ValueError(f"Model '{model}' is not supported for OpenAI Batch API")
+
+ logger.info(f"Using batch-supported model: '{actual_model}' (from '{model}')")
+
+ requests = []
+
+ for chapter in chapters:
+ # Validate messages
+ messages = chapter.get('messages', [])
+ if not messages:
+ print(f"Chapter {chapter['id']} has no messages!")
+ continue
+
+ # Ensure all messages have required fields
+ valid_messages = []
+ for msg in messages:
+ if not msg.get('role') or not msg.get('content'):
+ print(f"Skipping invalid message: {msg}")
+ continue
+
+ # Ensure content is string and not empty
+ content = str(msg['content']).strip()
+ if not content:
+ print(f"Skipping message with empty content")
+ continue
+
+ valid_messages.append({
+ 'role': msg['role'],
+ 'content': content
+ })
+
+ if not valid_messages:
+ print(f"No valid messages for chapter {chapter['id']}")
+ continue
+
+ request = {
+ "custom_id": chapter['id'],
+ "method": "POST",
+ "url": "/v1/chat/completions",
+ "body": {
+ "model": actual_model,
+ "messages": valid_messages,
+ "temperature": float(chapter.get('temperature', 0.3)),
+ "max_tokens": int(chapter.get('max_tokens', 8192))
+ }
+ }
+ # LOG THE FIRST REQUEST COMPLETELY
+ if len(requests) == 0:
+ print(f"=== FIRST REQUEST ===")
+ print(json.dumps(request, indent=2))
+ print(f"=== END FIRST REQUEST ===")
+
+ requests.append(request)
+
+ return {"requests": requests}
+
+ def _prepare_anthropic_batch(self, chapters: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+ """Prepare Anthropic batch format"""
+ requests = []
+
+ for chapter in chapters:
+ # Extract system message if present
+ system = None
+ messages = []
+
+ for msg in chapter['messages']:
+ if msg['role'] == 'system':
+ system = msg['content']
+ else:
+ messages.append(msg)
+
+ request = {
+ "custom_id": chapter['id'],
+ "params": {
+ "model": model,
+ "messages": messages,
+ "max_tokens": chapter.get('max_tokens', 8192),
+ "temperature": chapter.get('temperature', 0.3)
+ }
+ }
+
+ if system:
+ request["params"]["system"] = system
+
+ requests.append(request)
+
+ return {"requests": requests}
+
+ def _prepare_gemini_batch(self, chapters: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+ """Prepare Gemini batch format"""
+ requests = []
+
+ for chapter in chapters:
+ # Format messages for Gemini
+ prompt = self._format_messages_for_gemini(chapter['messages'])
+
+ request = {
+ "custom_id": chapter['id'],
+ "generateContentRequest": {
+ "model": f"models/{model}",
+ "contents": [{"parts": [{"text": prompt}]}],
+ "generationConfig": {
+ "temperature": chapter.get('temperature', 0.3),
+ "maxOutputTokens": chapter.get('max_tokens', 8192)
+ }
+ }
+ }
+
+ # Add safety settings if disabled
+ if os.getenv("DISABLE_GEMINI_SAFETY", "false").lower() == "true":
+ request["generateContentRequest"]["safetySettings"] = [
+ {"category": cat, "threshold": "BLOCK_NONE"}
+ for cat in ["HARM_CATEGORY_HARASSMENT", "HARM_CATEGORY_HATE_SPEECH",
+ "HARM_CATEGORY_SEXUALLY_EXPLICIT", "HARM_CATEGORY_DANGEROUS_CONTENT",
+ "HARM_CATEGORY_CIVIC_INTEGRITY"]
+ ]
+
+ requests.append(request)
+
+ return {"requests": requests}
+
+ def _prepare_mistral_batch(self, chapters: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+ """Prepare Mistral batch format"""
+ requests = []
+
+ for chapter in chapters:
+ request = {
+ "custom_id": chapter['id'],
+ "model": model,
+ "messages": chapter['messages'],
+ "temperature": chapter.get('temperature', 0.3),
+ "max_tokens": chapter.get('max_tokens', 8192)
+ }
+ requests.append(request)
+
+ return {"requests": requests}
+
+ def _prepare_groq_batch(self, chapters: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+ """Prepare Groq batch format (OpenAI-compatible)"""
+ return self._prepare_openai_batch(chapters, model)
+
+ def _format_messages_for_gemini(self, messages: List[Dict[str, str]]) -> str:
+ """Format messages for Gemini prompt"""
+ formatted_parts = []
+
+ for msg in messages:
+ role = msg.get('role', 'user').upper()
+ content = msg['content']
+
+ if role == 'SYSTEM':
+ formatted_parts.append(f"INSTRUCTIONS: {content}")
+ else:
+ formatted_parts.append(f"{role}: {content}")
+
+ return "\n\n".join(formatted_parts)
+
+ async def submit_batch(self, batch_data: Dict[str, Any], model: str, api_key: str) -> AsyncJobInfo:
+ """Submit batch to provider and create job entry"""
+ provider = self.get_provider_from_model(model)
+
+ if provider == 'openai':
+ return await self._submit_openai_batch(batch_data, model, api_key)
+ elif provider == 'anthropic':
+ return await self._submit_anthropic_batch(batch_data, model, api_key)
+ elif provider == 'gemini':
+ return await self._submit_gemini_batch(batch_data, model, api_key)
+ elif provider == 'mistral':
+ return await self._submit_mistral_batch(batch_data, model, api_key)
+ elif provider == 'groq':
+ return await self._submit_groq_batch(batch_data, model, api_key)
+ else:
+ raise ValueError(f"Unsupported provider: {provider}")
+
+ def _submit_openai_batch_sync(self, batch_data, model, api_key):
+ """Submit OpenAI batch synchronously"""
+ try:
+ # Remove aiofiles import - not needed for sync operations
+ import tempfile
+ import json
+
+ # Create temporary file for batch data
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f:
+ # Write each request as JSONL
+ for request in batch_data['requests']:
+ json.dump(request, f)
+ f.write('\n')
+ temp_path = f.name
+
+ try:
+ # Upload file to OpenAI
+ headers = {'Authorization': f'Bearer {api_key}'}
+
+ with open(temp_path, 'rb') as f:
+ files = {'file': ('batch.jsonl', f, 'application/jsonl')}
+ data = {'purpose': 'batch'}
+
+ response = requests.post(
+ 'https://api.openai.com/v1/files',
+ headers=headers,
+ files=files,
+ data=data
+ )
+
+ if response.status_code != 200:
+ raise Exception(f"File upload failed: {response.text}")
+
+ file_id = response.json()['id']
+
+ # Create batch job
+ batch_request = {
+ 'input_file_id': file_id,
+ 'endpoint': '/v1/chat/completions',
+ 'completion_window': '24h'
+ }
+
+ response = requests.post(
+ 'https://api.openai.com/v1/batches',
+ headers={**headers, 'Content-Type': 'application/json'},
+ json=batch_request
+ )
+
+ if response.status_code != 200:
+ raise Exception(f"Batch creation failed: {response.text}")
+
+ batch_info = response.json()
+
+ # Calculate cost estimate
+ total_tokens = sum(r.get('token_count', 15000) for r in batch_data['requests'])
+ async_cost, _ = self.estimate_cost(
+ len(batch_data['requests']),
+ total_tokens // len(batch_data['requests']),
+ model
+ )
+
+ job = AsyncJobInfo(
+ job_id=batch_info['id'],
+ provider='openai',
+ model=model,
+ status=AsyncAPIStatus.PENDING,
+ created_at=datetime.now(),
+ updated_at=datetime.now(),
+ total_requests=len(batch_data['requests']),
+ cost_estimate=async_cost,
+ metadata={'file_id': file_id, 'batch_info': batch_info}
+ )
+
+ return job
+
+ finally:
+ # Clean up temp file
+ if os.path.exists(temp_path):
+ os.unlink(temp_path)
+
+ except Exception as e:
+ print(f"OpenAI batch submission failed: {e}")
+ raise
+
+ def _submit_anthropic_batch_sync(self, batch_data: Dict[str, Any], model: str, api_key: str) -> AsyncJobInfo:
+ """Submit Anthropic batch (synchronous version)"""
+ try:
+ headers = {
+ 'X-API-Key': api_key,
+ 'Content-Type': 'application/json',
+ 'anthropic-version': '2023-06-01',
+ 'anthropic-beta': 'message-batches-2024-09-24'
+ }
+
+ response = requests.post(
+ 'https://api.anthropic.com/v1/messages/batches',
+ headers=headers,
+ json=batch_data
+ )
+
+ if response.status_code != 200:
+ raise Exception(f"Batch creation failed: {response.text}")
+
+ batch_info = response.json()
+
+ job = AsyncJobInfo(
+ job_id=batch_info['id'],
+ provider='anthropic',
+ model=model,
+ status=AsyncAPIStatus.PENDING,
+ created_at=datetime.now(),
+ updated_at=datetime.now(),
+ total_requests=len(batch_data['requests']),
+ metadata={'batch_info': batch_info}
+ )
+
+ return job
+
+ except Exception as e:
+ print(f"Anthropic batch submission failed: {e}")
+ raise
+
+ def check_job_status(self, job_id: str) -> AsyncJobInfo:
+ """Check the status of a batch job"""
+ job = self.jobs.get(job_id)
+ if not job:
+ raise ValueError(f"Job {job_id} not found")
+
+ try:
+ provider = job.provider
+
+ if provider == 'openai':
+ self._check_openai_status(job)
+ elif provider == 'gemini':
+ self._check_gemini_status(job)
+ elif provider == 'anthropic':
+ self._check_anthropic_status(job)
+ else:
+ print(f"Unknown provider: {provider}")
+
+ # Update timestamp
+ job.updated_at = datetime.now()
+ self._save_jobs()
+
+ except Exception as e:
+ print(f"Error checking job status: {e}")
+ job.metadata['last_error'] = str(e)
+
+ return job
+
+ def _check_gemini_status(self, job: AsyncJobInfo):
+ """Check Gemini batch status"""
+ try:
+ # First try the Python SDK approach
+ try:
+ from google import genai
+
+ api_key = self._get_api_key()
+ client = genai.Client(api_key=api_key)
+
+ # Get batch job status
+ batch_job = client.batches.get(name=job.job_id)
+
+ # Log the actual response for debugging
+ logger.info(f"Gemini batch job state: {batch_job.state.name if hasattr(batch_job, 'state') else 'Unknown'}")
+
+ # Map Gemini states to our status
+ state_map = {
+ 'JOB_STATE_PENDING': AsyncAPIStatus.PENDING,
+ 'JOB_STATE_RUNNING': AsyncAPIStatus.PROCESSING,
+ 'JOB_STATE_SUCCEEDED': AsyncAPIStatus.COMPLETED,
+ 'JOB_STATE_FAILED': AsyncAPIStatus.FAILED,
+ 'JOB_STATE_CANCELLED': AsyncAPIStatus.CANCELLED,
+ 'JOB_STATE_CANCELLING': AsyncAPIStatus.PROCESSING
+ }
+
+ job.status = state_map.get(batch_job.state.name, AsyncAPIStatus.PENDING)
+
+ # Update metadata
+ if not job.metadata:
+ job.metadata = {}
+ if 'batch_info' not in job.metadata:
+ job.metadata['batch_info'] = {}
+
+ job.metadata['batch_info']['state'] = batch_job.state.name
+ job.metadata['raw_state'] = batch_job.state.name
+ job.metadata['last_check'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+ # Try to get progress information
+ if hasattr(batch_job, 'completed_count'):
+ job.completed_requests = batch_job.completed_count
+ elif job.status == AsyncAPIStatus.PROCESSING:
+ # If processing but no progress info, show as 1 to indicate it started
+ job.completed_requests = 1
+ elif job.status == AsyncAPIStatus.COMPLETED:
+ # If completed, all requests are done
+ job.completed_requests = job.total_requests
+
+ # If completed, store the result file info
+ if batch_job.state.name == 'JOB_STATE_SUCCEEDED' and hasattr(batch_job, 'dest'):
+ job.output_file = batch_job.dest.file_name if hasattr(batch_job.dest, 'file_name') else None
+
+ except Exception as sdk_error:
+ # Fallback to REST API if SDK fails
+ print(f"Gemini SDK failed, trying REST API: {sdk_error}")
+
+ api_key = self._get_api_key()
+ headers = {'x-goog-api-key': api_key}
+
+ batch_name = job.job_id if job.job_id.startswith('batches/') else f'batches/{job.job_id}'
+
+ response = requests.get(
+ f'https://generativelanguage.googleapis.com/v1beta/{batch_name}',
+ headers=headers
+ )
+
+ if response.status_code == 200:
+ data = response.json()
+
+ # Update job status
+ state = data.get('metadata', {}).get('state', 'JOB_STATE_PENDING')
+
+ # Map states
+ state_map = {
+ 'JOB_STATE_PENDING': AsyncAPIStatus.PENDING,
+ 'JOB_STATE_RUNNING': AsyncAPIStatus.PROCESSING,
+ 'JOB_STATE_SUCCEEDED': AsyncAPIStatus.COMPLETED,
+ 'JOB_STATE_FAILED': AsyncAPIStatus.FAILED,
+ 'JOB_STATE_CANCELLED': AsyncAPIStatus.CANCELLED,
+ }
+
+ job.status = state_map.get(state, AsyncAPIStatus.PENDING)
+
+ # Extract progress from metadata
+ metadata = data.get('metadata', {})
+
+ # Gemini might provide progress info
+ if 'completedRequestCount' in metadata:
+ job.completed_requests = metadata['completedRequestCount']
+ if 'failedRequestCount' in metadata:
+ job.failed_requests = metadata['failedRequestCount']
+ if 'totalRequestCount' in metadata:
+ job.total_requests = metadata['totalRequestCount']
+
+ # Store raw state
+ if not job.metadata:
+ job.metadata = {}
+ job.metadata['raw_state'] = state
+ job.metadata['last_check'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+ # Check if completed
+ if state == 'JOB_STATE_SUCCEEDED' and 'response' in data:
+ job.status = AsyncAPIStatus.COMPLETED
+ if 'responsesFile' in data.get('response', {}):
+ job.output_file = data['response']['responsesFile']
+ else:
+ print(f"Gemini status check failed: {response.status_code} - {response.text}")
+
+ except Exception as e:
+ print(f"Gemini status check failed: {e}")
+ if not job.metadata:
+ job.metadata = {}
+ job.metadata['last_error'] = str(e)
+
+ def _check_openai_status(self, job: AsyncJobInfo):
+ """Check OpenAI batch status"""
+ try:
+ api_key = self._get_api_key()
+ headers = {'Authorization': f'Bearer {api_key}'}
+
+ response = requests.get(
+ f'https://api.openai.com/v1/batches/{job.job_id}',
+ headers=headers
+ )
+
+ if response.status_code != 200:
+ print(f"Status check failed: {response.text}")
+ return
+
+ data = response.json()
+
+ # Log the full response for debugging
+ logger.debug(f"OpenAI batch status response: {json.dumps(data, indent=2)}")
+ # Check for high failure rate while in progress
+ request_counts = data.get('request_counts', {})
+ total = request_counts.get('total', 0)
+ failed = request_counts.get('failed', 0)
+ completed = request_counts.get('completed', 0)
+
+ # Map OpenAI status to our status
+ status_map = {
+ 'validating': AsyncAPIStatus.PENDING,
+ 'in_progress': AsyncAPIStatus.PROCESSING,
+ 'finalizing': AsyncAPIStatus.PROCESSING,
+ 'completed': AsyncAPIStatus.COMPLETED,
+ 'failed': AsyncAPIStatus.FAILED,
+ 'expired': AsyncAPIStatus.EXPIRED,
+ 'cancelled': AsyncAPIStatus.CANCELLED,
+ 'cancelling': AsyncAPIStatus.CANCELLED,
+ }
+
+ job.status = status_map.get(data['status'], AsyncAPIStatus.PENDING)
+
+ # Update progress
+ request_counts = data.get('request_counts', {})
+ job.completed_requests = request_counts.get('completed', 0)
+ job.failed_requests = request_counts.get('failed', 0)
+ job.total_requests = request_counts.get('total', job.total_requests)
+
+ # Store metadata
+ if not job.metadata:
+ job.metadata = {}
+ job.metadata['raw_state'] = data['status']
+ job.metadata['last_check'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+ # Handle completion
+ if data['status'] == 'completed':
+ # Check if all requests failed
+ if job.failed_requests > 0 and job.completed_requests == 0:
+ print(f"OpenAI job completed but all {job.failed_requests} requests failed")
+ job.status = AsyncAPIStatus.FAILED
+ job.metadata['all_failed'] = True
+
+ # Store error file if available
+ if data.get('error_file_id'):
+ job.metadata['error_file_id'] = data['error_file_id']
+ logger.info(f"Error file available: {data['error_file_id']}")
+ else:
+ # Normal completion with some successes
+ if 'output_file_id' in data and data['output_file_id']:
+ job.output_file = data['output_file_id']
+ logger.info(f"OpenAI job completed with output file: {job.output_file}")
+
+ # If there were also failures, note that
+ if job.failed_requests > 0:
+ job.metadata['partial_failure'] = True
+ print(f"Job completed with {job.failed_requests} failed requests out of {job.total_requests}")
+ else:
+ print(f"OpenAI job marked as completed but no output_file_id found: {data}")
+
+ # Always store error file if present
+ if data.get('error_file_id'):
+ job.metadata['error_file_id'] = data['error_file_id']
+
+ except Exception as e:
+ print(f"OpenAI status check failed: {e}")
+ if not job.metadata:
+ job.metadata = {}
+ job.metadata['last_error'] = str(e)
+
+ def _check_anthropic_status(self, job: AsyncJobInfo):
+ """Check Anthropic batch status"""
+ try:
+ api_key = self._get_api_key()
+ headers = {
+ 'X-API-Key': api_key,
+ 'anthropic-version': '2023-06-01',
+ 'anthropic-beta': 'message-batches-2024-09-24'
+ }
+
+ response = requests.get(
+ f'https://api.anthropic.com/v1/messages/batches/{job.job_id}',
+ headers=headers
+ )
+
+ if response.status_code != 200:
+ print(f"Status check failed: {response.text}")
+ return
+
+ data = response.json()
+
+ # Map Anthropic status
+ status_map = {
+ 'created': AsyncAPIStatus.PENDING,
+ 'processing': AsyncAPIStatus.PROCESSING,
+ 'ended': AsyncAPIStatus.COMPLETED,
+ 'failed': AsyncAPIStatus.FAILED,
+ 'expired': AsyncAPIStatus.EXPIRED,
+ 'canceled': AsyncAPIStatus.CANCELLED,
+ }
+
+ job.status = status_map.get(data['processing_status'], AsyncAPIStatus.PENDING)
+
+ # Update progress
+ results_summary = data.get('results_summary', {})
+ job.completed_requests = results_summary.get('succeeded', 0)
+ job.failed_requests = results_summary.get('failed', 0)
+ job.total_requests = results_summary.get('total', job.total_requests)
+
+ # Store metadata
+ if not job.metadata:
+ job.metadata = {}
+ job.metadata['raw_state'] = data['processing_status']
+ job.metadata['last_check'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+ if data.get('results_url'):
+ job.output_file = data['results_url']
+
+ except Exception as e:
+ print(f"Anthropic status check failed: {e}")
+ if not job.metadata:
+ job.metadata = {}
+ job.metadata['last_error'] = str(e)
+
+ def _get_api_key(self) -> str:
+ """Get API key from GUI settings"""
+ if hasattr(self.gui, 'api_key_entry'):
+ return self.gui.api_key_entry.get().strip()
+ elif hasattr(self.gui, 'api_key_var'):
+ return self.gui.api_key_var.get().strip()
+ else:
+ # Fallback to environment variable
+ return os.getenv('API_KEY', '') or os.getenv('GEMINI_API_KEY', '') or os.getenv('GOOGLE_API_KEY', '')
+
+ def retrieve_results(self, job_id: str) -> List[Dict[str, Any]]:
+ """Retrieve results from a completed batch job"""
+ job = self.jobs.get(job_id)
+ if not job:
+ raise ValueError(f"Job {job_id} not found")
+
+ if job.status != AsyncAPIStatus.COMPLETED:
+ raise ValueError(f"Job is not completed. Current status: {job.status.value}")
+
+ # If output file is missing, try to refresh status first
+ if not job.output_file:
+ print(f"No output file for completed job {job_id}, refreshing status...")
+ self.check_job_status(job_id)
+
+ # Re-check after status update
+ if not job.output_file:
+ # Log the job details for debugging
+ print(f"Job details: {json.dumps(job.to_dict(), indent=2)}")
+ raise ValueError(f"No output file available for job {job_id} even after status refresh")
+
+ provider = job.provider
+
+ if provider == 'openai':
+ return self._retrieve_openai_results(job)
+ elif provider == 'gemini':
+ return self._retrieve_gemini_results(job)
+ elif provider == 'anthropic':
+ return self._retrieve_anthropic_results(job)
+ else:
+ raise ValueError(f"Unknown provider: {provider}")
+
+ def _retrieve_gemini_results(self, job: AsyncJobInfo) -> List[Dict[str, Any]]:
+ """Retrieve Gemini batch results"""
+ try:
+ from google import genai
+
+ api_key = self._get_api_key()
+
+ # Create client with API key
+ client = genai.Client(api_key=api_key)
+
+ # Get the batch job
+ batch_job = client.batches.get(name=job.job_id)
+
+ if batch_job.state != 'JOB_STATE_SUCCEEDED':
+ raise ValueError(f"Batch job not completed: {batch_job.state}")
+
+ # Download results
+ if hasattr(batch_job, 'dest') and batch_job.dest:
+ # Extract the file name from the destination object
+ if hasattr(batch_job.dest, 'output_uri'):
+ # For BigQuery or Cloud Storage destinations
+ file_name = batch_job.dest.output_uri
+ elif hasattr(batch_job.dest, 'file_name'):
+ # For file-based destinations
+ file_name = batch_job.dest.file_name
+ else:
+ # Try to get any file reference from the dest object
+ # Log the object to understand its structure
+ logger.info(f"BatchJobDestination object: {batch_job.dest}")
+ logger.info(f"BatchJobDestination attributes: {dir(batch_job.dest)}")
+ raise ValueError(f"Cannot extract file name from destination: {batch_job.dest}")
+
+ # Download the results file
+ results_content_bytes = client.files.download(file=file_name)
+ results_content = results_content_bytes.decode('utf-8')
+
+ results = []
+ # Parse JSONL results
+ for line in results_content.splitlines():
+ if line.strip():
+ result_data = json.loads(line)
+
+ # Extract the response content
+ text_content = ""
+
+ # Handle different response formats
+ if 'response' in result_data:
+ response = result_data['response']
+
+ # Check for different content structures
+ if isinstance(response, dict):
+ if 'candidates' in response and response['candidates']:
+ candidate = response['candidates'][0]
+ if 'content' in candidate and 'parts' in candidate['content']:
+ for part in candidate['content']['parts']:
+ if 'text' in part:
+ text_content += part['text']
+ elif 'text' in candidate:
+ text_content = candidate['text']
+ elif 'text' in response:
+ text_content = response['text']
+ elif 'content' in response:
+ text_content = response['content']
+ elif isinstance(response, str):
+ text_content = response
+
+ results.append({
+ 'custom_id': result_data.get('key', ''),
+ 'content': text_content,
+ 'finish_reason': 'stop'
+ })
+
+ return results
+ else:
+ raise ValueError("No output file available for completed job")
+
+ except ImportError:
+ raise ImportError(
+ "google-genai package not installed. "
+ "Run: pip install google-genai"
+ )
+ except Exception as e:
+ print(f"Failed to retrieve Gemini results: {e}")
+ raise
+
+ def _retrieve_openai_results(self, job: AsyncJobInfo) -> List[Dict[str, Any]]:
+ """Retrieve OpenAI batch results"""
+ if not job.output_file:
+ # Try one more status check
+ self._check_openai_status(job)
+ if not job.output_file:
+ raise ValueError(f"No output file available for OpenAI job {job.job_id}")
+
+ try:
+ api_key = self._get_api_key()
+ headers = {'Authorization': f'Bearer {api_key}'}
+
+ # Download results file
+ response = requests.get(
+ f'https://api.openai.com/v1/files/{job.output_file}/content',
+ headers=headers
+ )
+
+ if response.status_code != 200:
+ raise Exception(f"Failed to download results: {response.status_code} - {response.text}")
+
+ # Parse JSONL results
+ results = []
+ for line in response.text.strip().split('\n'):
+ if line:
+ try:
+ result = json.loads(line)
+ # Extract the actual response content
+ if 'response' in result and 'body' in result['response']:
+ results.append({
+ 'custom_id': result.get('custom_id', ''),
+ 'content': result['response']['body']['choices'][0]['message']['content'],
+ 'finish_reason': result['response']['body']['choices'][0].get('finish_reason', 'stop')
+ })
+ else:
+ print(f"Unexpected result format: {result}")
+ except json.JSONDecodeError as e:
+ print(f"Failed to parse result line: {line} - {e}")
+
+ return results
+
+ except Exception as e:
+ print(f"Failed to retrieve OpenAI results: {e}")
+ print(f"Job details: {json.dumps(job.to_dict(), indent=2)}")
+ raise
+
+ def _retrieve_anthropic_results(self, job: AsyncJobInfo) -> List[Dict[str, Any]]:
+ """Retrieve Anthropic batch results"""
+ if not job.output_file:
+ raise ValueError("No output file available")
+
+ api_key = self._get_api_key()
+ headers = {
+ 'X-API-Key': api_key,
+ 'anthropic-version': '2023-06-01'
+ }
+
+ # Download results
+ response = requests.get(job.output_file, headers=headers)
+
+ if response.status_code != 200:
+ raise Exception(f"Failed to download results: {response.text}")
+
+ # Parse JSONL results
+ results = []
+ for line in response.text.strip().split('\n'):
+ if line:
+ result = json.loads(line)
+ if result['result']['type'] == 'succeeded':
+ message = result['result']['message']
+ results.append({
+ 'custom_id': result['custom_id'],
+ 'content': message['content'][0]['text'],
+ 'finish_reason': message.get('stop_reason', 'stop')
+ })
+
+ return results
+
+
+class AsyncProcessingDialog:
+ """GUI dialog for async processing"""
+
+ def __init__(self, parent, translator_gui):
+ """Initialize dialog
+
+ Args:
+ parent: Parent window
+ translator_gui: Reference to main TranslatorGUI instance
+ """
+ self.parent = parent
+ self.gui = translator_gui
+
+ # Fix for PyInstaller - ensure processor uses correct directory
+ self.processor = AsyncAPIProcessor(translator_gui)
+
+ # If running as exe, update the jobs file path
+ if getattr(sys, 'frozen', False):
+ # Running as compiled exe
+ application_path = os.path.dirname(sys.executable)
+ self.processor.jobs_file = os.path.join(application_path, 'async_jobs.json')
+ # Reload jobs from the correct location
+ self.processor._load_jobs()
+
+ self.selected_job_id = None
+ self.polling_jobs = set() # Track which jobs are being polled
+
+ # Use the correct attribute name 'wm' instead of 'window_manager'
+ self.window_manager = translator_gui.wm # WindowManager is stored as 'wm'
+
+ self._create_dialog()
+ self._refresh_jobs_list()
+
+ def _create_dialog(self):
+ """Create the async processing dialog"""
+ # Create scrollable dialog (stays hidden)
+ self.dialog, scrollable_frame, canvas = self.window_manager.setup_scrollable(
+ self.parent,
+ "Async Batch Processing (50% Discount)",
+ width=0, # Will be auto-sized
+ height=None,
+ max_width_ratio=0.9,
+ max_height_ratio=1.00
+ )
+
+ # Store references
+ self.scrollable_frame = scrollable_frame
+ self.canvas = canvas
+
+ # Main container in scrollable_frame
+ main_frame = ttk.Frame(scrollable_frame)
+ main_frame.pack(fill=tk.BOTH, expand=True, padx=10, pady=10)
+
+ # Top section - Information and controls
+ self._create_info_section(main_frame)
+
+ # Middle section - Configuration
+ self._create_config_section(main_frame)
+
+ # Bottom section - Active jobs
+ self._create_jobs_section(main_frame)
+
+ # Button frame goes in the DIALOG, not scrollable_frame
+ button_frame = ttk.Frame(self.dialog)
+ button_frame.pack(fill=tk.X, padx=10, pady=10)
+ self._create_button_frame(button_frame)
+
+ # Load active jobs
+ self._refresh_jobs_list()
+
+ # Auto-resize and show - THIS is what applies the height ratio!
+ self.window_manager.auto_resize_dialog(
+ self.dialog,
+ canvas,
+ max_width_ratio=0.9,
+ max_height_ratio=0.92 # Can override to any value like 1.43
+ )
+
+ # Handle window close
+ self.dialog.protocol("WM_DELETE_WINDOW",
+ lambda: [self.dialog._cleanup_scrolling(), self.dialog.destroy()])
+
+ self._start_auto_refresh(30)
+
+ def _create_info_section(self, parent):
+ """Create information section"""
+ info_frame = ttk.LabelFrame(parent, text="Async Processing Information", padding=10)
+ info_frame.pack(fill=tk.X, pady=(0, 10))
+
+ # Model and provider info
+ model_frame = ttk.Frame(info_frame)
+ model_frame.pack(fill=tk.X)
+
+ ttk.Label(model_frame, text="Current Model:").pack(side=tk.LEFT, padx=(0, 5))
+ model_name = self.gui.model_var.get() if hasattr(self.gui, 'model_var') else "Not selected"
+ self.model_label = ttk.Label(model_frame, text=model_name, style="info.TLabel")
+ self.model_label.pack(side=tk.LEFT, padx=(0, 20))
+
+ # Check if model supports async
+ provider = self.processor.get_provider_from_model(model_name)
+ if provider and provider in self.processor.PROVIDER_CONFIGS:
+ status_text = f"✓ Supported ({provider.upper()})"
+ status_style = "success.TLabel"
+ else:
+ status_text = "✗ Not supported for async"
+ status_style = "danger.TLabel"
+
+ ttk.Label(model_frame, text=status_text, style=status_style).pack(side=tk.LEFT)
+
+ # Cost estimation
+ cost_frame = ttk.Frame(info_frame)
+ cost_frame.pack(fill=tk.X, pady=(10, 0))
+
+ ttk.Label(cost_frame, text="Cost Estimation:", font=("", 10, "bold")).pack(anchor=tk.W)
+
+ self.cost_info_label = ttk.Label(cost_frame, text="Select chapters to see cost estimate")
+ self.cost_info_label.pack(anchor=tk.W, pady=(5, 0))
+
+ def _create_config_section(self, parent):
+ """Create configuration section"""
+ config_frame = ttk.LabelFrame(parent, text="Async Processing Configuration", padding=10)
+ config_frame.pack(fill=tk.X, pady=(0, 10))
+
+ # Processing options
+ options_frame = ttk.Frame(config_frame)
+ options_frame.pack(fill=tk.X)
+
+ # Wait for completion
+ self.wait_for_completion_var = tk.BooleanVar(value=False)
+ ttk.Checkbutton(
+ options_frame,
+ text="Wait for completion (blocks GUI)",
+ variable=self.wait_for_completion_var
+ ).pack(anchor=tk.W)
+
+ # Poll interval
+ poll_frame = ttk.Frame(options_frame)
+ poll_frame.pack(fill=tk.X, pady=(5, 0))
+
+ ttk.Label(poll_frame, text="Poll interval (seconds):").pack(side=tk.LEFT, padx=(0, 5))
+ self.poll_interval_var = tk.IntVar(value=60)
+ ttk.Spinbox(
+ poll_frame,
+ from_=10,
+ to=600,
+ textvariable=self.poll_interval_var,
+ width=10
+ ).pack(side=tk.LEFT)
+
+ # Chapter selection info
+ chapter_frame = ttk.Frame(config_frame)
+ chapter_frame.pack(fill=tk.X, pady=(10, 0))
+
+ self.chapter_info_label = ttk.Label(
+ chapter_frame,
+ text="Note: Async processing will skip chapters that require chunking",
+ style="warning.TLabel"
+ )
+ self.chapter_info_label.pack(anchor=tk.W)
+
+ def _create_jobs_section(self, parent):
+ """Create active jobs section"""
+ jobs_frame = ttk.LabelFrame(parent, text="Active Async Jobs", padding=10)
+ jobs_frame.pack(fill=tk.BOTH, expand=True, pady=(0, 10))
+
+ # Jobs treeview
+ tree_frame = ttk.Frame(jobs_frame)
+ tree_frame.pack(fill=tk.BOTH, expand=True)
+
+ # Scrollbars
+ vsb = ttk.Scrollbar(tree_frame, orient="vertical")
+ hsb = ttk.Scrollbar(tree_frame, orient="horizontal")
+
+ # Treeview
+ self.jobs_tree = ttk.Treeview(
+ tree_frame,
+ columns=("Provider", "Model", "Status", "Progress", "Created", "Cost"),
+ show="tree headings",
+ yscrollcommand=vsb.set,
+ xscrollcommand=hsb.set
+ )
+
+ vsb.config(command=self.jobs_tree.yview)
+ hsb.config(command=self.jobs_tree.xview)
+
+ # Add a progress bar for the selected job
+ progress_frame = ttk.Frame(jobs_frame)
+ progress_frame.pack(fill=tk.X, pady=(5, 0))
+
+ ttk.Label(progress_frame, text="Selected Job Progress:").pack(side=tk.LEFT, padx=(0, 5))
+
+ self.job_progress_bar = ttk.Progressbar(
+ progress_frame,
+ mode='determinate',
+ style='success.Horizontal.TProgressbar'
+ )
+ self.job_progress_bar.pack(side=tk.LEFT, fill=tk.X, expand=True)
+
+ self.progress_label = ttk.Label(progress_frame, text="0%")
+ self.progress_label.pack(side=tk.LEFT, padx=(5, 0))
+
+ # Configure columns
+ self.jobs_tree.heading("#0", text="Job ID")
+ self.jobs_tree.heading("Provider", text="Provider")
+ self.jobs_tree.heading("Model", text="Model")
+ self.jobs_tree.heading("Status", text="Status")
+ self.jobs_tree.heading("Progress", text="Progress")
+ self.jobs_tree.heading("Created", text="Created")
+ self.jobs_tree.heading("Cost", text="Est. Cost")
+
+ self.jobs_tree.column("#0", width=200)
+ self.jobs_tree.column("Provider", width=100)
+ self.jobs_tree.column("Model", width=150)
+ self.jobs_tree.column("Status", width=100)
+ self.jobs_tree.column("Progress", width=150)
+ self.jobs_tree.column("Created", width=150)
+ self.jobs_tree.column("Cost", width=100)
+
+ # Add right-click menu
+ self.jobs_context_menu = tk.Menu(self.jobs_tree, tearoff=0)
+ self.jobs_context_menu.add_command(label="Check Status", command=self._check_selected_status)
+ self.jobs_context_menu.add_command(label="Retrieve Results", command=self._retrieve_selected_results)
+ self.jobs_context_menu.add_separator()
+ self.jobs_context_menu.add_command(label="Delete", command=self._delete_selected_job)
+
+ # Context menu binding function - use unique name to avoid conflicts
+ def show_jobs_context_menu(event):
+ # Select the item under cursor
+ item = self.jobs_tree.identify_row(event.y)
+ if item:
+ self.jobs_tree.selection_set(item)
+ self._on_job_select(None) # Update selection
+ self.jobs_context_menu.post(event.x_root, event.y_root)
+
+ # Bind right-click
+ self.jobs_tree.bind("", show_jobs_context_menu) # Right-click on Windows/Linux
+ if sys.platform == "darwin":
+ self.jobs_tree.bind("", show_jobs_context_menu) # Right-click on macOS
+
+ # Pack treeview and scrollbars
+ self.jobs_tree.grid(row=0, column=0, sticky="nsew")
+ vsb.grid(row=0, column=1, sticky="ns")
+ hsb.grid(row=1, column=0, sticky="ew")
+
+ tree_frame.grid_rowconfigure(0, weight=1)
+ tree_frame.grid_columnconfigure(0, weight=1)
+
+ # Bind selection
+ self.jobs_tree.bind('<>', self._on_job_select)
+
+ # Job action buttons
+ action_frame = ttk.Frame(jobs_frame)
+ action_frame.pack(fill=tk.X, pady=(10, 0))
+
+ button_width = 15
+
+ ttk.Button(
+ action_frame,
+ text="Check Status",
+ command=self._check_selected_status,
+ style="info.TButton",
+ width=button_width
+ ).pack(side=tk.LEFT, padx=(0, 5))
+
+ ttk.Button(
+ action_frame,
+ text="Retrieve Results",
+ command=self._retrieve_selected_results,
+ style="success.TButton",
+ width=button_width
+ ).pack(side=tk.LEFT, padx=(0, 5))
+
+ ttk.Button(
+ action_frame,
+ text="Cancel Job",
+ command=self._cancel_selected_job,
+ style="warning.TButton",
+ width=button_width
+ ).pack(side=tk.LEFT, padx=(0, 5))
+
+ # delete buttons
+ ttk.Button(
+ action_frame,
+ text="Delete Selected",
+ command=self._delete_selected_job,
+ style="danger.TButton",
+ width=button_width
+ ).pack(side=tk.LEFT, padx=(30, 5)) # Extra padding to separate
+
+ ttk.Button(
+ action_frame,
+ text="Clear Completed",
+ command=self._clear_completed_jobs,
+ style="secondary.TButton",
+ width=button_width
+ ).pack(side=tk.LEFT)
+
+ def _create_button_frame(self, parent):
+ """Create bottom button frame"""
+ button_frame = ttk.Frame(parent)
+ button_frame.pack(fill=tk.X, pady=(20, 0))
+
+ # Start processing button
+ self.start_button = ttk.Button(
+ button_frame,
+ text="Start Async Processing",
+ command=self._start_processing,
+ style="success.TButton"
+ )
+ self.start_button.pack(side=tk.LEFT, padx=(0, 5))
+
+ # Estimate only button
+ ttk.Button(
+ button_frame,
+ text="Estimate Cost Only",
+ command=self._estimate_cost,
+ style="info.TButton"
+ ).pack(side=tk.LEFT, padx=(0, 5))
+
+ # Close button - need to handle cleanup if using WindowManager
+ if hasattr(self.dialog, '_cleanup_scrolling'):
+ ttk.Button(
+ button_frame,
+ text="Close",
+ command=lambda: [self.dialog._cleanup_scrolling(), self.dialog.destroy()]
+ ).pack(side=tk.RIGHT)
+ else:
+ ttk.Button(
+ button_frame,
+ text="Close",
+ command=self.dialog.destroy
+ ).pack(side=tk.RIGHT)
+
+ def _update_selected_job_progress(self, job):
+ """Update progress display for selected job"""
+ if hasattr(self, 'job_progress_bar'):
+ if job.total_requests > 0:
+ progress = int((job.completed_requests / job.total_requests) * 100)
+ self.job_progress_bar['value'] = progress
+
+ # Update progress label if exists
+ if hasattr(self, 'progress_label'):
+ self.progress_label.config(
+ text=f"{progress}% ({job.completed_requests}/{job.total_requests} chapters)"
+ )
+ else:
+ self.job_progress_bar['value'] = 0
+ if hasattr(self, 'progress_label'):
+ self.progress_label.config(text="0% (Waiting)")
+
+ def _refresh_jobs_list(self):
+ """Refresh the jobs list"""
+ # Clear existing items
+ for item in self.jobs_tree.get_children():
+ self.jobs_tree.delete(item)
+
+ # Add jobs
+ for job_id, job in self.processor.jobs.items():
+ # Calculate progress percentage and format progress text
+ if job.total_requests > 0:
+ progress_pct = int((job.completed_requests / job.total_requests) * 100)
+ progress_text = f"{progress_pct}% ({job.completed_requests}/{job.total_requests})"
+ else:
+ progress_pct = 0
+ progress_text = "0% (0/0)"
+
+ # Override progress text for completed/failed/cancelled statuses
+ if job.status == AsyncAPIStatus.COMPLETED:
+ progress_text = "100% (Complete)"
+ elif job.status == AsyncAPIStatus.FAILED:
+ progress_text = f"{progress_pct}% (Failed)"
+ elif job.status == AsyncAPIStatus.CANCELLED:
+ progress_text = f"{progress_pct}% (Cancelled)"
+ elif job.status == AsyncAPIStatus.PENDING:
+ progress_text = "0% (Waiting)"
+
+ created = job.created_at.strftime("%Y-%m-%d %H:%M")
+ cost = f"${job.cost_estimate:.2f}" if job.cost_estimate else "N/A"
+
+ # Determine status style
+ status_text = job.status.value.capitalize()
+
+ # Shorten job ID for display
+ display_id = job_id[:20] + "..." if len(job_id) > 20 else job_id
+
+ self.jobs_tree.insert(
+ "",
+ "end",
+ text=display_id,
+ values=(
+ job.provider.upper(),
+ job.model[:15] + "..." if len(job.model) > 15 else job.model, # Shorten model name
+ status_text,
+ progress_text, # Now shows percentage and counts
+ created,
+ cost
+ ),
+ tags=(job.status.value,)
+ )
+
+ # Configure tags for status colors
+ self.jobs_tree.tag_configure("pending", foreground="#FFA500") # Orange
+ self.jobs_tree.tag_configure("processing", foreground="#007BFF") # Blue
+ self.jobs_tree.tag_configure("completed", foreground="#28A745") # Green
+ self.jobs_tree.tag_configure("failed", foreground="#DC3545") # Red
+ self.jobs_tree.tag_configure("cancelled", foreground="#6C757D") # Gray
+
+ # Update progress bar if a job is selected
+ if hasattr(self, 'selected_job_id') and self.selected_job_id:
+ job = self.processor.jobs.get(self.selected_job_id)
+ if job:
+ self._update_selected_job_progress(job)
+
+ def _on_job_select(self, event):
+ """Handle job selection"""
+ selection = self.jobs_tree.selection()
+ if selection:
+ item = self.jobs_tree.item(selection[0])
+ # Get full job ID from the item
+ job_id_prefix = item['text'].rstrip('...')
+
+ # Find matching job
+ for job_id in self.processor.jobs:
+ if job_id.startswith(job_id_prefix):
+ self.selected_job_id = job_id
+
+ # Update progress display for selected job
+ job = self.processor.jobs.get(job_id)
+ if job:
+ # Update progress bar if it exists
+ if hasattr(self, 'job_progress_bar'):
+ if job.total_requests > 0:
+ progress = int((job.completed_requests / job.total_requests) * 100)
+ self.job_progress_bar['value'] = progress
+ else:
+ self.job_progress_bar['value'] = 0
+
+ # Update progress label if it exists
+ if hasattr(self, 'progress_label'):
+ if job.total_requests > 0:
+ progress = int((job.completed_requests / job.total_requests) * 100)
+ self.progress_label.config(
+ text=f"{progress}% ({job.completed_requests}/{job.total_requests} chapters)"
+ )
+ else:
+ self.progress_label.config(text="0% (Waiting)")
+
+ # Log selection
+ logger.info(f"Selected job: {job_id[:30]}... - Status: {job.status.value}")
+
+ break
+
+ def _check_selected_status(self):
+ """Check status of selected job"""
+ if not self.selected_job_id:
+ messagebox.showwarning("No Selection", "Please select a job to check status")
+ return
+
+ try:
+ job = self.processor.check_job_status(self.selected_job_id)
+ self._refresh_jobs_list()
+
+ # Build detailed status message
+ status_text = f"Job ID: {job.job_id}\n"
+ status_text += f"Provider: {job.provider.upper()}\n"
+ status_text += f"Status: {job.status.value}\n"
+ status_text += f"State: {job.metadata.get('raw_state', 'Unknown')}\n\n"
+
+ # Progress information
+ if job.completed_requests > 0 or job.status == AsyncAPIStatus.PROCESSING:
+ status_text += f"Progress: {job.completed_requests}/{job.total_requests}\n"
+ else:
+ status_text += f"Progress: Waiting to start (0/{job.total_requests})\n"
+
+ status_text += f"Failed: {job.failed_requests}\n\n"
+
+ # Time information
+ status_text += f"Created: {job.created_at.strftime('%Y-%m-%d %H:%M:%S')}\n"
+ status_text += f"Last Updated: {job.updated_at.strftime('%Y-%m-%d %H:%M:%S')}\n"
+
+ if 'last_check' in job.metadata:
+ status_text += f"Last Checked: {job.metadata['last_check']}\n"
+
+ # Show output file if available
+ if job.output_file:
+ status_text += f"\nOutput Ready: {job.output_file}\n"
+
+ messagebox.showinfo("Job Status", status_text)
+
+ except Exception as e:
+ messagebox.showerror("Error", f"Failed to check status: {str(e)}")
+
+ def _start_auto_refresh(self, interval_seconds=30):
+ """Start automatic status refresh"""
+ def refresh():
+ if hasattr(self, 'dialog') and self.dialog.winfo_exists():
+ # Refresh all jobs
+ for job_id in list(self.processor.jobs.keys()):
+ try:
+ job = self.processor.jobs[job_id]
+ if job.status in [AsyncAPIStatus.PENDING, AsyncAPIStatus.PROCESSING]:
+ self.processor.check_job_status(job_id)
+ except:
+ pass
+
+ self._refresh_jobs_list()
+
+ # Schedule next refresh
+ self.dialog.after(interval_seconds * 1000, refresh)
+
+ # Start the refresh cycle
+ refresh()
+
+ def _retrieve_selected_results(self):
+ """Retrieve results from selected job"""
+ if not self.selected_job_id:
+ messagebox.showwarning("No Selection", "Please select a job to retrieve results")
+ return
+
+ # Check job status first
+ job = self.processor.jobs.get(self.selected_job_id)
+ if not job:
+ messagebox.showerror("Error", "Selected job not found")
+ return
+
+ if job.status != AsyncAPIStatus.COMPLETED:
+ messagebox.showwarning(
+ "Job Not Complete",
+ f"This job is currently {job.status.value}.\n"
+ "Only completed jobs can have results retrieved."
+ )
+ return
+
+ try:
+ # Set cursor to busy (with safety check)
+ if hasattr(self, 'dialog') and self.dialog.winfo_exists():
+ self.dialog.config(cursor="wait")
+ self.dialog.update()
+
+ # Retrieve results
+ self._handle_completed_job(self.selected_job_id)
+
+ except Exception as e:
+ self._log(f"❌ Error retrieving results: {e}")
+ messagebox.showerror("Error", f"Failed to retrieve results: {str(e)}")
+ finally:
+ # Reset cursor (with safety check)
+ if hasattr(self, 'dialog') and self.dialog.winfo_exists():
+ try:
+ self.dialog.config(cursor="")
+ except tk.TclError:
+ # Dialog was closed, ignore
+ pass
+
+ def _cancel_selected_job(self):
+ """Cancel selected job"""
+ if not self.selected_job_id:
+ messagebox.showwarning("No Selection", "Please select a job to cancel")
+ return
+
+ job = self.processor.jobs.get(self.selected_job_id)
+ if not job:
+ messagebox.showerror("Error", "Selected job not found")
+ return
+
+ if job.status in [AsyncAPIStatus.COMPLETED, AsyncAPIStatus.FAILED, AsyncAPIStatus.CANCELLED]:
+ messagebox.showwarning(
+ "Cannot Cancel",
+ f"This job is already {job.status.value}"
+ )
+ return
+
+ # Confirm cancellation
+ if not messagebox.askyesno(
+ "Cancel Job",
+ f"Are you sure you want to cancel this job?\n\n"
+ f"Job ID: {job.job_id}\n"
+ f"Status: {job.status.value}"
+ ):
+ return
+
+ try:
+ api_key = self.gui.api_key_entry.get().strip()
+
+ if job.provider == 'openai':
+ headers = {'Authorization': f'Bearer {api_key}'}
+
+ response = requests.post(
+ f'https://api.openai.com/v1/batches/{job.job_id}/cancel',
+ headers=headers
+ )
+
+ if response.status_code == 200:
+ job.status = AsyncAPIStatus.CANCELLED
+ job.updated_at = datetime.now()
+ self.processor._save_jobs()
+ self._refresh_jobs_list()
+ messagebox.showinfo("Job Cancelled", "Job has been cancelled successfully")
+ else:
+ messagebox.showerror("Error", f"Failed to cancel job: {response.text}")
+
+ elif job.provider == 'gemini':
+ # Gemini batch cancellation using REST API
+ headers = {'x-goog-api-key': api_key}
+
+ # Format: batches/123456 -> https://generativelanguage.googleapis.com/v1beta/batches/123456:cancel
+ batch_name = job.job_id if job.job_id.startswith('batches/') else f'batches/{job.job_id}'
+
+ response = requests.post(
+ f'https://generativelanguage.googleapis.com/v1beta/{batch_name}:cancel',
+ headers=headers
+ )
+
+ if response.status_code == 200:
+ job.status = AsyncAPIStatus.CANCELLED
+ job.updated_at = datetime.now()
+ self.processor._save_jobs()
+ self._refresh_jobs_list()
+ messagebox.showinfo("Job Cancelled", "Gemini batch job has been cancelled successfully")
+ else:
+ messagebox.showerror("Error", f"Failed to cancel Gemini job: {response.text}")
+
+ elif job.provider == 'anthropic':
+ # Anthropic doesn't support cancellation via API yet
+ messagebox.showinfo(
+ "Not Supported",
+ "Anthropic doesn't support job cancellation via API.\n"
+ "The job will be marked as cancelled locally only."
+ )
+ job.status = AsyncAPIStatus.CANCELLED
+ job.updated_at = datetime.now()
+ self.processor._save_jobs()
+ self._refresh_jobs_list()
+
+ else:
+ # For other providers, just mark as cancelled locally
+ messagebox.showinfo(
+ "Local Cancellation",
+ f"{job.provider.title()} cancellation not implemented.\n"
+ "The job will be marked as cancelled locally only."
+ )
+ job.status = AsyncAPIStatus.CANCELLED
+ job.updated_at = datetime.now()
+ self.processor._save_jobs()
+ self._refresh_jobs_list()
+
+ except Exception as e:
+ messagebox.showerror("Error", f"Failed to cancel job: {str(e)}")
+
+ def _cancel_openai_job(self, job, api_key):
+ """Cancel OpenAI batch job"""
+ headers = {
+ 'Authorization': f'Bearer {api_key}',
+ 'Content-Type': 'application/json'
+ }
+
+ # OpenAI has a specific cancel endpoint
+ cancel_url = f"https://api.openai.com/v1/batches/{job.job_id}/cancel"
+
+ response = requests.post(cancel_url, headers=headers)
+
+ if response.status_code not in [200, 204]:
+ raise Exception(f"OpenAI cancellation failed: {response.text}")
+
+ logger.info(f"OpenAI job {job.job_id} cancelled successfully")
+
+ def _cancel_anthropic_job(self, job, api_key):
+ """Cancel Anthropic batch job"""
+ headers = {
+ 'X-API-Key': api_key,
+ 'anthropic-version': '2023-06-01',
+ 'anthropic-beta': 'message-batches-2024-09-24'
+ }
+
+ # Anthropic uses DELETE method for cancellation
+ cancel_url = f"https://api.anthropic.com/v1/messages/batches/{job.job_id}"
+
+ response = requests.delete(cancel_url, headers=headers)
+
+ if response.status_code not in [200, 204]:
+ raise Exception(f"Anthropic cancellation failed: {response.text}")
+
+ logger.info(f"Anthropic job {job.job_id} cancelled successfully")
+
+ def _cancel_gemini_job(self, job, api_key):
+ """Cancel Gemini batch job"""
+ try:
+ from google import genai
+
+ # Create client
+ client = genai.Client(api_key=api_key)
+
+ # Try to cancel using the SDK
+ # Note: The SDK might not have a cancel method yet
+ if hasattr(client.batches, 'cancel'):
+ client.batches.cancel(name=job.job_id)
+ logger.info(f"Gemini job {job.job_id} cancelled successfully")
+ else:
+ # If SDK doesn't support cancellation, inform the user
+ raise Exception(
+ "Gemini batch cancellation is not supported yet.\n"
+ "The job will continue to run and complete within 24 hours.\n"
+ "You can check the status later to retrieve results."
+ )
+
+ except AttributeError:
+ # SDK doesn't have cancel method
+ raise Exception(
+ "Gemini batch cancellation is not available in the current SDK.\n"
+ "The job will continue to run and complete within 24 hours."
+ )
+ except Exception as e:
+ # Check if it's a permission error
+ if "403" in str(e) or "PERMISSION_DENIED" in str(e):
+ raise Exception(
+ "Gemini batch jobs cannot be cancelled once submitted.\n"
+ "The job will complete within 24 hours and you can retrieve results then."
+ )
+ else:
+ # Re-raise other errors
+ raise
+
+ def _cancel_mistral_job(self, job, api_key):
+ """Cancel Mistral batch job"""
+ headers = {
+ 'Authorization': f'Bearer {api_key}',
+ 'Content-Type': 'application/json'
+ }
+
+ # Mistral batch cancellation endpoint
+ cancel_url = f"https://api.mistral.ai/v1/batch/jobs/{job.job_id}/cancel"
+
+ response = requests.post(cancel_url, headers=headers)
+
+ if response.status_code not in [200, 204]:
+ raise Exception(f"Mistral cancellation failed: {response.text}")
+
+ logger.info(f"Mistral job {job.job_id} cancelled successfully")
+
+ def _cancel_groq_job(self, job, api_key):
+ """Cancel Groq batch job"""
+ # Groq uses OpenAI-compatible endpoints
+ headers = {
+ 'Authorization': f'Bearer {api_key}',
+ 'Content-Type': 'application/json'
+ }
+
+ cancel_url = f"https://api.groq.com/openai/v1/batch/{job.job_id}/cancel"
+
+ response = requests.post(cancel_url, headers=headers)
+
+ if response.status_code not in [200, 204]:
+ raise Exception(f"Groq cancellation failed: {response.text}")
+
+ logger.info(f"Groq job {job.job_id} cancelled successfully")
+
+ def _estimate_cost(self):
+ """Estimate cost for current file"""
+ # Get current file info
+ if not hasattr(self.gui, 'file_path') or not self.gui.file_path:
+ messagebox.showwarning("No File", "Please select a file first")
+ return
+
+ try:
+ # Show analyzing message
+ self.cost_info_label.config(text="Analyzing file...")
+ self.dialog.update()
+
+ file_path = self.gui.file_path
+ model = self.gui.model_var.get()
+
+ # Calculate overhead tokens (system prompt + glossary)
+ overhead_tokens = 0
+
+ # Count system prompt tokens
+ system_prompt = self.gui.prompt_text.get("1.0", "end").strip()
+ if system_prompt:
+ overhead_tokens += self.count_tokens(system_prompt, model)
+ logger.info(f"System prompt tokens: {overhead_tokens}")
+
+ # Count glossary tokens if enabled
+ glossary_tokens = 0
+
+ # Check if glossary should be appended - match the logic from _prepare_environment_variables
+ if (hasattr(self.gui, 'manual_glossary_path') and
+ self.gui.manual_glossary_path and
+ hasattr(self.gui, 'append_glossary_var') and
+ self.gui.append_glossary_var.get()): # This is the key check!
+
+ try:
+ glossary_path = self.gui.manual_glossary_path
+ logger.info(f"Loading glossary from: {glossary_path}")
+
+ if os.path.exists(glossary_path):
+ with open(glossary_path, 'r', encoding='utf-8') as f:
+ glossary_data = json.load(f)
+
+ # Format glossary same way as in translation
+ #glossary_text = self._format_glossary_for_prompt(glossary_data)
+
+ # Add append prompt if available
+ append_prompt = self.gui.append_glossary_prompt if hasattr(self.gui, 'append_glossary_prompt') else ''
+
+ if append_prompt:
+ if '{glossary}' in append_prompt:
+ glossary_text = append_prompt.replace('{glossary}', glossary_text)
+ else:
+ glossary_text = f"{append_prompt}\n{glossary_text}"
+ else:
+ glossary_text = f"Glossary:\n{glossary_text}"
+
+ glossary_tokens = self.count_tokens(glossary_text, model)
+ overhead_tokens += glossary_tokens
+ logger.info(f"Loaded glossary with {glossary_tokens} tokens")
+ else:
+ print(f"Glossary file not found: {glossary_path}")
+
+ except Exception as e:
+ print(f"Failed to load glossary: {e}")
+
+ logger.info(f"Total overhead per chapter: {overhead_tokens} tokens")
+
+ # Actually extract chapters and count tokens
+ num_chapters = 0
+ total_content_tokens = 0 # Just the chapter content
+ chapters_needing_chunking = 0
+
+ if file_path.lower().endswith('.epub'):
+ # Import and use EPUB extraction
+ try:
+ import ebooklib
+ from ebooklib import epub
+ from bs4 import BeautifulSoup
+
+ book = epub.read_epub(file_path)
+ chapters = []
+
+ # Extract text chapters
+ for item in book.get_items():
+ if item.get_type() == ebooklib.ITEM_DOCUMENT:
+ soup = BeautifulSoup(item.get_content(), 'html.parser')
+ text = soup.get_text(separator='\n').strip()
+ if len(text) > 500: # Minimum chapter length
+ chapters.append(text)
+
+ num_chapters = len(chapters)
+
+ # Count tokens for each chapter (sample more for better accuracy)
+ sample_size = min(20, num_chapters) # Sample up to 20 chapters for better accuracy
+ sampled_content_tokens = 0
+
+ for i, chapter_text in enumerate(chapters[:sample_size]):
+ # Count just the content tokens
+ content_tokens = self.count_tokens(chapter_text, model)
+ sampled_content_tokens += content_tokens
+
+ # Check if needs chunking (including overhead)
+ total_chapter_tokens = content_tokens + overhead_tokens
+ token_limit = int(self.gui.token_limit_entry.get() or 200000)
+ if total_chapter_tokens > token_limit * 0.8:
+ chapters_needing_chunking += 1
+
+ # Update progress
+ if i % 5 == 0:
+ self.cost_info_label.config(text=f"Analyzing chapters... {i+1}/{sample_size}")
+ self.dialog.update()
+
+ # Calculate average based on actual sample
+ if sample_size > 0:
+ avg_content_tokens_per_chapter = sampled_content_tokens // sample_size
+ # Extrapolate chunking needs if we didn't sample all
+ if num_chapters > sample_size:
+ chapters_needing_chunking = int(chapters_needing_chunking * (num_chapters / sample_size))
+ else:
+ avg_content_tokens_per_chapter = 15000 # Default
+
+ except Exception as e:
+ print(f"Failed to analyze EPUB: {e}")
+ # Fall back to estimates
+ num_chapters = 50
+ avg_content_tokens_per_chapter = 15000
+
+ elif file_path.lower().endswith('.txt'):
+ # Import and use TXT extraction
+ try:
+ from txt_processor import TextFileProcessor
+
+ processor = TextFileProcessor(file_path, '')
+ chapters = processor.extract_chapters()
+ num_chapters = len(chapters)
+
+ # Count tokens
+ sample_size = min(20, num_chapters) # Sample up to 20 chapters
+ sampled_content_tokens = 0
+
+ for i, chapter_text in enumerate(chapters[:sample_size]):
+ # Count just the content tokens
+ content_tokens = self.count_tokens(chapter_text, model)
+ sampled_content_tokens += content_tokens
+
+ # Check if needs chunking (including overhead)
+ total_chapter_tokens = content_tokens + overhead_tokens
+ token_limit = int(self.gui.token_limit_entry.get() or 200000)
+ if total_chapter_tokens > token_limit * 0.8:
+ chapters_needing_chunking += 1
+
+ # Update progress
+ if i % 5 == 0:
+ self.cost_info_label.config(text=f"Analyzing chapters... {i+1}/{sample_size}")
+ self.dialog.update()
+
+ # Calculate average based on actual sample
+ if sample_size > 0:
+ avg_content_tokens_per_chapter = sampled_content_tokens // sample_size
+ # Extrapolate chunking needs
+ if num_chapters > sample_size:
+ chapters_needing_chunking = int(chapters_needing_chunking * (num_chapters / sample_size))
+ else:
+ avg_content_tokens_per_chapter = 15000 # Default
+
+ except Exception as e:
+ print(f"Failed to analyze TXT: {e}")
+ # Fall back to estimates
+ num_chapters = 50
+ avg_content_tokens_per_chapter = 15000
+ else:
+ # Unsupported format
+ self.cost_info_label.config(
+ text="Unsupported file format. Only EPUB and TXT are supported."
+ )
+ return
+
+ # Calculate costs
+ processable_chapters = num_chapters - chapters_needing_chunking
+
+ if processable_chapters <= 0:
+ self.cost_info_label.config(
+ text=f"Warning: All {num_chapters} chapters require chunking.\n"
+ f"Async APIs do not support chunked chapters.\n"
+ f"Consider using regular batch translation instead."
+ )
+ return
+
+ # Add overhead to get total average tokens per chapter
+ avg_total_tokens_per_chapter = avg_content_tokens_per_chapter + overhead_tokens
+
+ # Get the translation compression factor from GUI
+ compression_factor = float(self.gui.compression_factor_var.get() or 1.0)
+
+ # Get accurate cost estimate
+ async_cost, regular_cost = self.processor.estimate_cost(
+ processable_chapters,
+ avg_total_tokens_per_chapter, # Now includes content + system prompt + glossary
+ model,
+ compression_factor
+ )
+
+ # Update any existing jobs for this file with the accurate estimate
+ current_file = self.gui.file_path
+ for job_id, job in self.processor.jobs.items():
+ # Check if this job is for the current file and model
+ if (job.metadata and
+ job.metadata.get('source_file') == current_file and
+ job.model == model and
+ job.status in [AsyncAPIStatus.PENDING, AsyncAPIStatus.PROCESSING]):
+ # Update the cost estimate
+ job.cost_estimate = async_cost
+ job.updated_at = datetime.now()
+
+ # Save updated jobs
+ self.processor._save_jobs()
+
+ # Refresh the display
+ self._refresh_jobs_list()
+
+ # Build detailed message
+ cost_text = f"File analysis complete!\n\n"
+ cost_text += f"Total chapters: {num_chapters}\n"
+ cost_text += f"Average content tokens per chapter: {avg_content_tokens_per_chapter:,}\n"
+ cost_text += f"Overhead per chapter: {overhead_tokens:,} tokens"
+ if glossary_tokens > 0:
+ cost_text += f" (system: {overhead_tokens - glossary_tokens:,}, glossary: {glossary_tokens:,})"
+ cost_text += f"\nTotal input tokens per chapter: {avg_total_tokens_per_chapter:,}\n"
+
+ if chapters_needing_chunking > 0:
+ cost_text += f"\nChapters requiring chunking: {chapters_needing_chunking} (will be skipped)\n"
+ cost_text += f"Processable chapters: {processable_chapters}\n"
+
+ cost_text += f"\nEstimated cost for {processable_chapters} chapters:\n"
+ cost_text += f"Regular processing: ${regular_cost:.2f}\n"
+ cost_text += f"Async processing: ${async_cost:.2f} (50% savings: ${regular_cost - async_cost:.2f})"
+
+ # Add note about token calculation
+ cost_text += f"\n\nNote: Costs include input (~{avg_total_tokens_per_chapter:,}) and "
+ cost_text += f"output (~{int(avg_content_tokens_per_chapter * compression_factor):,}) tokens per chapter."
+
+
+ self.cost_info_label.config(text=cost_text)
+
+ except Exception as e:
+ self.cost_info_label.config(
+ text=f"Error estimating cost: {str(e)}"
+ )
+ print(f"Cost estimation error: {traceback.format_exc()}")
+
+ def count_tokens(self, text, model):
+ """Count tokens in text (content only - system prompt and glossary are counted separately)"""
+ try:
+ import tiktoken
+
+ # Get base encoding for model
+ if model.startswith(('gpt-4', 'gpt-3')):
+ try:
+ encoding = tiktoken.encoding_for_model(model)
+ except KeyError:
+ encoding = tiktoken.get_encoding("cl100k_base")
+ elif model.startswith('claude'):
+ encoding = tiktoken.get_encoding("cl100k_base")
+ else:
+ encoding = tiktoken.get_encoding("cl100k_base")
+
+ # Just count the text tokens - don't include system/glossary here
+ # They are counted separately in _estimate_cost to avoid confusion
+ text_tokens = len(encoding.encode(text))
+
+ return text_tokens
+
+ except Exception as e:
+ # Fallback: estimate ~4 characters per token
+ return len(text) // 4
+
+ def _start_processing(self):
+ """Start async processing"""
+ model = self.gui.model_var.get()
+
+ if not self.processor.supports_async(model):
+ messagebox.showerror(
+ "Not Supported",
+ f"Model '{model}' does not support async processing.\n"
+ "Supported providers: Gemini, Anthropic, OpenAI, Mistral, Groq"
+ )
+ return
+
+ # Add special check for Gemini
+ if model.lower().startswith('gemini'):
+ response = messagebox.askyesno(
+ "Gemini Batch API",
+ "Note: Gemini's batch API may not be publicly available yet.\n"
+ "This feature is experimental for Gemini models.\n\n"
+ "Would you like to try anyway?"
+ )
+ if not response:
+ return
+
+ if not self.processor.supports_async(model):
+ messagebox.showerror(
+ "Not Supported",
+ f"Model '{model}' does not support async processing.\n"
+ "Supported providers: Gemini, Anthropic, OpenAI, Mistral, Groq"
+ )
+ return
+
+ if not hasattr(self.gui, 'file_path') or not self.gui.file_path:
+ messagebox.showwarning("No File", "Please select a file to translate first")
+ return
+
+ # Confirm start
+ if not messagebox.askyesno(
+ "Start Async Processing",
+ "Start async batch processing?\n\n"
+ "This will submit all chapters for processing at 50% discount.\n"
+ "Processing may take up to 24 hours."
+ ):
+ return
+
+ # Disable buttons during processing
+ self.start_button.config(state='disabled')
+
+ # Start processing in background thread
+ self.processing_thread = threading.Thread(
+ target=self._async_processing_worker,
+ daemon=True
+ )
+ self.processing_thread.start()
+
+ def _async_processing_worker(self):
+ """Worker thread for async processing"""
+ try:
+ self._log("Starting async processing preparation...")
+
+ # Get all settings from GUI
+ file_path = self.gui.file_path
+ model = self.gui.model_var.get()
+ api_key = self.gui.api_key_entry.get().strip()
+
+ if not api_key:
+ self._show_error("API key is required")
+ return
+
+ # Prepare environment variables like the main translation
+ env_vars = self._prepare_environment_variables()
+
+ # Extract chapters
+ self._log("Extracting chapters from file...")
+ chapters, chapter_mapping = self._extract_chapters_for_async(file_path, env_vars) # CHANGED: Now unpacking both values
+
+ if not chapters:
+ self._show_error("No chapters found in file")
+ return
+
+ self._log(f"Found {len(chapters)} chapters to process")
+
+ # Check for chapters that need chunking
+ chapters_to_process = []
+ skipped_count = 0
+
+ for chapter in chapters:
+ if chapter.get('needs_chunking', False):
+ skipped_count += 1
+ self._log(f"Skipping chapter {chapter['number']} - requires chunking")
+ else:
+ chapters_to_process.append(chapter)
+
+ if skipped_count > 0:
+ self._log(f"⚠️ Skipped {skipped_count} chapters that require chunking")
+
+ if not chapters_to_process:
+ self._show_error("All chapters require chunking. Async APIs don't support chunked chapters.")
+ return
+
+ # Prepare batch request
+ self._log("Preparing batch request...")
+ batch_data = self.processor.prepare_batch_request(chapters_to_process, model)
+
+ # Submit batch
+ self._log("Submitting batch to API...")
+ job = self._submit_batch_sync(batch_data, model, api_key)
+
+ # Save job with chapter mapping in metadata
+ job.metadata = job.metadata or {}
+ job.metadata['chapter_mapping'] = chapter_mapping # ADDED: Store mapping for later use
+
+ # Save job
+ self.processor.jobs[job.job_id] = job
+ self.processor._save_jobs()
+
+ # Update UI
+ self.dialog.after(0, self._refresh_jobs_list)
+
+ self._log(f"✅ Batch submitted successfully! Job ID: {job.job_id}")
+
+ # Show success message
+ self._show_info(
+ "Batch Submitted",
+ f"Successfully submitted {len(chapters_to_process)} chapters for async processing.\n\n"
+ f"Job ID: {job.job_id}\n\n"
+ "You can close this dialog and check back later for results.\n\n"
+ "Tip: Use the 'Estimate Cost Only' button to get accurate cost estimates before submitting."
+ )
+
+ # Start polling if requested
+ if self.wait_for_completion_var.get():
+ self._start_polling(job.job_id)
+
+ except Exception as e:
+ self._log(f"❌ Error: {str(e)}")
+ print(f"Async processing error: {traceback.format_exc()}")
+ self._show_error(f"Failed to start async processing: {str(e)}")
+ finally:
+ # Re-enable button
+ self.dialog.after(0, lambda: self.start_button.config(state='normal'))
+
+ def _prepare_environment_variables(self):
+ """Prepare environment variables from GUI settings"""
+ env_vars = {}
+
+ # Core settings
+ env_vars['MODEL'] = self.gui.model_var.get()
+ env_vars['API_KEY'] = self.gui.api_key_entry.get().strip()
+ env_vars['OPENAI_API_KEY'] = env_vars['API_KEY']
+ env_vars['OPENAI_OR_Gemini_API_KEY'] = env_vars['API_KEY']
+ env_vars['GEMINI_API_KEY'] = env_vars['API_KEY']
+ env_vars['PROFILE_NAME'] = self.gui.lang_var.get().lower()
+ env_vars['CONTEXTUAL'] = '1' if self.gui.contextual_var.get() else '0'
+ env_vars['MAX_OUTPUT_TOKENS'] = str(self.gui.max_output_tokens)
+ env_vars['SYSTEM_PROMPT'] = self.gui.prompt_text.get("1.0", "end").strip()
+ env_vars['TRANSLATION_TEMPERATURE'] = str(self.gui.trans_temp.get())
+ env_vars['TRANSLATION_HISTORY_LIMIT'] = str(self.gui.trans_history.get())
+
+ # API settings
+ env_vars['SEND_INTERVAL_SECONDS'] = str(self.gui.delay_entry.get())
+ env_vars['TOKEN_LIMIT'] = self.gui.token_limit_entry.get() if hasattr(self.gui, 'token_limit_entry') else '200000'
+
+ # Book title translation
+ env_vars['TRANSLATE_BOOK_TITLE'] = "1" if self.gui.translate_book_title_var.get() else "0"
+ env_vars['BOOK_TITLE_PROMPT'] = self.gui.book_title_prompt if hasattr(self.gui, 'book_title_prompt') else ''
+ env_vars['BOOK_TITLE_SYSTEM_PROMPT'] = self.gui.config.get('book_title_system_prompt',
+ "You are a translator. Respond with only the translated text, nothing else. Do not add any explanation or additional content.")
+
+ # Processing options
+ env_vars['CHAPTER_RANGE'] = self.gui.chapter_range_entry.get().strip() if hasattr(self.gui, 'chapter_range_entry') else ''
+ env_vars['REMOVE_AI_ARTIFACTS'] = "1" if self.gui.REMOVE_AI_ARTIFACTS_var.get() else "0"
+ env_vars['BATCH_TRANSLATION'] = "1" if self.gui.batch_translation_var.get() else "0"
+ env_vars['BATCH_SIZE'] = self.gui.batch_size_var.get()
+ env_vars['CONSERVATIVE_BATCHING'] = "1" if self.gui.conservative_batching_var.get() else "0"
+
+ # Anti-duplicate parameters
+ env_vars['ENABLE_ANTI_DUPLICATE'] = '1' if hasattr(self.gui, 'enable_anti_duplicate_var') and self.gui.enable_anti_duplicate_var.get() else '0'
+ env_vars['TOP_P'] = str(self.gui.top_p_var.get()) if hasattr(self.gui, 'top_p_var') else '1.0'
+ env_vars['TOP_K'] = str(self.gui.top_k_var.get()) if hasattr(self.gui, 'top_k_var') else '0'
+ env_vars['FREQUENCY_PENALTY'] = str(self.gui.frequency_penalty_var.get()) if hasattr(self.gui, 'frequency_penalty_var') else '0.0'
+ env_vars['PRESENCE_PENALTY'] = str(self.gui.presence_penalty_var.get()) if hasattr(self.gui, 'presence_penalty_var') else '0.0'
+ env_vars['REPETITION_PENALTY'] = str(self.gui.repetition_penalty_var.get()) if hasattr(self.gui, 'repetition_penalty_var') else '1.0'
+ env_vars['CANDIDATE_COUNT'] = str(self.gui.candidate_count_var.get()) if hasattr(self.gui, 'candidate_count_var') else '1'
+ env_vars['CUSTOM_STOP_SEQUENCES'] = self.gui.custom_stop_sequences_var.get() if hasattr(self.gui, 'custom_stop_sequences_var') else ''
+ env_vars['LOGIT_BIAS_ENABLED'] = '1' if hasattr(self.gui, 'logit_bias_enabled_var') and self.gui.logit_bias_enabled_var.get() else '0'
+ env_vars['LOGIT_BIAS_STRENGTH'] = str(self.gui.logit_bias_strength_var.get()) if hasattr(self.gui, 'logit_bias_strength_var') else '-0.5'
+ env_vars['BIAS_COMMON_WORDS'] = '1' if hasattr(self.gui, 'bias_common_words_var') and self.gui.bias_common_words_var.get() else '0'
+ env_vars['BIAS_REPETITIVE_PHRASES'] = '1' if hasattr(self.gui, 'bias_repetitive_phrases_var') and self.gui.bias_repetitive_phrases_var.get() else '0'
+
+ # Glossary settings
+ env_vars['MANUAL_GLOSSARY'] = self.gui.manual_glossary_path if hasattr(self.gui, 'manual_glossary_path') and self.gui.manual_glossary_path else ''
+ env_vars['DISABLE_AUTO_GLOSSARY'] = "0" if self.gui.enable_auto_glossary_var.get() else "1"
+ env_vars['DISABLE_GLOSSARY_TRANSLATION'] = "0" if self.gui.enable_auto_glossary_var.get() else "1"
+ env_vars['APPEND_GLOSSARY'] = "1" if self.gui.append_glossary_var.get() else "0"
+ env_vars['APPEND_GLOSSARY_PROMPT'] = self.gui.append_glossary_prompt if hasattr(self.gui, 'append_glossary_prompt') else ''
+ env_vars['GLOSSARY_MIN_FREQUENCY'] = self.gui.glossary_min_frequency_var.get()
+ env_vars['GLOSSARY_MAX_NAMES'] = self.gui.glossary_max_names_var.get()
+ env_vars['GLOSSARY_MAX_TITLES'] = self.gui.glossary_max_titles_var.get()
+ env_vars['GLOSSARY_BATCH_SIZE'] = self.gui.glossary_batch_size_var.get()
+ env_vars['GLOSSARY_DUPLICATE_KEY_MODE'] = self.gui.config.get('glossary_duplicate_key_mode', 'auto')
+ env_vars['GLOSSARY_DUPLICATE_CUSTOM_FIELD'] = self.gui.config.get('glossary_duplicate_custom_field', '')
+
+ # History and summary settings
+ env_vars['TRANSLATION_HISTORY_ROLLING'] = "1" if self.gui.translation_history_rolling_var.get() else "0"
+ env_vars['USE_ROLLING_SUMMARY'] = "1" if self.gui.config.get('use_rolling_summary') else "0"
+ env_vars['SUMMARY_ROLE'] = self.gui.config.get('summary_role', 'user')
+ env_vars['ROLLING_SUMMARY_EXCHANGES'] = self.gui.rolling_summary_exchanges_var.get()
+ env_vars['ROLLING_SUMMARY_MODE'] = self.gui.rolling_summary_mode_var.get()
+ env_vars['ROLLING_SUMMARY_SYSTEM_PROMPT'] = self.gui.rolling_summary_system_prompt if hasattr(self.gui, 'rolling_summary_system_prompt') else ''
+ env_vars['ROLLING_SUMMARY_USER_PROMPT'] = self.gui.rolling_summary_user_prompt if hasattr(self.gui, 'rolling_summary_user_prompt') else ''
+ env_vars['ROLLING_SUMMARY_MAX_ENTRIES'] = self.gui.rolling_summary_max_entries_var.get() if hasattr(self.gui, 'rolling_summary_max_entries_var') else '10'
+
+ # Retry and error handling settings
+ env_vars['EMERGENCY_PARAGRAPH_RESTORE'] = "1" if self.gui.emergency_restore_var.get() else "0"
+ env_vars['RETRY_TRUNCATED'] = "1" if self.gui.retry_truncated_var.get() else "0"
+ env_vars['MAX_RETRY_TOKENS'] = self.gui.max_retry_tokens_var.get()
+ env_vars['RETRY_DUPLICATE_BODIES'] = "1" if self.gui.retry_duplicate_var.get() else "0"
+ env_vars['RETRY_TIMEOUT'] = "1" if self.gui.retry_timeout_var.get() else "0"
+ env_vars['CHUNK_TIMEOUT'] = self.gui.chunk_timeout_var.get()
+
+ # Image processing
+ env_vars['ENABLE_IMAGE_TRANSLATION'] = "1" if self.gui.enable_image_translation_var.get() else "0"
+ env_vars['PROCESS_WEBNOVEL_IMAGES'] = "1" if self.gui.process_webnovel_images_var.get() else "0"
+ env_vars['WEBNOVEL_MIN_HEIGHT'] = self.gui.webnovel_min_height_var.get()
+ env_vars['MAX_IMAGES_PER_CHAPTER'] = self.gui.max_images_per_chapter_var.get()
+ env_vars['IMAGE_API_DELAY'] = '1.0'
+ env_vars['SAVE_IMAGE_TRANSLATIONS'] = '1'
+ env_vars['IMAGE_CHUNK_HEIGHT'] = self.gui.image_chunk_height_var.get()
+ env_vars['HIDE_IMAGE_TRANSLATION_LABEL'] = "1" if self.gui.hide_image_translation_label_var.get() else "0"
+
+ # Advanced settings
+ env_vars['REINFORCEMENT_FREQUENCY'] = self.gui.reinforcement_freq_var.get()
+ env_vars['RESET_FAILED_CHAPTERS'] = "1" if self.gui.reset_failed_chapters_var.get() else "0"
+ env_vars['DUPLICATE_LOOKBACK_CHAPTERS'] = self.gui.duplicate_lookback_var.get()
+ env_vars['DUPLICATE_DETECTION_MODE'] = self.gui.duplicate_detection_mode_var.get()
+ env_vars['CHAPTER_NUMBER_OFFSET'] = str(self.gui.chapter_number_offset_var.get())
+ env_vars['COMPRESSION_FACTOR'] = self.gui.compression_factor_var.get()
+ extraction_mode = self.gui.extraction_mode_var.get() if hasattr(self.gui, 'extraction_mode_var') else 'smart'
+ env_vars['COMPREHENSIVE_EXTRACTION'] = "1" if extraction_mode in ['comprehensive', 'full'] else "0"
+ env_vars['EXTRACTION_MODE'] = extraction_mode
+ env_vars['DISABLE_ZERO_DETECTION'] = "1" if self.gui.disable_zero_detection_var.get() else "0"
+ env_vars['USE_HEADER_AS_OUTPUT'] = "1" if self.gui.use_header_as_output_var.get() else "0"
+ env_vars['ENABLE_DECIMAL_CHAPTERS'] = "1" if self.gui.enable_decimal_chapters_var.get() else "0"
+ env_vars['ENABLE_WATERMARK_REMOVAL'] = "1" if self.gui.enable_watermark_removal_var.get() else "0"
+ env_vars['ADVANCED_WATERMARK_REMOVAL'] = "1" if self.gui.advanced_watermark_removal_var.get() else "0"
+ env_vars['SAVE_CLEANED_IMAGES'] = "1" if self.gui.save_cleaned_images_var.get() else "0"
+
+ # EPUB specific settings
+ env_vars['DISABLE_EPUB_GALLERY'] = "1" if self.gui.disable_epub_gallery_var.get() else "0"
+ env_vars['FORCE_NCX_ONLY'] = '1' if self.gui.force_ncx_only_var.get() else '0'
+
+ # Special handling for Gemini safety filters
+ env_vars['DISABLE_GEMINI_SAFETY'] = str(self.gui.config.get('disable_gemini_safety', False)).lower()
+
+ # AI Hunter settings (if enabled)
+ if 'ai_hunter_config' in self.gui.config:
+ env_vars['AI_HUNTER_CONFIG'] = json.dumps(self.gui.config['ai_hunter_config'])
+
+ # Output settings
+ env_vars['EPUB_OUTPUT_DIR'] = os.getcwd()
+ output_path = self.gui.output_entry.get().strip() if hasattr(self.gui, 'output_entry') else ''
+ if output_path:
+ env_vars['OUTPUT_DIR'] = output_path
+
+ # File path (needed by some modules)
+ env_vars['EPUB_PATH'] = self.gui.file_path
+
+ return env_vars
+
+ def _extract_chapters_for_async(self, file_path, env_vars):
+ """Extract chapters and prepare them for async processing"""
+ chapters = []
+ original_basename = None
+ chapter_mapping = {} # Map custom_id to chapter info
+
+ try:
+ if file_path.lower().endswith('.epub'):
+ # Use direct ZIP reading to avoid ebooklib's manifest validation
+ import zipfile
+ from bs4 import BeautifulSoup
+
+ raw_chapters = []
+
+ try:
+ with zipfile.ZipFile(file_path, 'r') as zf:
+ # Get all HTML/XHTML files
+ html_files = [f for f in zf.namelist() if f.endswith(('.html', '.xhtml', '.htm')) and not f.startswith('__MACOSX')]
+ html_files.sort() # Sort to maintain order
+
+ for idx, html_file in enumerate(html_files):
+ try:
+ content = zf.read(html_file)
+ soup = BeautifulSoup(content, 'html.parser')
+
+ # Remove all image tags
+ for img in soup.find_all('img'):
+ img.decompose()
+
+ # Remove all link tags that might reference CSS or other files
+ for link in soup.find_all('link'):
+ link.decompose()
+
+ chapter_text = soup.get_text(separator='\n').strip()
+
+ if len(chapter_text) > 500: # Minimum chapter length
+ chapter_num = idx + 1
+
+ # Try to extract chapter number from content
+ for element in soup.find_all(['h1', 'h2', 'h3', 'title']):
+ text = element.get_text().strip()
+ match = re.search(r'chapter\s*(\d+)', text, re.IGNORECASE)
+ if match:
+ chapter_num = int(match.group(1))
+ break
+
+ raw_chapters.append((chapter_num, chapter_text, html_file))
+
+ except Exception as e:
+ print(f"Error reading {html_file}: {e}")
+ continue
+
+ except Exception as e:
+ print(f"Failed to read EPUB as ZIP: {e}")
+ raise ValueError(f"Cannot read EPUB file: {str(e)}")
+
+ elif file_path.lower().endswith('.txt'):
+ # Import TXT processing
+ from txt_processor import TextFileProcessor
+
+ processor = TextFileProcessor(file_path, '')
+ txt_chapters = processor.extract_chapters()
+ raw_chapters = [(i+1, text, f"section_{i+1:04d}.txt") for i, text in enumerate(txt_chapters)]
+
+ else:
+ raise ValueError(f"Unsupported file type: {file_path}")
+
+ if not raw_chapters:
+ raise ValueError("No valid chapters found in file")
+
+ # Process each chapter to prepare for API
+ for idx, (chapter_num, content, original_filename) in enumerate(raw_chapters):
+ # Count tokens
+ token_count = self.count_tokens(content, env_vars['MODEL'])
+
+ # Check if needs chunking
+ token_limit = int(env_vars.get('TOKEN_LIMIT', '200000'))
+ needs_chunking = token_count > token_limit * 0.8 # 80% threshold
+
+ # Prepare messages format
+ messages = self._prepare_chapter_messages(content, env_vars)
+
+ custom_id = f"chapter_{chapter_num}"
+
+ chapter_data = {
+ 'id': custom_id,
+ 'number': chapter_num,
+ 'content': content,
+ 'messages': messages,
+ 'temperature': float(env_vars.get('TRANSLATION_TEMPERATURE', '0.3')),
+ 'max_tokens': int(env_vars['MAX_OUTPUT_TOKENS']),
+ 'needs_chunking': needs_chunking,
+ 'token_count': token_count,
+ 'original_basename': original_filename # Use original_filename instead of undefined original_basename
+ }
+
+ chapters.append(chapter_data)
+
+ # Store mapping
+ chapter_mapping[custom_id] = {
+ 'original_filename': original_filename,
+ 'chapter_num': chapter_num
+ }
+
+ except Exception as e:
+ print(f"Failed to extract chapters: {e}")
+ raise
+
+ # Return both chapters and mapping
+ return chapters, chapter_mapping
+
+ def _delete_selected_job(self):
+ """Delete selected job from the list"""
+ if not self.selected_job_id:
+ messagebox.showwarning("No Selection", "Please select a job to delete")
+ return
+
+ # Get job details for confirmation
+ job = self.processor.jobs.get(self.selected_job_id)
+ if not job:
+ messagebox.showerror("Error", "Selected job not found")
+ return
+
+ # Confirm deletion
+ response = messagebox.askyesno(
+ "Confirm Delete",
+ f"Are you sure you want to delete this job?\n\n"
+ f"Job ID: {job.job_id}\n"
+ f"Status: {job.status.value}\n"
+ f"Created: {job.created_at.strftime('%Y-%m-%d %H:%M:%S')}\n\n"
+ "Note: This only removes the job from your local list.\n"
+ "The job may still be running on the server."
+ )
+
+ if response:
+ # Remove from jobs dictionary
+ del self.processor.jobs[self.selected_job_id]
+
+ # Save updated jobs
+ self.processor._save_jobs()
+
+ # Clear selection
+ self.selected_job_id = None
+
+ # Refresh the display
+ self._refresh_jobs_list()
+
+ messagebox.showinfo("Job Deleted", "Job removed from local list.")
+
+ def _clear_completed_jobs(self):
+ """Clear all completed/failed/cancelled jobs"""
+ # Get list of jobs to remove
+ jobs_to_remove = []
+ for job_id, job in self.processor.jobs.items():
+ if job.status in [AsyncAPIStatus.COMPLETED, AsyncAPIStatus.FAILED,
+ AsyncAPIStatus.CANCELLED, AsyncAPIStatus.EXPIRED]:
+ jobs_to_remove.append(job_id)
+
+ if not jobs_to_remove:
+ messagebox.showinfo("No Jobs to Clear", "No completed/failed/cancelled jobs to clear.")
+ return
+
+ # Confirm
+ response = messagebox.askyesno(
+ "Clear Completed Jobs",
+ f"Remove {len(jobs_to_remove)} completed/failed/cancelled jobs from the list?\n\n"
+ "This will not affect any running jobs."
+ )
+
+ if response:
+ # Remove jobs
+ for job_id in jobs_to_remove:
+ del self.processor.jobs[job_id]
+
+ # Save
+ self.processor._save_jobs()
+
+ # Refresh
+ self._refresh_jobs_list()
+
+ messagebox.showinfo("Jobs Cleared", f"Removed {len(jobs_to_remove)} jobs from the list.")
+
+ def _prepare_chapter_messages(self, content, env_vars):
+ """Prepare messages array for a chapter"""
+ messages = []
+
+ # System prompt
+ system_prompt = env_vars.get('SYSTEM_PROMPT', '')
+
+ # DEBUG: Log what we're sending
+ logger.info(f"Model: {env_vars.get('MODEL')}")
+ logger.info(f"System prompt length: {len(system_prompt)}")
+ logger.info(f"Content length: {len(content)}")
+
+ # Log the system prompt (first 200 chars)
+ logger.info(f"Using system prompt: {system_prompt[:200]}...")
+
+ # Add glossary if enabled
+ if (env_vars.get('MANUAL_GLOSSARY') and
+ env_vars.get('APPEND_GLOSSARY') == '1' and
+ env_vars.get('DISABLE_GLOSSARY_TRANSLATION') != '1'):
+ try:
+ glossary_path = env_vars['MANUAL_GLOSSARY']
+ with open(glossary_path, 'r', encoding='utf-8') as f:
+ glossary_data = json.load(f)
+
+ # TRUE BRUTE FORCE: Just dump the entire JSON
+ glossary_text = json.dumps(glossary_data, ensure_ascii=False, indent=2)
+
+ # Use the append prompt format if provided
+ append_prompt = env_vars.get('APPEND_GLOSSARY_PROMPT', '')
+ if append_prompt:
+ # Replace placeholder with actual glossary
+ if '{glossary}' in append_prompt:
+ glossary_section = append_prompt.replace('{glossary}', glossary_text)
+ else:
+ glossary_section = f"{append_prompt}\n{glossary_text}"
+ system_prompt = f"{system_prompt}\n\n{glossary_section}"
+ else:
+ # Default format
+ system_prompt = f"{system_prompt}\n\nGlossary:\n{glossary_text}"
+
+ logger.info(f"Glossary appended to system prompt ({len(glossary_text)} chars)")
+
+ # Log preview for debugging
+ if len(glossary_text) > 200:
+ logger.info(f"Glossary preview: {glossary_text[:200]}...")
+ else:
+ logger.info(f"Glossary: {glossary_text}")
+
+ except FileNotFoundError:
+ print(f"Glossary file not found: {env_vars.get('MANUAL_GLOSSARY')}")
+ except json.JSONDecodeError:
+ print(f"Invalid JSON in glossary file")
+ except Exception as e:
+ print(f"Failed to load glossary: {e}")
+ else:
+ # Log why glossary wasn't added
+ if not env_vars.get('MANUAL_GLOSSARY'):
+ logger.info("No glossary path specified")
+ elif env_vars.get('APPEND_GLOSSARY') != '1':
+ logger.info("Glossary append is disabled")
+ elif env_vars.get('DISABLE_GLOSSARY_TRANSLATION') == '1':
+ logger.info("Glossary translation is disabled")
+
+ messages.append({
+ 'role': 'system',
+ 'content': system_prompt
+ })
+
+ # Add context if enabled
+ if env_vars.get('CONTEXTUAL') == '1':
+ # This would need to load context from history
+ # For async, we might need to pre-generate context
+ logger.info("Note: Contextual mode enabled but not implemented for async yet")
+
+ # User message with chapter content
+ messages.append({
+ 'role': 'user',
+ 'content': content
+ })
+
+ return messages
+
+ def _submit_batch_sync(self, batch_data, model, api_key):
+ """Submit batch synchronously (wrapper for async method)"""
+ provider = self.processor.get_provider_from_model(model)
+
+ if provider == 'openai':
+ return self.processor._submit_openai_batch_sync(batch_data, model, api_key)
+ elif provider == 'anthropic':
+ return self.processor._submit_anthropic_batch_sync(batch_data, model, api_key)
+ elif provider == 'gemini':
+ return self._submit_gemini_batch_sync(batch_data, model, api_key)
+ elif provider == 'mistral':
+ return self._submit_mistral_batch_sync(batch_data, model, api_key)
+ elif provider == 'groq':
+ return self._submit_groq_batch_sync(batch_data, model, api_key)
+ else:
+ raise ValueError(f"Unsupported provider: {provider}")
+
+ def _submit_gemini_batch_sync(self, batch_data, model, api_key):
+ """Submit Gemini batch using the official Batch Mode API"""
+ try:
+ # Use the new Google Gen AI SDK
+ from google import genai
+ from google.genai import types
+
+ # Configure client with API key
+ client = genai.Client(api_key=api_key)
+
+ # Log for debugging
+ logger.info(f"Submitting Gemini batch with model: {model}")
+ logger.info(f"Number of requests: {len(batch_data['requests'])}")
+
+ # Create JSONL file for batch requests
+ import tempfile
+
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False, encoding='utf-8') as f:
+ for request in batch_data['requests']:
+ # Format for Gemini batch API
+ batch_line = {
+ "key": request['custom_id'],
+ "request": {
+ "contents": request['generateContentRequest']['contents'],
+ "generation_config": request['generateContentRequest'].get('generationConfig', {})
+ }
+ }
+
+ # Add safety settings if present
+ if 'safetySettings' in request['generateContentRequest']:
+ batch_line['request']['safety_settings'] = request['generateContentRequest']['safetySettings']
+
+ f.write(json.dumps(batch_line) + '\n')
+
+ batch_file_path = f.name
+
+ # Upload the batch file with explicit mime type
+ logger.info("Uploading batch file...")
+
+ # Use the upload config to specify mime type
+ upload_config = types.UploadFileConfig(
+ mime_type='application/jsonl', # Explicit JSONL mime type
+ display_name=f"batch_requests_{datetime.now().strftime('%Y%m%d_%H%M%S')}.jsonl"
+ )
+
+ uploaded_file = client.files.upload(
+ file=batch_file_path,
+ config=upload_config
+ )
+
+ logger.info(f"File uploaded: {uploaded_file.name}")
+
+ # Create batch job
+ batch_job = client.batches.create(
+ model=model,
+ src=uploaded_file.name,
+ config={
+ 'display_name': f"glossarion_batch_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+ }
+ )
+
+ logger.info(f"Gemini batch job created: {batch_job.name}")
+
+ # Clean up temp file
+ os.unlink(batch_file_path)
+
+ # Calculate cost estimate
+ total_tokens = sum(r.get('token_count', 15000) for r in batch_data['requests'])
+ async_cost, _ = self.processor.estimate_cost(
+ len(batch_data['requests']),
+ total_tokens // len(batch_data['requests']),
+ model
+ )
+
+ # Create job info
+ job = AsyncJobInfo(
+ job_id=batch_job.name,
+ provider='gemini',
+ model=model,
+ status=AsyncAPIStatus.PENDING,
+ created_at=datetime.now(),
+ updated_at=datetime.now(),
+ total_requests=len(batch_data['requests']),
+ cost_estimate=0.0, # No estimate initially
+ metadata={
+ 'batch_info': {
+ 'name': batch_job.name,
+ 'state': batch_job.state.name if hasattr(batch_job, 'state') else 'PENDING',
+ 'src_file': uploaded_file.name
+ },
+ 'source_file': self.gui.file_path # Add this to track which file this job is for
+ }
+ )
+
+ return job
+
+ except ImportError:
+ print("Google Gen AI SDK not installed. Run: pip install google-genai")
+ raise Exception("Google Gen AI SDK not installed. Please run: pip install google-genai")
+ except Exception as e:
+ print(f"Gemini batch submission failed: {e}")
+ print(f"Full error: {traceback.format_exc()}")
+ raise
+
+ def _submit_mistral_batch_sync(self, batch_data, model, api_key):
+ """Submit Mistral batch (synchronous version)"""
+ try:
+ headers = {
+ 'Authorization': f'Bearer {api_key}',
+ 'Content-Type': 'application/json'
+ }
+
+ response = requests.post(
+ 'https://api.mistral.ai/v1/batch/jobs',
+ headers=headers,
+ json=batch_data
+ )
+
+ if response.status_code != 200:
+ raise Exception(f"Batch creation failed: {response.text}")
+
+ batch_info = response.json()
+
+ # Calculate cost estimate
+ total_tokens = sum(r.get('token_count', 15000) for r in batch_data['requests'])
+ async_cost, _ = self.processor.estimate_cost(
+ len(batch_data['requests']),
+ total_tokens // len(batch_data['requests']),
+ model
+ )
+
+ job = AsyncJobInfo(
+ job_id=batch_info['id'],
+ provider='mistral',
+ model=model,
+ status=AsyncAPIStatus.PENDING,
+ created_at=datetime.now(),
+ updated_at=datetime.now(),
+ total_requests=len(batch_data['requests']),
+ cost_estimate=async_cost,
+ metadata={'batch_info': batch_info}
+ )
+
+ return job
+
+ except Exception as e:
+ print(f"Mistral batch submission failed: {e}")
+ raise
+
+ def _submit_groq_batch_sync(self, batch_data, model, api_key):
+ """Submit Groq batch (synchronous version)"""
+ # Groq uses OpenAI-compatible format
+ return self.processor._submit_openai_batch_sync(batch_data, model, api_key)
+
+ def _start_polling(self, job_id):
+ """Start polling for job completion with progress updates"""
+ def poll():
+ try:
+ job = self.processor.check_job_status(job_id)
+ self._refresh_jobs_list()
+
+ # Update progress message
+ if job.total_requests > 0:
+ progress_pct = int((job.completed_requests / job.total_requests) * 100)
+ self._log(f"Progress: {progress_pct}% ({job.completed_requests}/{job.total_requests} chapters)")
+
+ if job.status == AsyncAPIStatus.COMPLETED:
+ self._log(f"✅ Job {job_id} completed!")
+ self._handle_completed_job(job_id)
+ elif job.status in [AsyncAPIStatus.FAILED, AsyncAPIStatus.CANCELLED]:
+ self._log(f"❌ Job {job_id} {job.status.value}")
+ else:
+ # Continue polling with progress update
+ poll_interval = self.poll_interval_var.get() * 1000
+ self.dialog.after(poll_interval, poll)
+
+ except Exception as e:
+ self._log(f"❌ Polling error: {e}")
+
+ # Start polling
+ poll()
+
+ def _handle_completed_job(self, job_id):
+ """Handle a completed job - retrieve results and save"""
+ try:
+ # Retrieve results
+ results = self.processor.retrieve_results(job_id)
+
+ if not results:
+ self._log("❌ No results retrieved from completed job")
+ return
+
+ # Get output directory - same name as input file, in exe location
+ if getattr(sys, 'frozen', False):
+ # Running as compiled exe - use exe directory
+ app_dir = os.path.dirname(sys.executable)
+ else:
+ # Running as script - use script directory
+ app_dir = os.path.dirname(os.path.abspath(__file__))
+
+ base_name = os.path.splitext(os.path.basename(self.gui.file_path))[0]
+ output_dir = os.path.join(app_dir, base_name)
+
+ # Handle existing directory
+ if os.path.exists(output_dir):
+ response = messagebox.askyesnocancel(
+ "Directory Exists",
+ f"The output directory already exists:\n{output_dir}\n\n"
+ "Yes = Overwrite\n"
+ "No = Create new with number\n"
+ "Cancel = Cancel operation"
+ )
+
+ if response is None:
+ return
+ elif response is False:
+ counter = 1
+ while os.path.exists(f"{output_dir}_{counter}"):
+ counter += 1
+ output_dir = f"{output_dir}_{counter}"
+
+ os.makedirs(output_dir, exist_ok=True)
+
+ # Extract ALL resources from EPUB (CSS, fonts, images)
+ self._log("📦 Extracting EPUB resources...")
+ import zipfile
+
+ with zipfile.ZipFile(self.gui.file_path, 'r') as zf:
+ # Create resource directories
+ for res_type in ['css', 'fonts', 'images']:
+ os.makedirs(os.path.join(output_dir, res_type), exist_ok=True)
+
+ # Extract all resources
+ for file_path in zf.namelist():
+ if file_path.endswith('/'):
+ continue
+
+ file_lower = file_path.lower()
+ file_name = os.path.basename(file_path)
+
+ # Skip empty filenames
+ if not file_name:
+ continue
+
+ # Determine resource type and extract
+ if file_lower.endswith('.css'):
+ zf.extract(file_path, os.path.join(output_dir, 'css'))
+ elif file_lower.endswith(('.ttf', '.otf', '.woff', '.woff2')):
+ zf.extract(file_path, os.path.join(output_dir, 'fonts'))
+ elif file_lower.endswith(('.jpg', '.jpeg', '.png', '.gif', '.svg', '.webp')):
+ zf.extract(file_path, os.path.join(output_dir, 'images'))
+
+ # Extract chapter info and metadata from source EPUB
+ self._log("📋 Extracting metadata from source EPUB...")
+
+ import ebooklib
+ from ebooklib import epub
+ from bs4 import BeautifulSoup
+ from TransateKRtoEN import get_content_hash, should_retain_source_extension
+
+ # Extract metadata
+ metadata = {}
+ book = epub.read_epub(self.gui.file_path)
+
+ # Get book metadata
+ if book.get_metadata('DC', 'title'):
+ metadata['title'] = book.get_metadata('DC', 'title')[0][0]
+ if book.get_metadata('DC', 'creator'):
+ metadata['creator'] = book.get_metadata('DC', 'creator')[0][0]
+ if book.get_metadata('DC', 'language'):
+ metadata['language'] = book.get_metadata('DC', 'language')[0][0]
+
+ # Save metadata.json
+ metadata_path = os.path.join(output_dir, 'metadata.json')
+ with open(metadata_path, 'w', encoding='utf-8') as f:
+ json.dump(metadata, f, ensure_ascii=False, indent=2)
+
+ # Map chapter numbers to original info
+ chapter_map = {}
+ chapters_info = []
+ actual_chapter_num = 0
+
+ for item in book.get_items():
+ if item.get_type() == ebooklib.ITEM_DOCUMENT:
+ original_name = item.get_name()
+ original_basename = os.path.splitext(os.path.basename(original_name))[0]
+
+ soup = BeautifulSoup(item.get_content(), 'html.parser')
+ text = soup.get_text().strip()
+
+ if len(text) > 500: # Valid chapter
+ actual_chapter_num += 1
+
+ # Try to find chapter number in content
+ chapter_num = actual_chapter_num
+ for element in soup.find_all(['h1', 'h2', 'h3', 'title']):
+ element_text = element.get_text().strip()
+ match = re.search(r'chapter\s*(\d+)', element_text, re.IGNORECASE)
+ if match:
+ chapter_num = int(match.group(1))
+ break
+
+ # Calculate real content hash
+ content_hash = get_content_hash(text)
+
+ chapter_map[chapter_num] = {
+ 'original_basename': original_basename,
+ 'original_extension': os.path.splitext(original_name)[1],
+ 'content_hash': content_hash,
+ 'text_length': len(text),
+ 'has_images': bool(soup.find_all('img'))
+ }
+
+ chapters_info.append({
+ 'num': chapter_num,
+ 'title': element_text if 'element_text' in locals() else f"Chapter {chapter_num}",
+ 'original_filename': original_name,
+ 'original_basename': original_basename,
+ 'has_images': bool(soup.find_all('img')),
+ 'text_length': len(text),
+ 'content_hash': content_hash
+ })
+
+ # Save chapters_info.json
+ chapters_info_path = os.path.join(output_dir, 'chapters_info.json')
+ with open(chapters_info_path, 'w', encoding='utf-8') as f:
+ json.dump(chapters_info, f, ensure_ascii=False, indent=2)
+
+ # Create realistic progress tracking
+ progress_data = {
+ "version": "3.0",
+ "chapters": {},
+ "chapter_chunks": {},
+ "content_hashes": {},
+ "created": datetime.now().isoformat(),
+ "last_updated": datetime.now().isoformat(),
+ "total_chapters": len(results),
+ "completed_chapters": len(results),
+ "failed_chapters": 0,
+ "async_translated": True
+ }
+
+ # Sort results and save with proper filenames
+ sorted_results = sorted(results, key=lambda x: self._extract_chapter_number(x['custom_id']))
+
+ self._log("💾 Saving translated chapters...")
+ for result in sorted_results:
+ chapter_num = self._extract_chapter_number(result['custom_id'])
+
+ # Get chapter info
+ chapter_info = chapter_map.get(chapter_num, {})
+ original_basename = chapter_info.get('original_basename', f"{chapter_num:04d}")
+ content_hash = chapter_info.get('content_hash', hashlib.md5(f"chapter_{chapter_num}".encode()).hexdigest())
+
+ # Save file with correct name (only once!)
+ retain_ext = should_retain_source_extension()
+ # Preserve compound extensions like .htm.xhtml when retaining
+ orig_name = chapter_info.get('original_filename') or chapter_info.get('original_basename')
+ if retain_ext and orig_name:
+ # Compute full extension suffix beyond the first dot from the left of the basename
+ full = os.path.basename(orig_name)
+ bn, ext1 = os.path.splitext(full)
+ full_ext = ''
+ while ext1:
+ full_ext = ext1 + full_ext
+ bn, ext1 = os.path.splitext(bn)
+ # If no extension found, default to .html
+ suffix = full_ext if full_ext else '.html'
+ filename = f"{original_basename}{suffix}"
+ elif retain_ext:
+ filename = f"{original_basename}.html"
+ else:
+ filename = f"response_{original_basename}.html"
+ file_path = os.path.join(output_dir, filename)
+
+ with open(file_path, 'w', encoding='utf-8') as f:
+ f.write(result['content'])
+
+ # Add realistic progress entry
+ progress_data["chapters"][content_hash] = {
+ "status": "completed",
+ "output_file": filename,
+ "actual_num": chapter_num,
+ "chapter_num": chapter_num,
+ "content_hash": content_hash,
+ "original_basename": original_basename,
+ "started_at": datetime.now().isoformat(),
+ "completed_at": datetime.now().isoformat(),
+ "translation_time": 2.5, # Fake but realistic
+ "token_count": chapter_info.get('text_length', 5000) // 4, # Rough estimate
+ "model": self.gui.model_var.get(),
+ "from_async": True
+ }
+
+ # Add content hash tracking
+ progress_data["content_hashes"][content_hash] = {
+ "chapter_key": content_hash,
+ "chapter_num": chapter_num,
+ "status": "completed",
+ "index": chapter_num - 1
+ }
+
+ # Save realistic progress file
+ progress_file = os.path.join(output_dir, 'translation_progress.json')
+ with open(progress_file, 'w', encoding='utf-8') as f:
+ json.dump(progress_data, f, indent=2)
+
+ self._log(f"✅ Saved {len(sorted_results)} chapters to: {output_dir}")
+
+ messagebox.showinfo(
+ "Async Translation Complete",
+ f"Successfully saved {len(sorted_results)} translated chapters to:\n{output_dir}\n\n"
+ "Ready for EPUB conversion or further processing."
+ )
+
+ except Exception as e:
+ self._log(f"❌ Error handling completed job: {e}")
+ import traceback
+ self._log(traceback.format_exc())
+ messagebox.showerror("Error", f"Failed to process results: {str(e)}")
+
+ def _show_error_details(self, job):
+ """Show details from error file"""
+ if not job.metadata.get('error_file_id'):
+ return
+
+ try:
+ api_key = self.gui.api_key_entry.get().strip()
+ headers = {'Authorization': f'Bearer {api_key}'}
+
+ # Download error file
+ response = requests.get(
+ f'https://api.openai.com/v1/files/{job.metadata["error_file_id"]}/content',
+ headers=headers
+ )
+
+ if response.status_code == 200:
+ # Parse first few errors
+ errors = []
+ for i, line in enumerate(response.text.strip().split('\n')[:5]): # Show first 5 errors
+ if line:
+ try:
+ error_data = json.loads(line)
+ error_msg = error_data.get('error', {}).get('message', 'Unknown error')
+ errors.append(f"• {error_msg}")
+ except:
+ pass
+
+ error_text = '\n'.join(errors)
+ if len(response.text.strip().split('\n')) > 5:
+ newline = '\n'
+ error_text += f"\n\n... and {len(response.text.strip().split(newline)) - 5} more errors"
+
+ messagebox.showerror(
+ "Batch Processing Errors",
+ f"All requests failed with errors:\n\n{error_text}\n\n"
+ "Common causes:\n"
+ "• Invalid API key or insufficient permissions\n"
+ "• Model not available in your region\n"
+ "• Malformed request format"
+ )
+
+ except Exception as e:
+ print(f"Failed to retrieve error details: {e}")
+
+ def _extract_chapter_number(self, custom_id):
+ """Extract chapter number from custom ID"""
+ match = re.search(r'chapter[_-](\d+)', custom_id, re.IGNORECASE)
+ if match:
+ return int(match.group(1))
+ return 0
+
+ # Helper methods for thread-safe UI updates
+ def _log(self, message, level="info"):
+ """Thread-safe logging to GUI"""
+ # Log based on level
+ if level == "error":
+ print(f"❌ {message}") # This will show in GUI
+ elif level == "warning":
+ print(f"⚠️ {message}") # This will show in GUI
+ else:
+ logger.info(message) # This only goes to log file
+ # Also display info messages in GUI
+ if hasattr(self.gui, 'append_log'):
+ self.dialog.after(0, lambda: self.gui.append_log(message))
+
+ def _show_error(self, message):
+ """Thread-safe error dialog"""
+ self._log(f"Error: {message}", level="error")
+ self.dialog.after(0, lambda: messagebox.showerror("Error", message))
+
+ def _show_info(self, title, message):
+ """Thread-safe info dialog"""
+ self._log(f"{title}: {message}", level="info")
+ self.dialog.after(0, lambda: messagebox.showinfo(title, message))
+
+ def _show_warning(self, message):
+ """Thread-safe warning display"""
+ self._log(f"Warning: {message}", level="warning")
+
+
+def show_async_processing_dialog(parent, translator_gui):
+ """Show the async processing dialog
+
+ Args:
+ parent: Parent window
+ translator_gui: Reference to main TranslatorGUI instance
+ """
+ dialog = AsyncProcessingDialog(parent, translator_gui)
+ return dialog.dialog
+
+
+# Integration function for translator_gui.py
+def add_async_processing_button(translator_gui, parent_frame):
+ """Add async processing button to GUI
+
+ This function should be called from translator_gui.py to add the button
+
+ Args:
+ translator_gui: TranslatorGUI instance
+ parent_frame: Frame to add button to
+ """
+ # Create button with appropriate styling
+ async_button = ttk.Button(
+ parent_frame,
+ text="Async Processing (50% Off)",
+ command=lambda: show_async_processing_dialog(translator_gui.master, translator_gui),
+ style="primary.TButton"
+ )
+
+ # Place button appropriately
+ async_button.pack(side=tk.LEFT, padx=5)
+
+ # Store reference
+ translator_gui.async_button = async_button
+
+ return async_button
diff --git a/bubble_detector.py b/bubble_detector.py
new file mode 100644
index 0000000000000000000000000000000000000000..4502a2b1dc81480581d1305210844024f036d9a8
--- /dev/null
+++ b/bubble_detector.py
@@ -0,0 +1,1881 @@
+"""
+bubble_detector.py - Modified version that works in frozen PyInstaller executables
+Replace your bubble_detector.py with this version
+"""
+import os
+import sys
+import json
+import numpy as np
+import cv2
+from typing import List, Tuple, Optional, Dict, Any
+import logging
+import traceback
+import hashlib
+from pathlib import Path
+import threading
+import time
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Check if we're running in a frozen environment
+IS_FROZEN = getattr(sys, 'frozen', False)
+if IS_FROZEN:
+ # In frozen environment, set proper paths for ML libraries
+ MEIPASS = sys._MEIPASS
+ os.environ['TORCH_HOME'] = MEIPASS
+ os.environ['TRANSFORMERS_CACHE'] = os.path.join(MEIPASS, 'transformers')
+ os.environ['HF_HOME'] = os.path.join(MEIPASS, 'huggingface')
+ logger.info(f"Running in frozen environment: {MEIPASS}")
+
+# Modified import checks for frozen environment
+YOLO_AVAILABLE = False
+YOLO = None
+torch = None
+TORCH_AVAILABLE = False
+ONNX_AVAILABLE = False
+TRANSFORMERS_AVAILABLE = False
+RTDetrForObjectDetection = None
+RTDetrImageProcessor = None
+PIL_AVAILABLE = False
+
+# Try to import YOLO dependencies with better error handling
+if IS_FROZEN:
+ # In frozen environment, try harder to import
+ try:
+ # First try to import torch components individually
+ import torch
+ import torch.nn
+ import torch.cuda
+ TORCH_AVAILABLE = True
+ logger.info("✓ PyTorch loaded in frozen environment")
+ except Exception as e:
+ logger.warning(f"PyTorch not available in frozen environment: {e}")
+ TORCH_AVAILABLE = False
+ torch = None
+
+ # Try ultralytics after torch
+ if TORCH_AVAILABLE:
+ try:
+ from ultralytics import YOLO
+ YOLO_AVAILABLE = True
+ logger.info("✓ Ultralytics YOLO loaded in frozen environment")
+ except Exception as e:
+ logger.warning(f"Ultralytics not available in frozen environment: {e}")
+ YOLO_AVAILABLE = False
+
+ # Try transformers
+ try:
+ import transformers
+ # Try specific imports
+ try:
+ from transformers import RTDetrForObjectDetection, RTDetrImageProcessor
+ TRANSFORMERS_AVAILABLE = True
+ logger.info("✓ Transformers RT-DETR loaded in frozen environment")
+ except ImportError:
+ # Try alternative import
+ try:
+ from transformers import AutoModel, AutoImageProcessor
+ RTDetrForObjectDetection = AutoModel
+ RTDetrImageProcessor = AutoImageProcessor
+ TRANSFORMERS_AVAILABLE = True
+ logger.info("✓ Transformers loaded with AutoModel fallback")
+ except:
+ TRANSFORMERS_AVAILABLE = False
+ logger.warning("Transformers RT-DETR not available in frozen environment")
+ except Exception as e:
+ logger.warning(f"Transformers not available in frozen environment: {e}")
+ TRANSFORMERS_AVAILABLE = False
+else:
+ # Normal environment - original import logic
+ try:
+ from ultralytics import YOLO
+ YOLO_AVAILABLE = True
+ except:
+ YOLO_AVAILABLE = False
+ logger.warning("Ultralytics YOLO not available")
+
+ try:
+ import torch
+ # Test if cuda attribute exists
+ _ = torch.cuda
+ TORCH_AVAILABLE = True
+ except (ImportError, AttributeError):
+ TORCH_AVAILABLE = False
+ torch = None
+ logger.warning("PyTorch not available or incomplete")
+
+ try:
+ from transformers import RTDetrForObjectDetection, RTDetrImageProcessor
+ try:
+ from transformers import RTDetrV2ForObjectDetection
+ RTDetrForObjectDetection = RTDetrV2ForObjectDetection
+ except ImportError:
+ pass
+ TRANSFORMERS_AVAILABLE = True
+ except:
+ TRANSFORMERS_AVAILABLE = False
+ logger.info("Transformers not available for RT-DETR")
+
+# Configure ORT memory behavior before importing
+try:
+ os.environ.setdefault('ORT_DISABLE_MEMORY_ARENA', '1')
+except Exception:
+ pass
+# ONNX Runtime - works well in frozen environments
+try:
+ import onnxruntime as ort
+ ONNX_AVAILABLE = True
+ logger.info("✓ ONNX Runtime available")
+except ImportError:
+ ONNX_AVAILABLE = False
+ logger.warning("ONNX Runtime not available")
+
+# PIL
+try:
+ from PIL import Image
+ PIL_AVAILABLE = True
+except ImportError:
+ PIL_AVAILABLE = False
+ logger.info("PIL not available")
+
+
+class BubbleDetector:
+ """
+ Combined YOLOv8 and RT-DETR speech bubble detector for comics and manga.
+ Supports multiple model formats and provides configurable detection.
+ Backward compatible with existing code while adding RT-DETR support.
+ """
+
+ # Process-wide shared RT-DETR to avoid concurrent meta-device loads
+ _rtdetr_init_lock = threading.Lock()
+ _rtdetr_shared_model = None
+ _rtdetr_shared_processor = None
+ _rtdetr_loaded = False
+ _rtdetr_repo_id = 'ogkalu/comic-text-and-bubble-detector'
+
+ # Shared RT-DETR (ONNX) across process to avoid device/context storms
+ _rtdetr_onnx_init_lock = threading.Lock()
+ _rtdetr_onnx_shared_session = None
+ _rtdetr_onnx_loaded = False
+ _rtdetr_onnx_providers = None
+ _rtdetr_onnx_model_path = None
+ # Limit DML concurrent runs to avoid DXGI device hang. Adjustable via env DML_MAX_CONCURRENT
+ try:
+ _rtdetr_onnx_max_concurrent = int(os.environ.get('DML_MAX_CONCURRENT', '1'))
+ except Exception:
+ _rtdetr_onnx_max_concurrent = 1
+ _rtdetr_onnx_sema = threading.Semaphore(max(1, _rtdetr_onnx_max_concurrent))
+
+ def __init__(self, config_path: str = "config.json"):
+ """
+ Initialize the bubble detector.
+
+ Args:
+ config_path: Path to configuration file
+ """
+ self.config_path = config_path
+ self.config = self._load_config()
+
+ # YOLOv8 components (original)
+ self.model = None
+ self.model_loaded = False
+ self.model_type = None # 'yolo', 'onnx', or 'torch'
+ self.onnx_session = None
+
+ # RT-DETR components (new)
+ self.rtdetr_model = None
+ self.rtdetr_processor = None
+ self.rtdetr_loaded = False
+ self.rtdetr_repo = 'ogkalu/comic-text-and-bubble-detector'
+
+ # RT-DETR (ONNX) backend components
+ self.rtdetr_onnx_session = None
+ self.rtdetr_onnx_loaded = False
+ self.rtdetr_onnx_repo = 'ogkalu/comic-text-and-bubble-detector'
+
+ # RT-DETR class definitions
+ self.CLASS_BUBBLE = 0 # Empty speech bubble
+ self.CLASS_TEXT_BUBBLE = 1 # Bubble with text
+ self.CLASS_TEXT_FREE = 2 # Text without bubble
+
+ # Detection settings
+ self.default_confidence = 0.5
+ self.default_iou_threshold = 0.45
+ # Allow override from settings
+ try:
+ ocr_cfg = self.config.get('manga_settings', {}).get('ocr', {}) if isinstance(self.config, dict) else {}
+ self.default_max_detections = int(ocr_cfg.get('bubble_max_detections', 100))
+ self.max_det_yolo = int(ocr_cfg.get('bubble_max_detections_yolo', self.default_max_detections))
+ self.max_det_rtdetr = int(ocr_cfg.get('bubble_max_detections_rtdetr', self.default_max_detections))
+ except Exception:
+ self.default_max_detections = 100
+ self.max_det_yolo = 100
+ self.max_det_rtdetr = 100
+
+ # Cache directory for ONNX conversions
+ self.cache_dir = os.environ.get('BUBBLE_CACHE_DIR', 'models')
+ os.makedirs(self.cache_dir, exist_ok=True)
+
+ # GPU availability
+ self.use_gpu = TORCH_AVAILABLE and torch.cuda.is_available()
+ self.device = 'cuda' if self.use_gpu else 'cpu'
+
+ # Quantization/precision settings
+ adv_cfg = self.config.get('manga_settings', {}).get('advanced', {}) if isinstance(self.config, dict) else {}
+ ocr_cfg = self.config.get('manga_settings', {}).get('ocr', {}) if isinstance(self.config, dict) else {}
+ env_quant = os.environ.get('MODEL_QUANTIZE', 'false').lower() == 'true'
+ self.quantize_enabled = bool(env_quant or adv_cfg.get('quantize_models', False) or ocr_cfg.get('quantize_bubble_detector', False))
+ self.quantize_dtype = str(adv_cfg.get('torch_precision', os.environ.get('TORCH_PRECISION', 'auto'))).lower()
+ # Prefer advanced.onnx_quantize; fall back to env or global quantize
+ self.onnx_quantize_enabled = bool(adv_cfg.get('onnx_quantize', os.environ.get('ONNX_QUANTIZE', 'false').lower() == 'true' or self.quantize_enabled))
+
+ # Stop flag support
+ self.stop_flag = None
+ self._stopped = False
+ self.log_callback = None
+
+ logger.info(f"🗨️ BubbleDetector initialized")
+ logger.info(f" GPU: {'Available' if self.use_gpu else 'Not available'}")
+ logger.info(f" YOLO: {'Available' if YOLO_AVAILABLE else 'Not installed'}")
+ logger.info(f" ONNX: {'Available' if ONNX_AVAILABLE else 'Not installed'}")
+ logger.info(f" RT-DETR: {'Available' if TRANSFORMERS_AVAILABLE else 'Not installed'}")
+ logger.info(f" Quantization: {'ENABLED' if self.quantize_enabled else 'disabled'} (torch_precision={self.quantize_dtype}, onnx_quantize={'on' if self.onnx_quantize_enabled else 'off'})" )
+
+ def _load_config(self) -> Dict[str, Any]:
+ """Load configuration from file."""
+ if os.path.exists(self.config_path):
+ try:
+ with open(self.config_path, 'r', encoding='utf-8') as f:
+ return json.load(f)
+ except Exception as e:
+ logger.warning(f"Failed to load config: {e}")
+ return {}
+
+ def _save_config(self):
+ """Save configuration to file."""
+ try:
+ with open(self.config_path, 'w', encoding='utf-8') as f:
+ json.dump(self.config, f, indent=2)
+ except Exception as e:
+ logger.error(f"Failed to save config: {e}")
+
+ def set_stop_flag(self, stop_flag):
+ """Set the stop flag for checking interruptions"""
+ self.stop_flag = stop_flag
+ self._stopped = False
+
+ def set_log_callback(self, log_callback):
+ """Set log callback for GUI integration"""
+ self.log_callback = log_callback
+
+ def _check_stop(self) -> bool:
+ """Check if stop has been requested"""
+ if self._stopped:
+ return True
+ if self.stop_flag and self.stop_flag.is_set():
+ self._stopped = True
+ return True
+ # Check global manga translator cancellation
+ try:
+ from manga_translator import MangaTranslator
+ if MangaTranslator.is_globally_cancelled():
+ self._stopped = True
+ return True
+ except Exception:
+ pass
+ return False
+
+ def _log(self, message: str, level: str = "info"):
+ """Log message with stop suppression"""
+ # Suppress logs when stopped (allow only essential stop confirmation messages)
+ if self._check_stop():
+ essential_stop_keywords = [
+ "⏹️ Translation stopped by user",
+ "⏹️ Bubble detection stopped",
+ "cleanup", "🧹"
+ ]
+ if not any(keyword in message for keyword in essential_stop_keywords):
+ return
+
+ if self.log_callback:
+ self.log_callback(message, level)
+ else:
+ logger.info(message) if level == 'info' else getattr(logger, level, logger.info)(message)
+
+ def reset_stop_flags(self):
+ """Reset stop flags when starting new processing"""
+ self._stopped = False
+
+ def load_model(self, model_path: str, force_reload: bool = False) -> bool:
+ """
+ Load a YOLOv8 model for bubble detection.
+
+ Args:
+ model_path: Path to model file (.pt, .onnx, or .torchscript)
+ force_reload: Force reload even if model is already loaded
+
+ Returns:
+ True if model loaded successfully, False otherwise
+ """
+ try:
+ # If given a Hugging Face repo ID (e.g., 'owner/name'), fetch detector.onnx into models/
+ if model_path and (('/' in model_path) and not os.path.exists(model_path)):
+ try:
+ from huggingface_hub import hf_hub_download
+ os.makedirs(self.cache_dir, exist_ok=True)
+ logger.info(f"📥 Resolving repo '{model_path}' to detector.onnx in {self.cache_dir}...")
+ resolved = hf_hub_download(repo_id=model_path, filename='detector.onnx', cache_dir=self.cache_dir, local_dir=self.cache_dir, local_dir_use_symlinks=False)
+ if resolved and os.path.exists(resolved):
+ model_path = resolved
+ logger.info(f"✅ Downloaded detector.onnx to: {model_path}")
+ except Exception as repo_err:
+ logger.error(f"Failed to download from repo '{model_path}': {repo_err}")
+ if not os.path.exists(model_path):
+ logger.error(f"Model file not found: {model_path}")
+ return False
+
+ # Check if it's the same model already loaded
+ if self.model_loaded and not force_reload:
+ last_path = self.config.get('last_model_path', '')
+ if last_path == model_path:
+ logger.info("Model already loaded (same path)")
+ return True
+ else:
+ logger.info(f"Model path changed from {last_path} to {model_path}, reloading...")
+ force_reload = True
+
+ # Clear previous model if force reload
+ if force_reload:
+ logger.info("Force reloading model...")
+ self.model = None
+ self.onnx_session = None
+ self.model_loaded = False
+ self.model_type = None
+
+ logger.info(f"📥 Loading bubble detection model: {model_path}")
+
+ # Determine model type by extension
+ ext = Path(model_path).suffix.lower()
+
+ if ext in ['.pt', '.pth']:
+ if not YOLO_AVAILABLE:
+ logger.warning("Ultralytics package not available in this build")
+ logger.info("Bubble detection will be disabled - this is normal for lightweight builds")
+ # Don't return False immediately, try other fallbacks
+ self.model_loaded = False
+ return False
+
+ # Load YOLOv8 model
+ try:
+ self.model = YOLO(model_path)
+ self.model_type = 'yolo'
+
+ # Set to eval mode
+ if hasattr(self.model, 'model'):
+ self.model.model.eval()
+
+ # Move to GPU if available
+ if self.use_gpu and TORCH_AVAILABLE:
+ try:
+ self.model.to('cuda')
+ except Exception as gpu_error:
+ logger.warning(f"Could not move model to GPU: {gpu_error}")
+
+ logger.info("✅ YOLOv8 model loaded successfully")
+ # Apply optional FP16 precision to reduce VRAM if enabled
+ if self.quantize_enabled and self.use_gpu and TORCH_AVAILABLE:
+ try:
+ m = self.model.model if hasattr(self.model, 'model') else self.model
+ m.half()
+ logger.info("🔻 Applied FP16 precision to YOLO model (GPU)")
+ except Exception as _e:
+ logger.warning(f"Could not switch YOLO model to FP16: {_e}")
+
+ except Exception as yolo_error:
+ logger.error(f"Failed to load YOLO model: {yolo_error}")
+ return False
+
+ elif ext == '.onnx':
+ if not ONNX_AVAILABLE:
+ logger.warning("ONNX Runtime not available in this build")
+ logger.info("ONNX model support disabled - this is normal for lightweight builds")
+ return False
+
+ try:
+ # Load ONNX model
+ providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] if self.use_gpu else ['CPUExecutionProvider']
+ session_path = model_path
+ if self.quantize_enabled:
+ try:
+ from onnxruntime.quantization import quantize_dynamic, QuantType
+ quant_path = os.path.splitext(model_path)[0] + ".int8.onnx"
+ if not os.path.exists(quant_path) or os.environ.get('FORCE_ONNX_REBUILD', 'false').lower() == 'true':
+ logger.info("🔻 Quantizing ONNX model weights to INT8 (dynamic)...")
+ quantize_dynamic(model_input=model_path, model_output=quant_path, weight_type=QuantType.QInt8, op_types_to_quantize=['Conv', 'MatMul'])
+ session_path = quant_path
+ self.config['last_onnx_quantized_path'] = quant_path
+ self._save_config()
+ logger.info(f"✅ Using quantized ONNX model: {quant_path}")
+ except Exception as qe:
+ logger.warning(f"ONNX quantization not applied: {qe}")
+ # Use conservative ORT memory options to reduce RAM growth
+ so = ort.SessionOptions()
+ try:
+ so.enable_mem_pattern = False
+ so.enable_cpu_mem_arena = False
+ except Exception:
+ pass
+ self.onnx_session = ort.InferenceSession(session_path, sess_options=so, providers=providers)
+ self.model_type = 'onnx'
+
+ logger.info("✅ ONNX model loaded successfully")
+
+ except Exception as onnx_error:
+ logger.error(f"Failed to load ONNX model: {onnx_error}")
+ return False
+
+ elif ext == '.torchscript':
+ if not TORCH_AVAILABLE:
+ logger.warning("PyTorch not available in this build")
+ logger.info("TorchScript model support disabled - this is normal for lightweight builds")
+ return False
+
+ try:
+ # Add safety check for torch being None
+ if torch is None:
+ logger.error("PyTorch module is None - cannot load TorchScript model")
+ return False
+
+ # Load TorchScript model
+ self.model = torch.jit.load(model_path, map_location='cpu')
+ self.model.eval()
+ self.model_type = 'torch'
+
+ if self.use_gpu:
+ try:
+ self.model = self.model.cuda()
+ except Exception as gpu_error:
+ logger.warning(f"Could not move TorchScript model to GPU: {gpu_error}")
+
+ logger.info("✅ TorchScript model loaded successfully")
+
+ # Optional FP16 precision on GPU
+ if self.quantize_enabled and self.use_gpu and TORCH_AVAILABLE:
+ try:
+ self.model = self.model.half()
+ logger.info("🔻 Applied FP16 precision to TorchScript model (GPU)")
+ except Exception as _e:
+ logger.warning(f"Could not switch TorchScript model to FP16: {_e}")
+
+ except Exception as torch_error:
+ logger.error(f"Failed to load TorchScript model: {torch_error}")
+ return False
+
+ else:
+ logger.error(f"Unsupported model format: {ext}")
+ logger.info("Supported formats: .pt/.pth (YOLOv8), .onnx (ONNX), .torchscript (TorchScript)")
+ return False
+
+ # Only set loaded if we actually succeeded
+ self.model_loaded = True
+ self.config['last_model_path'] = model_path
+ self.config['model_type'] = self.model_type
+ self._save_config()
+
+ return True
+
+ except Exception as e:
+ logger.error(f"Failed to load model: {e}")
+ logger.error(traceback.format_exc())
+ self.model_loaded = False
+
+ # Provide helpful context for .exe users
+ logger.info("Note: If running from .exe, some ML libraries may not be included")
+ logger.info("This is normal for lightweight builds - bubble detection will be disabled")
+
+ return False
+
+ def load_rtdetr_model(self, model_path: str = None, model_id: str = None, force_reload: bool = False) -> bool:
+ """
+ Load RT-DETR model for advanced bubble and text detection.
+ This implementation avoids the 'meta tensor' copy error by:
+ - Serializing the entire load under a class lock (no concurrent loads)
+ - Loading directly onto the target device (CUDA if available) via device_map='auto'
+ - Avoiding .to() on a potentially-meta model; no device migration post-load
+
+ Args:
+ model_path: Optional path to local model
+ model_id: Optional HuggingFace model ID (default: 'ogkalu/comic-text-and-bubble-detector')
+ force_reload: Force reload even if already loaded
+
+ Returns:
+ True if successful, False otherwise
+ """
+ if not TRANSFORMERS_AVAILABLE:
+ logger.error("Transformers library required for RT-DETR. Install with: pip install transformers")
+ return False
+
+ if not PIL_AVAILABLE:
+ logger.error("PIL required for RT-DETR. Install with: pip install pillow")
+ return False
+
+ if self.rtdetr_loaded and not force_reload:
+ logger.info("RT-DETR model already loaded")
+ return True
+
+ # Fast path: if shared already loaded and not forcing reload, attach
+ if BubbleDetector._rtdetr_loaded and not force_reload:
+ self.rtdetr_model = BubbleDetector._rtdetr_shared_model
+ self.rtdetr_processor = BubbleDetector._rtdetr_shared_processor
+ self.rtdetr_loaded = True
+ logger.info("RT-DETR model attached from shared cache")
+ return True
+
+ # Serialize the ENTIRE loading sequence to avoid concurrent init issues
+ with BubbleDetector._rtdetr_init_lock:
+ try:
+ # Re-check after acquiring lock
+ if BubbleDetector._rtdetr_loaded and not force_reload:
+ self.rtdetr_model = BubbleDetector._rtdetr_shared_model
+ self.rtdetr_processor = BubbleDetector._rtdetr_shared_processor
+ self.rtdetr_loaded = True
+ logger.info("RT-DETR model attached from shared cache (post-lock)")
+ return True
+
+ # Use custom model_id if provided, otherwise use default
+ repo_id = model_id if model_id else self.rtdetr_repo
+ logger.info(f"📥 Loading RT-DETR model from {repo_id}...")
+
+ # Ensure TorchDynamo/compile doesn't interfere on some builds
+ try:
+ os.environ.setdefault('TORCHDYNAMO_DISABLE', '1')
+ except Exception:
+ pass
+
+ # Decide device strategy
+ gpu_available = bool(TORCH_AVAILABLE and hasattr(torch, 'cuda') and torch.cuda.is_available())
+ device_map = 'auto' if gpu_available else None
+ # Choose dtype
+ dtype = None
+ if TORCH_AVAILABLE:
+ try:
+ dtype = torch.float16 if gpu_available else torch.float32
+ except Exception:
+ dtype = None
+ low_cpu = True if gpu_available else False
+
+ # Load processor (once)
+ self.rtdetr_processor = RTDetrImageProcessor.from_pretrained(
+ repo_id,
+ size={"width": 640, "height": 640},
+ cache_dir=self.cache_dir if not model_path else None
+ )
+
+ # Prepare kwargs for from_pretrained
+ from_kwargs = {
+ 'cache_dir': self.cache_dir if not model_path else None,
+ 'low_cpu_mem_usage': low_cpu,
+ 'device_map': device_map,
+ }
+ if dtype is not None:
+ from_kwargs['dtype'] = dtype
+
+ # First attempt: load directly to target (CUDA if available)
+ try:
+ self.rtdetr_model = RTDetrForObjectDetection.from_pretrained(
+ model_path if model_path else repo_id,
+ **from_kwargs,
+ )
+ except Exception as primary_err:
+ # Fallback to a simple CPU load (no device move) if CUDA path fails
+ logger.warning(f"RT-DETR primary load failed ({primary_err}); retrying on CPU...")
+ from_kwargs_fallback = {
+ 'cache_dir': self.cache_dir if not model_path else None,
+ 'low_cpu_mem_usage': False,
+ 'device_map': None,
+ }
+ if TORCH_AVAILABLE:
+ from_kwargs_fallback['dtype'] = torch.float32
+ self.rtdetr_model = RTDetrForObjectDetection.from_pretrained(
+ model_path if model_path else repo_id,
+ **from_kwargs_fallback,
+ )
+
+ # Optional dynamic quantization for linear layers (CPU only)
+ if self.quantize_enabled and TORCH_AVAILABLE and (not gpu_available):
+ try:
+ try:
+ import torch.ao.quantization as tq
+ quantize_dynamic = tq.quantize_dynamic # type: ignore
+ except Exception:
+ import torch.quantization as tq # type: ignore
+ quantize_dynamic = tq.quantize_dynamic # type: ignore
+ self.rtdetr_model = quantize_dynamic(self.rtdetr_model, {torch.nn.Linear}, dtype=torch.qint8)
+ logger.info("🔻 Applied dynamic INT8 quantization to RT-DETR linear layers (CPU)")
+ except Exception as qe:
+ logger.warning(f"RT-DETR dynamic quantization skipped: {qe}")
+
+ # Finalize
+ self.rtdetr_model.eval()
+
+ # Sanity check: ensure no parameter is left on 'meta' device
+ try:
+ for n, p in self.rtdetr_model.named_parameters():
+ dev = getattr(p, 'device', None)
+ if dev is not None and getattr(dev, 'type', '') == 'meta':
+ raise RuntimeError(f"Parameter {n} is on 'meta' device after load")
+ except Exception as e:
+ logger.error(f"RT-DETR load sanity check failed: {e}")
+ self.rtdetr_loaded = False
+ return False
+
+ # Publish shared cache
+ BubbleDetector._rtdetr_shared_model = self.rtdetr_model
+ BubbleDetector._rtdetr_shared_processor = self.rtdetr_processor
+ BubbleDetector._rtdetr_loaded = True
+ BubbleDetector._rtdetr_repo_id = repo_id
+
+ self.rtdetr_loaded = True
+
+ # Save the model ID that was used
+ self.config['rtdetr_loaded'] = True
+ self.config['rtdetr_model_id'] = repo_id
+ self._save_config()
+
+ loc = 'CUDA' if gpu_available else 'CPU'
+ logger.info(f"✅ RT-DETR model loaded successfully ({loc})")
+ logger.info(" Classes: Empty bubbles, Text bubbles, Free text")
+
+ # Auto-convert to ONNX for RT-DETR only if explicitly enabled
+ if os.environ.get('AUTO_CONVERT_RTDETR_ONNX', 'false').lower() == 'true':
+ onnx_path = os.path.join(self.cache_dir, 'rtdetr_comic.onnx')
+ if self.convert_to_onnx('rtdetr', onnx_path):
+ logger.info("🚀 RT-DETR converted to ONNX for faster inference")
+ # Store ONNX path for later use
+ self.config['rtdetr_onnx_path'] = onnx_path
+ self._save_config()
+ # Optionally quantize ONNX for reduced RAM
+ if self.onnx_quantize_enabled:
+ try:
+ from onnxruntime.quantization import quantize_dynamic, QuantType
+ quant_path = os.path.splitext(onnx_path)[0] + ".int8.onnx"
+ if not os.path.exists(quant_path) or os.environ.get('FORCE_ONNX_REBUILD', 'false').lower() == 'true':
+ logger.info("🔻 Quantizing RT-DETR ONNX to INT8 (dynamic)...")
+ quantize_dynamic(model_input=onnx_path, model_output=quant_path, weight_type=QuantType.QInt8, op_types_to_quantize=['Conv', 'MatMul'])
+ self.config['rtdetr_onnx_quantized_path'] = quant_path
+ self._save_config()
+ logger.info(f"✅ Quantized RT-DETR ONNX saved to: {quant_path}")
+ except Exception as qe:
+ logger.warning(f"ONNX quantization for RT-DETR skipped: {qe}")
+ else:
+ logger.info("ℹ️ Skipping RT-DETR ONNX export (converter not supported in current environment)")
+
+ return True
+ except Exception as e:
+ logger.error(f"❌ Failed to load RT-DETR: {e}")
+ self.rtdetr_loaded = False
+ return False
+
+ def check_rtdetr_available(self, model_id: str = None) -> bool:
+ """
+ Check if RT-DETR model is available (cached).
+
+ Args:
+ model_id: Optional HuggingFace model ID
+
+ Returns:
+ True if model is cached and available
+ """
+ try:
+ from pathlib import Path
+
+ # Use provided model_id or default
+ repo_id = model_id if model_id else self.rtdetr_repo
+
+ # Check HuggingFace cache
+ cache_dir = Path.home() / ".cache" / "huggingface" / "hub"
+ model_id_formatted = repo_id.replace("/", "--")
+
+ # Look for model folder
+ model_folders = list(cache_dir.glob(f"models--{model_id_formatted}*"))
+
+ if model_folders:
+ for folder in model_folders:
+ if (folder / "snapshots").exists():
+ snapshots = list((folder / "snapshots").iterdir())
+ if snapshots:
+ return True
+
+ return False
+
+ except Exception:
+ return False
+
+ def detect_bubbles(self,
+ image_path: str,
+ confidence: float = None,
+ iou_threshold: float = None,
+ max_detections: int = None,
+ use_rtdetr: bool = None) -> List[Tuple[int, int, int, int]]:
+ """
+ Detect speech bubbles in an image (backward compatible method).
+
+ Args:
+ image_path: Path to image file
+ confidence: Minimum confidence threshold (0-1)
+ iou_threshold: IOU threshold for NMS (0-1)
+ max_detections: Maximum number of detections to return
+ use_rtdetr: If True, use RT-DETR instead of YOLOv8 (if available)
+
+ Returns:
+ List of bubble bounding boxes as (x, y, width, height) tuples
+ """
+ # Check for stop at start
+ if self._check_stop():
+ self._log("⏹️ Bubble detection stopped by user", "warning")
+ return []
+
+ # Decide which model to use
+ if use_rtdetr is None:
+ # Auto-select: prefer RT-DETR if available
+ use_rtdetr = self.rtdetr_loaded
+
+ if use_rtdetr:
+ # Prefer ONNX backend if available, else PyTorch
+ if getattr(self, 'rtdetr_onnx_loaded', False):
+ results = self.detect_with_rtdetr_onnx(
+ image_path=image_path,
+ confidence=confidence,
+ return_all_bubbles=True
+ )
+ return results
+ if self.rtdetr_loaded:
+ results = self.detect_with_rtdetr(
+ image_path=image_path,
+ confidence=confidence,
+ return_all_bubbles=True
+ )
+ return results
+
+ # Original YOLOv8 detection
+ if not self.model_loaded:
+ logger.error("No model loaded. Call load_model() first.")
+ return []
+
+ # Use defaults if not specified
+ confidence = confidence or self.default_confidence
+ iou_threshold = iou_threshold or self.default_iou_threshold
+ max_detections = max_detections or self.default_max_detections
+
+ try:
+ # Load image
+ image = cv2.imread(image_path)
+ if image is None:
+ logger.error(f"Failed to load image: {image_path}")
+ return []
+
+ h, w = image.shape[:2]
+ self._log(f"🔍 Detecting bubbles in {w}x{h} image")
+
+ # Check for stop before inference
+ if self._check_stop():
+ self._log("⏹️ Bubble detection inference stopped by user", "warning")
+ return []
+
+ if self.model_type == 'yolo':
+ # YOLOv8 inference
+ results = self.model(
+ image_path,
+ conf=confidence,
+ iou=iou_threshold,
+ max_det=min(max_detections, getattr(self, 'max_det_yolo', max_detections)),
+ verbose=False
+ )
+
+ bubbles = []
+ for r in results:
+ if r.boxes is not None:
+ for box in r.boxes:
+ # Get box coordinates
+ x1, y1, x2, y2 = box.xyxy[0].cpu().numpy()
+ x, y = int(x1), int(y1)
+ width = int(x2 - x1)
+ height = int(y2 - y1)
+
+ # Get confidence
+ conf = float(box.conf[0])
+
+ # Add to list
+ if len(bubbles) < max_detections:
+ bubbles.append((x, y, width, height))
+
+ logger.debug(f" Bubble: ({x},{y}) {width}x{height} conf={conf:.2f}")
+
+ elif self.model_type == 'onnx':
+ # ONNX inference
+ bubbles = self._detect_with_onnx(image, confidence, iou_threshold, max_detections)
+
+ elif self.model_type == 'torch':
+ # TorchScript inference
+ bubbles = self._detect_with_torchscript(image, confidence, iou_threshold, max_detections)
+
+ else:
+ logger.error(f"Unknown model type: {self.model_type}")
+ return []
+
+ logger.info(f"✅ Detected {len(bubbles)} speech bubbles")
+ time.sleep(0.1) # Brief pause for stability
+ logger.debug("💤 Bubble detection pausing briefly for stability")
+ return bubbles
+
+ except Exception as e:
+ logger.error(f"Detection failed: {e}")
+ logger.error(traceback.format_exc())
+ return []
+
+ def detect_with_rtdetr(self,
+ image_path: str = None,
+ image: np.ndarray = None,
+ confidence: float = None,
+ return_all_bubbles: bool = False) -> Any:
+ """
+ Detect using RT-DETR model with 3-class detection (PyTorch backend).
+
+ Args:
+ image_path: Path to image file
+ image: Image array (BGR format)
+ confidence: Confidence threshold
+ return_all_bubbles: If True, return list of bubble boxes (for compatibility)
+ If False, return dict with all classes
+
+ Returns:
+ List of bubbles if return_all_bubbles=True, else dict with classes
+ """
+ # Check for stop at start
+ if self._check_stop():
+ self._log("⏹️ RT-DETR detection stopped by user", "warning")
+ if return_all_bubbles:
+ return []
+ return {'bubbles': [], 'text_bubbles': [], 'text_free': []}
+
+ if not self.rtdetr_loaded:
+ self._log("RT-DETR not loaded. Call load_rtdetr_model() first.", "warning")
+ if return_all_bubbles:
+ return []
+ return {'bubbles': [], 'text_bubbles': [], 'text_free': []}
+
+ confidence = confidence or self.default_confidence
+
+ try:
+ # Load image
+ if image_path:
+ image = cv2.imread(image_path)
+ elif image is None:
+ logger.error("No image provided")
+ if return_all_bubbles:
+ return []
+ return {'bubbles': [], 'text_bubbles': [], 'text_free': []}
+
+ # Convert BGR to RGB for PIL
+ image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+ pil_image = Image.fromarray(image_rgb)
+
+ # Prepare image for model
+ inputs = self.rtdetr_processor(images=pil_image, return_tensors="pt")
+
+ # Move inputs to the same device as the model and match model dtype for floating tensors
+ model_device = next(self.rtdetr_model.parameters()).device if self.rtdetr_model is not None else (torch.device('cpu') if TORCH_AVAILABLE else 'cpu')
+ model_dtype = None
+ if TORCH_AVAILABLE and self.rtdetr_model is not None:
+ try:
+ model_dtype = next(self.rtdetr_model.parameters()).dtype
+ except Exception:
+ model_dtype = None
+
+ if TORCH_AVAILABLE:
+ new_inputs = {}
+ for k, v in inputs.items():
+ if isinstance(v, torch.Tensor):
+ v = v.to(model_device)
+ if model_dtype is not None and torch.is_floating_point(v):
+ v = v.to(model_dtype)
+ new_inputs[k] = v
+ inputs = new_inputs
+
+ # Run inference with autocast when model is half/bfloat16 on CUDA
+ use_amp = TORCH_AVAILABLE and hasattr(model_device, 'type') and model_device.type == 'cuda' and (model_dtype in (torch.float16, torch.bfloat16))
+ autocast_dtype = model_dtype if model_dtype in (torch.float16, torch.bfloat16) else None
+
+ with torch.no_grad():
+ if use_amp and autocast_dtype is not None:
+ with torch.autocast('cuda', dtype=autocast_dtype):
+ outputs = self.rtdetr_model(**inputs)
+ else:
+ outputs = self.rtdetr_model(**inputs)
+
+ # Brief pause for stability after inference
+ time.sleep(0.1)
+ logger.debug("💤 RT-DETR inference pausing briefly for stability")
+
+ # Post-process results
+ target_sizes = torch.tensor([pil_image.size[::-1]]) if TORCH_AVAILABLE else None
+ if TORCH_AVAILABLE and hasattr(model_device, 'type') and model_device.type == "cuda":
+ target_sizes = target_sizes.to(model_device)
+
+ results = self.rtdetr_processor.post_process_object_detection(
+ outputs,
+ target_sizes=target_sizes,
+ threshold=confidence
+ )[0]
+
+ # Apply per-detector cap if configured
+ cap = getattr(self, 'max_det_rtdetr', self.default_max_detections)
+ if cap and len(results['boxes']) > cap:
+ # Keep top-scoring first
+ scores = results['scores']
+ top_idx = scores.topk(k=cap).indices if hasattr(scores, 'topk') else range(cap)
+ results = {
+ 'boxes': [results['boxes'][i] for i in top_idx],
+ 'scores': [results['scores'][i] for i in top_idx],
+ 'labels': [results['labels'][i] for i in top_idx]
+ }
+
+ logger.info(f"📊 RT-DETR found {len(results['boxes'])} detections above {confidence:.2f} confidence")
+
+ # Organize detections by class
+ detections = {
+ 'bubbles': [], # Empty speech bubbles
+ 'text_bubbles': [], # Bubbles with text
+ 'text_free': [] # Text without bubbles
+ }
+
+ for box, score, label in zip(results['boxes'], results['scores'], results['labels']):
+ x1, y1, x2, y2 = map(int, box.tolist())
+ width = x2 - x1
+ height = y2 - y1
+
+ # Store as (x, y, width, height) to match YOLOv8 format
+ bbox = (x1, y1, width, height)
+
+ label_id = label.item()
+ if label_id == self.CLASS_BUBBLE:
+ detections['bubbles'].append(bbox)
+ elif label_id == self.CLASS_TEXT_BUBBLE:
+ detections['text_bubbles'].append(bbox)
+ elif label_id == self.CLASS_TEXT_FREE:
+ detections['text_free'].append(bbox)
+
+ # Stop early if we hit the configured cap across all classes
+ total_count = len(detections['bubbles']) + len(detections['text_bubbles']) + len(detections['text_free'])
+ if total_count >= (self.config.get('manga_settings', {}).get('ocr', {}).get('bubble_max_detections', self.default_max_detections) if isinstance(self.config, dict) else self.default_max_detections):
+ break
+
+ # Log results
+ total = len(detections['bubbles']) + len(detections['text_bubbles']) + len(detections['text_free'])
+ logger.info(f"✅ RT-DETR detected {total} objects:")
+ logger.info(f" - Empty bubbles: {len(detections['bubbles'])}")
+ logger.info(f" - Text bubbles: {len(detections['text_bubbles'])}")
+ logger.info(f" - Free text: {len(detections['text_free'])}")
+
+ # Return format based on compatibility mode
+ if return_all_bubbles:
+ # Return all bubbles (empty + with text) for backward compatibility
+ all_bubbles = detections['bubbles'] + detections['text_bubbles']
+ return all_bubbles
+ else:
+ return detections
+
+ except Exception as e:
+ logger.error(f"RT-DETR detection failed: {e}")
+ logger.error(traceback.format_exc())
+ if return_all_bubbles:
+ return []
+ return {'bubbles': [], 'text_bubbles': [], 'text_free': []}
+
+ def detect_all_text_regions(self, image_path: str = None, image: np.ndarray = None) -> List[Tuple[int, int, int, int]]:
+ """
+ Detect all text regions using RT-DETR (both in bubbles and free text).
+
+ Returns:
+ List of bounding boxes for all text regions
+ """
+ if not self.rtdetr_loaded:
+ logger.warning("RT-DETR required for text detection")
+ return []
+
+ detections = self.detect_with_rtdetr(image_path=image_path, image=image, return_all_bubbles=False)
+
+ # Combine text bubbles and free text
+ all_text = detections['text_bubbles'] + detections['text_free']
+
+ logger.info(f"📝 Found {len(all_text)} text regions total")
+ return all_text
+
+ def _detect_with_onnx(self, image: np.ndarray, confidence: float,
+ iou_threshold: float, max_detections: int) -> List[Tuple[int, int, int, int]]:
+ """Run detection using ONNX model."""
+ # Preprocess image
+ img_size = 640 # Standard YOLOv8 input size
+ img_resized = cv2.resize(image, (img_size, img_size))
+ img_norm = img_resized.astype(np.float32) / 255.0
+ img_transposed = np.transpose(img_norm, (2, 0, 1))
+ img_batch = np.expand_dims(img_transposed, axis=0)
+
+ # Run inference
+ input_name = self.onnx_session.get_inputs()[0].name
+ outputs = self.onnx_session.run(None, {input_name: img_batch})
+
+ # Process outputs (YOLOv8 format)
+ predictions = outputs[0][0] # Remove batch dimension
+
+ # Filter by confidence and apply NMS
+ bubbles = []
+ boxes = []
+ scores = []
+
+ for pred in predictions.T: # Transpose to get predictions per detection
+ if len(pred) >= 5:
+ x_center, y_center, width, height, obj_conf = pred[:5]
+
+ if obj_conf >= confidence:
+ # Convert to corner coordinates
+ x1 = x_center - width / 2
+ y1 = y_center - height / 2
+
+ # Scale to original image size
+ h, w = image.shape[:2]
+ x1 = int(x1 * w / img_size)
+ y1 = int(y1 * h / img_size)
+ width = int(width * w / img_size)
+ height = int(height * h / img_size)
+
+ boxes.append([x1, y1, x1 + width, y1 + height])
+ scores.append(float(obj_conf))
+
+ # Apply NMS
+ if boxes:
+ indices = cv2.dnn.NMSBoxes(boxes, scores, confidence, iou_threshold)
+ if len(indices) > 0:
+ indices = indices.flatten()[:max_detections]
+ for i in indices:
+ x1, y1, x2, y2 = boxes[i]
+ bubbles.append((x1, y1, x2 - x1, y2 - y1))
+
+ return bubbles
+
+ def _detect_with_torchscript(self, image: np.ndarray, confidence: float,
+ iou_threshold: float, max_detections: int) -> List[Tuple[int, int, int, int]]:
+ """Run detection using TorchScript model."""
+ # Similar to ONNX but using PyTorch tensors
+ img_size = 640
+ img_resized = cv2.resize(image, (img_size, img_size))
+ img_norm = img_resized.astype(np.float32) / 255.0
+ img_tensor = torch.from_numpy(img_norm).permute(2, 0, 1).unsqueeze(0)
+
+ if self.use_gpu:
+ img_tensor = img_tensor.cuda()
+
+ with torch.no_grad():
+ outputs = self.model(img_tensor)
+
+ # Process outputs similar to ONNX
+ # Implementation depends on exact model output format
+ # This is a placeholder - adjust based on your model
+ return []
+
+ def visualize_detections(self, image_path: str, bubbles: List[Tuple[int, int, int, int]] = None,
+ output_path: str = None, use_rtdetr: bool = False) -> np.ndarray:
+ """
+ Visualize detected bubbles on the image.
+
+ Args:
+ image_path: Path to original image
+ bubbles: List of bubble bounding boxes (if None, will detect)
+ output_path: Optional path to save visualization
+ use_rtdetr: Use RT-DETR for visualization with class colors
+
+ Returns:
+ Image with drawn bounding boxes
+ """
+ image = cv2.imread(image_path)
+ if image is None:
+ logger.error(f"Failed to load image: {image_path}")
+ return None
+
+ vis_image = image.copy()
+
+ if use_rtdetr and self.rtdetr_loaded:
+ # RT-DETR visualization with different colors per class
+ detections = self.detect_with_rtdetr(image_path=image_path, return_all_bubbles=False)
+
+ # Colors for each class
+ colors = {
+ 'bubbles': (0, 255, 0), # Green for empty bubbles
+ 'text_bubbles': (255, 0, 0), # Blue for text bubbles
+ 'text_free': (0, 0, 255) # Red for free text
+ }
+
+ # Draw detections
+ for class_name, bboxes in detections.items():
+ color = colors[class_name]
+
+ for i, (x, y, w, h) in enumerate(bboxes):
+ # Draw rectangle
+ cv2.rectangle(vis_image, (x, y), (x + w, y + h), color, 2)
+
+ # Add label
+ label = f"{class_name.replace('_', ' ').title()} {i+1}"
+ label_size, _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
+ cv2.rectangle(vis_image, (x, y - label_size[1] - 4),
+ (x + label_size[0], y), color, -1)
+ cv2.putText(vis_image, label, (x, y - 2),
+ cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1)
+ else:
+ # Original YOLOv8 visualization
+ if bubbles is None:
+ bubbles = self.detect_bubbles(image_path)
+
+ # Draw bounding boxes
+ for i, (x, y, w, h) in enumerate(bubbles):
+ # Draw rectangle
+ color = (0, 255, 0) # Green
+ thickness = 2
+ cv2.rectangle(vis_image, (x, y), (x + w, y + h), color, thickness)
+
+ # Add label
+ label = f"Bubble {i+1}"
+ label_size, _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
+ cv2.rectangle(vis_image, (x, y - label_size[1] - 4), (x + label_size[0], y), color, -1)
+ cv2.putText(vis_image, label, (x, y - 2), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1)
+
+ # Save if output path provided
+ if output_path:
+ cv2.imwrite(output_path, vis_image)
+ logger.info(f"💾 Visualization saved to: {output_path}")
+
+ return vis_image
+
+ def convert_to_onnx(self, model_path: str, output_path: str = None) -> bool:
+ """
+ Convert a YOLOv8 or RT-DETR model to ONNX format.
+
+ Args:
+ model_path: Path to model file or 'rtdetr' for loaded RT-DETR
+ output_path: Path for ONNX output (auto-generated if None)
+
+ Returns:
+ True if conversion successful, False otherwise
+ """
+ try:
+ logger.info(f"🔄 Converting {model_path} to ONNX...")
+
+ # Generate output path if not provided
+ if output_path is None:
+ if model_path == 'rtdetr' and self.rtdetr_loaded:
+ base_name = 'rtdetr_comic'
+ else:
+ base_name = Path(model_path).stem
+ output_path = os.path.join(self.cache_dir, f"{base_name}.onnx")
+
+ # Check if already exists
+ if os.path.exists(output_path) and not os.environ.get('FORCE_ONNX_REBUILD', 'false').lower() == 'true':
+ logger.info(f"✅ ONNX model already exists: {output_path}")
+ return True
+
+ # Handle RT-DETR conversion
+ if model_path == 'rtdetr' and self.rtdetr_loaded:
+ if not TORCH_AVAILABLE:
+ logger.error("PyTorch required for RT-DETR ONNX conversion")
+ return False
+
+ # RT-DETR specific conversion
+ self.rtdetr_model.eval()
+
+ # Create dummy input (pixel values): BxCxHxW
+ dummy_input = torch.randn(1, 3, 640, 640)
+ if self.device == 'cuda':
+ dummy_input = dummy_input.to('cuda')
+
+ # Wrap the model to return only tensors (logits, pred_boxes)
+ class _RTDetrExportWrapper(torch.nn.Module):
+ def __init__(self, mdl):
+ super().__init__()
+ self.mdl = mdl
+ def forward(self, images):
+ out = self.mdl(pixel_values=images)
+ # Handle dict/ModelOutput/tuple outputs
+ logits = None
+ boxes = None
+ try:
+ if isinstance(out, dict):
+ logits = out.get('logits', None)
+ boxes = out.get('pred_boxes', out.get('boxes', None))
+ else:
+ logits = getattr(out, 'logits', None)
+ boxes = getattr(out, 'pred_boxes', getattr(out, 'boxes', None))
+ except Exception:
+ pass
+ if (logits is None or boxes is None) and isinstance(out, (tuple, list)) and len(out) >= 2:
+ logits, boxes = out[0], out[1]
+ return logits, boxes
+
+ wrapper = _RTDetrExportWrapper(self.rtdetr_model)
+ if self.device == 'cuda':
+ wrapper = wrapper.to('cuda')
+
+ # Try PyTorch 2.x dynamo_export first (more tolerant of newer aten ops)
+ try:
+ success = False
+ try:
+ from torch.onnx import dynamo_export
+ try:
+ exp = dynamo_export(wrapper, dummy_input)
+ except TypeError:
+ # Older PyTorch dynamo_export may not support this calling convention
+ exp = dynamo_export(wrapper, dummy_input)
+ # exp may have save(); otherwise, it may expose model_proto
+ try:
+ exp.save(output_path) # type: ignore
+ success = True
+ except Exception:
+ try:
+ import onnx as _onnx
+ _onnx.save(exp.model_proto, output_path) # type: ignore
+ success = True
+ except Exception as _se:
+ logger.warning(f"dynamo_export produced model but could not save: {_se}")
+ except Exception as de:
+ logger.warning(f"dynamo_export failed; falling back to legacy exporter: {de}")
+ if success:
+ logger.info(f"✅ RT-DETR ONNX saved to: {output_path} (dynamo_export)")
+ return True
+ except Exception as de2:
+ logger.warning(f"dynamo_export path error: {de2}")
+
+ # Legacy exporter with opset fallback
+ last_err = None
+ for opset in [19, 18, 17, 16, 15, 14, 13]:
+ try:
+ torch.onnx.export(
+ wrapper,
+ dummy_input,
+ output_path,
+ export_params=True,
+ opset_version=opset,
+ do_constant_folding=True,
+ input_names=['pixel_values'],
+ output_names=['logits', 'boxes'],
+ dynamic_axes={
+ 'pixel_values': {0: 'batch', 2: 'height', 3: 'width'},
+ 'logits': {0: 'batch'},
+ 'boxes': {0: 'batch'}
+ }
+ )
+ logger.info(f"✅ RT-DETR ONNX saved to: {output_path} (opset {opset})")
+ return True
+ except Exception as _e:
+ last_err = _e
+ try:
+ msg = str(_e)
+ except Exception:
+ msg = ''
+ logger.warning(f"RT-DETR ONNX export failed at opset {opset}: {msg}")
+ continue
+
+ logger.error(f"All RT-DETR ONNX export attempts failed. Last error: {last_err}")
+ return False
+
+ # Handle YOLOv8 conversion - FIXED
+ elif YOLO_AVAILABLE and os.path.exists(model_path):
+ logger.info(f"Loading YOLOv8 model from: {model_path}")
+
+ # Load model
+ model = YOLO(model_path)
+
+ # Export to ONNX - this returns the path to the exported model
+ logger.info("Exporting to ONNX format...")
+ exported_path = model.export(format='onnx', imgsz=640, simplify=True)
+
+ # exported_path could be a string or Path object
+ exported_path = str(exported_path) if exported_path else None
+
+ if exported_path and os.path.exists(exported_path):
+ # Move to desired location if different
+ if exported_path != output_path:
+ import shutil
+ logger.info(f"Moving ONNX from {exported_path} to {output_path}")
+ shutil.move(exported_path, output_path)
+
+ logger.info(f"✅ YOLOv8 ONNX saved to: {output_path}")
+ return True
+ else:
+ # Fallback: check if it was created with expected name
+ expected_onnx = model_path.replace('.pt', '.onnx')
+ if os.path.exists(expected_onnx):
+ if expected_onnx != output_path:
+ import shutil
+ shutil.move(expected_onnx, output_path)
+ logger.info(f"✅ YOLOv8 ONNX saved to: {output_path}")
+ return True
+ else:
+ logger.error(f"ONNX export failed - no output file found")
+ return False
+
+ else:
+ logger.error(f"Cannot convert {model_path}: Model not found or dependencies missing")
+ return False
+
+ except Exception as e:
+ logger.error(f"Conversion failed: {e}")
+ # Avoid noisy full stack trace in production logs; return False gracefully
+ return False
+
+ def batch_detect(self, image_paths: List[str], **kwargs) -> Dict[str, List[Tuple[int, int, int, int]]]:
+ """
+ Detect bubbles in multiple images.
+
+ Args:
+ image_paths: List of image paths
+ **kwargs: Detection parameters (confidence, iou_threshold, max_detections, use_rtdetr)
+
+ Returns:
+ Dictionary mapping image paths to bubble lists
+ """
+ results = {}
+
+ for i, image_path in enumerate(image_paths):
+ logger.info(f"Processing image {i+1}/{len(image_paths)}: {os.path.basename(image_path)}")
+ bubbles = self.detect_bubbles(image_path, **kwargs)
+ results[image_path] = bubbles
+
+ return results
+
+ def unload(self, release_shared: bool = False):
+ """Release model resources held by this detector instance.
+ Args:
+ release_shared: If True, also clear class-level shared RT-DETR caches.
+ """
+ try:
+ # Release instance-level models and sessions
+ try:
+ if getattr(self, 'onnx_session', None) is not None:
+ self.onnx_session = None
+ except Exception:
+ pass
+ try:
+ if getattr(self, 'rtdetr_onnx_session', None) is not None:
+ self.rtdetr_onnx_session = None
+ except Exception:
+ pass
+ for attr in ['model', 'rtdetr_model', 'rtdetr_processor']:
+ try:
+ if hasattr(self, attr):
+ setattr(self, attr, None)
+ except Exception:
+ pass
+ for flag in ['model_loaded', 'rtdetr_loaded', 'rtdetr_onnx_loaded']:
+ try:
+ if hasattr(self, flag):
+ setattr(self, flag, False)
+ except Exception:
+ pass
+
+ # Optional: release shared caches
+ if release_shared:
+ try:
+ BubbleDetector._rtdetr_shared_model = None
+ BubbleDetector._rtdetr_shared_processor = None
+ BubbleDetector._rtdetr_loaded = False
+ except Exception:
+ pass
+
+ # Free CUDA cache and trigger GC
+ try:
+ if TORCH_AVAILABLE and torch is not None and torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ except Exception:
+ pass
+ try:
+ import gc
+ gc.collect()
+ except Exception:
+ pass
+ except Exception:
+ # Best-effort only
+ pass
+
+ def get_bubble_masks(self, image_path: str, bubbles: List[Tuple[int, int, int, int]]) -> np.ndarray:
+ """
+ Create a mask image with bubble regions.
+
+ Args:
+ image_path: Path to original image
+ bubbles: List of bubble bounding boxes
+
+ Returns:
+ Binary mask with bubble regions as white (255)
+ """
+ image = cv2.imread(image_path)
+ if image is None:
+ return None
+
+ h, w = image.shape[:2]
+ mask = np.zeros((h, w), dtype=np.uint8)
+
+ # Fill bubble regions
+ for x, y, bw, bh in bubbles:
+ cv2.rectangle(mask, (x, y), (x + bw, y + bh), 255, -1)
+
+ return mask
+
+ def filter_bubbles_by_size(self, bubbles: List[Tuple[int, int, int, int]],
+ min_area: int = 100,
+ max_area: int = None) -> List[Tuple[int, int, int, int]]:
+ """
+ Filter bubbles by area.
+
+ Args:
+ bubbles: List of bubble bounding boxes
+ min_area: Minimum area in pixels
+ max_area: Maximum area in pixels (None for no limit)
+
+ Returns:
+ Filtered list of bubbles
+ """
+ filtered = []
+
+ for x, y, w, h in bubbles:
+ area = w * h
+ if area >= min_area and (max_area is None or area <= max_area):
+ filtered.append((x, y, w, h))
+
+ return filtered
+
+ def merge_overlapping_bubbles(self, bubbles: List[Tuple[int, int, int, int]],
+ overlap_threshold: float = 0.1) -> List[Tuple[int, int, int, int]]:
+ """
+ Merge overlapping bubble detections.
+
+ Args:
+ bubbles: List of bubble bounding boxes
+ overlap_threshold: Minimum overlap ratio to merge
+
+ Returns:
+ Merged list of bubbles
+ """
+ if not bubbles:
+ return []
+
+ # Convert to numpy array for easier manipulation
+ boxes = np.array([(x, y, x+w, y+h) for x, y, w, h in bubbles])
+
+ merged = []
+ used = set()
+
+ for i, box1 in enumerate(boxes):
+ if i in used:
+ continue
+
+ # Start with current box
+ x1, y1, x2, y2 = box1
+
+ # Check for overlaps with remaining boxes
+ for j in range(i + 1, len(boxes)):
+ if j in used:
+ continue
+
+ box2 = boxes[j]
+
+ # Calculate intersection
+ ix1 = max(x1, box2[0])
+ iy1 = max(y1, box2[1])
+ ix2 = min(x2, box2[2])
+ iy2 = min(y2, box2[3])
+
+ if ix1 < ix2 and iy1 < iy2:
+ # Calculate overlap ratio
+ intersection = (ix2 - ix1) * (iy2 - iy1)
+ area1 = (x2 - x1) * (y2 - y1)
+ area2 = (box2[2] - box2[0]) * (box2[3] - box2[1])
+ overlap = intersection / min(area1, area2)
+
+ if overlap >= overlap_threshold:
+ # Merge boxes
+ x1 = min(x1, box2[0])
+ y1 = min(y1, box2[1])
+ x2 = max(x2, box2[2])
+ y2 = max(y2, box2[3])
+ used.add(j)
+
+ merged.append((int(x1), int(y1), int(x2 - x1), int(y2 - y1)))
+
+ return merged
+
+ # ============================
+ # RT-DETR (ONNX) BACKEND
+ # ============================
+ def load_rtdetr_onnx_model(self, model_id: str = None, force_reload: bool = False) -> bool:
+ """
+ Load RT-DETR ONNX model using onnxruntime. Downloads detector.onnx and config.json
+ from the provided Hugging Face repo if not already cached.
+ """
+ if not ONNX_AVAILABLE:
+ logger.error("ONNX Runtime not available for RT-DETR ONNX backend")
+ return False
+ try:
+ # If singleton mode and already loaded, just attach shared session
+ try:
+ adv = (self.config or {}).get('manga_settings', {}).get('advanced', {}) if isinstance(self.config, dict) else {}
+ singleton = bool(adv.get('use_singleton_models', True))
+ except Exception:
+ singleton = True
+ if singleton and BubbleDetector._rtdetr_onnx_loaded and not force_reload and BubbleDetector._rtdetr_onnx_shared_session is not None:
+ self.rtdetr_onnx_session = BubbleDetector._rtdetr_onnx_shared_session
+ self.rtdetr_onnx_loaded = True
+ return True
+
+ repo = model_id or self.rtdetr_onnx_repo
+ try:
+ from huggingface_hub import hf_hub_download
+ except Exception as e:
+ logger.error(f"huggingface-hub required to fetch RT-DETR ONNX: {e}")
+ return False
+
+ # Ensure local models dir (use configured cache_dir directly: e.g., 'models')
+ cache_dir = self.cache_dir
+ os.makedirs(cache_dir, exist_ok=True)
+
+ # Download files into models/ and avoid symlinks so the file is visible there
+ try:
+ _ = hf_hub_download(repo_id=repo, filename='config.json', cache_dir=cache_dir, local_dir=cache_dir, local_dir_use_symlinks=False)
+ except Exception:
+ pass
+ onnx_fp = hf_hub_download(repo_id=repo, filename='detector.onnx', cache_dir=cache_dir, local_dir=cache_dir, local_dir_use_symlinks=False)
+ BubbleDetector._rtdetr_onnx_model_path = onnx_fp
+
+ # Pick providers: prefer CUDA if available; otherwise CPU. Do NOT use DML.
+ providers = ['CPUExecutionProvider']
+ try:
+ avail = ort.get_available_providers() if ONNX_AVAILABLE else []
+ if 'CUDAExecutionProvider' in avail:
+ providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']
+ except Exception:
+ pass
+
+ # Session options with reduced memory arena and optional thread limiting in singleton mode
+ so = ort.SessionOptions()
+ try:
+ so.enable_mem_pattern = False
+ so.enable_cpu_mem_arena = False
+ except Exception:
+ pass
+ # If singleton models mode is enabled in config, limit ORT threading to reduce CPU spikes
+ try:
+ adv = (self.config or {}).get('manga_settings', {}).get('advanced', {}) if isinstance(self.config, dict) else {}
+ if bool(adv.get('use_singleton_models', True)):
+ so.intra_op_num_threads = 1
+ so.inter_op_num_threads = 1
+ try:
+ so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
+ except Exception:
+ pass
+ try:
+ so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_BASIC
+ except Exception:
+ pass
+ except Exception:
+ pass
+
+ # Create session (serialize creation in singleton mode to avoid device storms)
+ if singleton:
+ with BubbleDetector._rtdetr_onnx_init_lock:
+ # Re-check after acquiring lock
+ if BubbleDetector._rtdetr_onnx_loaded and BubbleDetector._rtdetr_onnx_shared_session is not None and not force_reload:
+ self.rtdetr_onnx_session = BubbleDetector._rtdetr_onnx_shared_session
+ self.rtdetr_onnx_loaded = True
+ return True
+ sess = ort.InferenceSession(onnx_fp, providers=providers, sess_options=so)
+ BubbleDetector._rtdetr_onnx_shared_session = sess
+ BubbleDetector._rtdetr_onnx_loaded = True
+ BubbleDetector._rtdetr_onnx_providers = providers
+ self.rtdetr_onnx_session = sess
+ self.rtdetr_onnx_loaded = True
+ else:
+ self.rtdetr_onnx_session = ort.InferenceSession(onnx_fp, providers=providers, sess_options=so)
+ self.rtdetr_onnx_loaded = True
+ logger.info("✅ RT-DETR (ONNX) model ready")
+ return True
+ except Exception as e:
+ logger.error(f"Failed to load RT-DETR ONNX: {e}")
+ self.rtdetr_onnx_session = None
+ self.rtdetr_onnx_loaded = False
+ return False
+
+ def detect_with_rtdetr_onnx(self,
+ image_path: str = None,
+ image: np.ndarray = None,
+ confidence: float = 0.3,
+ return_all_bubbles: bool = False) -> Any:
+ """Detect using RT-DETR ONNX backend.
+ Returns bubbles list if return_all_bubbles else dict by classes similar to PyTorch path.
+ """
+ if not self.rtdetr_onnx_loaded or self.rtdetr_onnx_session is None:
+ logger.warning("RT-DETR ONNX not loaded")
+ return [] if return_all_bubbles else {'bubbles': [], 'text_bubbles': [], 'text_free': []}
+ try:
+ # Acquire image
+ if image_path is not None:
+ import cv2
+ image = cv2.imread(image_path)
+ if image is None:
+ raise RuntimeError(f"Failed to read image: {image_path}")
+ image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+ else:
+ if image is None:
+ raise RuntimeError("No image provided")
+ # Assume image is BGR np.ndarray if from OpenCV
+ try:
+ import cv2
+ image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+ except Exception:
+ image_rgb = image
+
+ # To PIL then resize 640x640 as in reference
+ from PIL import Image as _PILImage
+ pil_image = _PILImage.fromarray(image_rgb)
+ im_resized = pil_image.resize((640, 640))
+ arr = np.asarray(im_resized, dtype=np.float32) / 255.0
+ arr = np.transpose(arr, (2, 0, 1)) # (3,H,W)
+ im_data = arr[np.newaxis, ...]
+
+ w, h = pil_image.size
+ orig_size = np.array([[w, h]], dtype=np.int64)
+
+ # Run with a concurrency guard when using DML to prevent device hangs
+ providers = BubbleDetector._rtdetr_onnx_providers or []
+ def _do_run(session):
+ return session.run(None, {
+ 'images': im_data,
+ 'orig_target_sizes': orig_size
+ })
+ if 'DmlExecutionProvider' in providers:
+ acquired = False
+ try:
+ BubbleDetector._rtdetr_onnx_sema.acquire()
+ acquired = True
+ outputs = _do_run(self.rtdetr_onnx_session)
+ except Exception as dml_err:
+ msg = str(dml_err)
+ if '887A0005' in msg or '887A0006' in msg or 'Dml' in msg:
+ # Rebuild CPU session and retry once
+ try:
+ base_path = BubbleDetector._rtdetr_onnx_model_path
+ if base_path:
+ so = ort.SessionOptions()
+ so.enable_mem_pattern = False
+ so.enable_cpu_mem_arena = False
+ cpu_providers = ['CPUExecutionProvider']
+ # Serialize rebuild
+ with BubbleDetector._rtdetr_onnx_init_lock:
+ sess = ort.InferenceSession(base_path, providers=cpu_providers, sess_options=so)
+ BubbleDetector._rtdetr_onnx_shared_session = sess
+ BubbleDetector._rtdetr_onnx_providers = cpu_providers
+ self.rtdetr_onnx_session = sess
+ outputs = _do_run(self.rtdetr_onnx_session)
+ else:
+ raise
+ except Exception:
+ raise
+ else:
+ raise
+ finally:
+ if acquired:
+ try:
+ BubbleDetector._rtdetr_onnx_sema.release()
+ except Exception:
+ pass
+ else:
+ outputs = _do_run(self.rtdetr_onnx_session)
+
+ # outputs expected: labels, boxes, scores
+ labels, boxes, scores = outputs[:3]
+ if labels.ndim == 2 and labels.shape[0] == 1:
+ labels = labels[0]
+ if scores.ndim == 2 and scores.shape[0] == 1:
+ scores = scores[0]
+ if boxes.ndim == 3 and boxes.shape[0] == 1:
+ boxes = boxes[0]
+
+ detections = {'bubbles': [], 'text_bubbles': [], 'text_free': []}
+ bubbles_all = []
+ for lab, box, scr in zip(labels, boxes, scores):
+ if float(scr) < float(confidence):
+ continue
+ x1, y1, x2, y2 = map(int, box)
+ bbox = (x1, y1, x2 - x1, y2 - y1)
+ label_id = int(lab)
+ if label_id == self.CLASS_BUBBLE:
+ detections['bubbles'].append(bbox)
+ bubbles_all.append(bbox)
+ elif label_id == self.CLASS_TEXT_BUBBLE:
+ detections['text_bubbles'].append(bbox)
+ bubbles_all.append(bbox)
+ elif label_id == self.CLASS_TEXT_FREE:
+ detections['text_free'].append(bbox)
+
+ return bubbles_all if return_all_bubbles else detections
+ except Exception as e:
+ logger.error(f"RT-DETR ONNX detection failed: {e}")
+ return [] if return_all_bubbles else {'bubbles': [], 'text_bubbles': [], 'text_free': []}
+
+
+# Standalone utility functions
+def download_model_from_huggingface(repo_id: str = "ogkalu/comic-speech-bubble-detector-yolov8m",
+ filename: str = "comic-speech-bubble-detector-yolov8m.pt",
+ cache_dir: str = "models") -> str:
+ """
+ Download model from Hugging Face Hub.
+
+ Args:
+ repo_id: Hugging Face repository ID
+ filename: Model filename in the repository
+ cache_dir: Local directory to cache the model
+
+ Returns:
+ Path to downloaded model file
+ """
+ try:
+ from huggingface_hub import hf_hub_download
+
+ os.makedirs(cache_dir, exist_ok=True)
+
+ logger.info(f"📥 Downloading {filename} from {repo_id}...")
+
+ model_path = hf_hub_download(
+ repo_id=repo_id,
+ filename=filename,
+ cache_dir=cache_dir,
+ local_dir=cache_dir
+ )
+
+ logger.info(f"✅ Model downloaded to: {model_path}")
+ return model_path
+
+ except ImportError:
+ logger.error("huggingface-hub package required. Install with: pip install huggingface-hub")
+ return None
+ except Exception as e:
+ logger.error(f"Download failed: {e}")
+ return None
+
+
+def download_rtdetr_model(cache_dir: str = "models") -> bool:
+ """
+ Download RT-DETR model for advanced detection.
+
+ Args:
+ cache_dir: Directory to cache the model
+
+ Returns:
+ True if successful
+ """
+ if not TRANSFORMERS_AVAILABLE:
+ logger.error("Transformers required. Install with: pip install transformers")
+ return False
+
+ try:
+ logger.info("📥 Downloading RT-DETR model...")
+ from transformers import RTDetrForObjectDetection, RTDetrImageProcessor
+
+ # This will download and cache the model
+ processor = RTDetrImageProcessor.from_pretrained(
+ "ogkalu/comic-text-and-bubble-detector",
+ cache_dir=cache_dir
+ )
+ model = RTDetrForObjectDetection.from_pretrained(
+ "ogkalu/comic-text-and-bubble-detector",
+ cache_dir=cache_dir
+ )
+
+ logger.info("✅ RT-DETR model downloaded successfully")
+ return True
+
+ except Exception as e:
+ logger.error(f"Download failed: {e}")
+ return False
+
+
+# Example usage and testing
+if __name__ == "__main__":
+ import sys
+
+ # Create detector
+ detector = BubbleDetector()
+
+ if len(sys.argv) > 1:
+ if sys.argv[1] == "download":
+ # Download model from Hugging Face
+ model_path = download_model_from_huggingface()
+ if model_path:
+ print(f"YOLOv8 model downloaded to: {model_path}")
+
+ # Also download RT-DETR
+ if download_rtdetr_model():
+ print("RT-DETR model downloaded")
+
+ elif sys.argv[1] == "detect" and len(sys.argv) > 3:
+ # Detect bubbles in an image
+ model_path = sys.argv[2]
+ image_path = sys.argv[3]
+
+ # Load appropriate model
+ if 'rtdetr' in model_path.lower():
+ if detector.load_rtdetr_model():
+ # Use RT-DETR
+ results = detector.detect_with_rtdetr(image_path)
+ print(f"RT-DETR Detection:")
+ print(f" Empty bubbles: {len(results['bubbles'])}")
+ print(f" Text bubbles: {len(results['text_bubbles'])}")
+ print(f" Free text: {len(results['text_free'])}")
+ else:
+ if detector.load_model(model_path):
+ bubbles = detector.detect_bubbles(image_path, confidence=0.5)
+ print(f"YOLOv8 detected {len(bubbles)} bubbles:")
+ for i, (x, y, w, h) in enumerate(bubbles):
+ print(f" Bubble {i+1}: position=({x},{y}) size=({w}x{h})")
+
+ # Optionally visualize
+ if len(sys.argv) > 4:
+ output_path = sys.argv[4]
+ detector.visualize_detections(image_path, output_path=output_path,
+ use_rtdetr='rtdetr' in model_path.lower())
+
+ elif sys.argv[1] == "test-both" and len(sys.argv) > 2:
+ # Test both models
+ image_path = sys.argv[2]
+
+ # Load YOLOv8
+ yolo_path = "models/comic-speech-bubble-detector-yolov8m.pt"
+ if os.path.exists(yolo_path):
+ detector.load_model(yolo_path)
+ yolo_bubbles = detector.detect_bubbles(image_path, use_rtdetr=False)
+ print(f"YOLOv8: {len(yolo_bubbles)} bubbles")
+
+ # Load RT-DETR
+ if detector.load_rtdetr_model():
+ rtdetr_bubbles = detector.detect_bubbles(image_path, use_rtdetr=True)
+ print(f"RT-DETR: {len(rtdetr_bubbles)} bubbles")
+
+ else:
+ print("Usage:")
+ print(" python bubble_detector.py download")
+ print(" python bubble_detector.py detect [output_path]")
+ print(" python bubble_detector.py test-both ")
+
+ else:
+ print("Bubble Detector Module (YOLOv8 + RT-DETR)")
+ print("Usage:")
+ print(" python bubble_detector.py download")
+ print(" python bubble_detector.py detect [output_path]")
+ print(" python bubble_detector.py test-both ")
diff --git a/chapter_extraction_manager.py b/chapter_extraction_manager.py
new file mode 100644
index 0000000000000000000000000000000000000000..f3cefa5cd83c34cd2081c9697c34378d0af799a7
--- /dev/null
+++ b/chapter_extraction_manager.py
@@ -0,0 +1,403 @@
+#!/usr/bin/env python3
+"""
+Chapter Extraction Manager - Manages chapter extraction in subprocess to prevent GUI freezing
+"""
+
+import subprocess
+import sys
+import os
+import json
+import threading
+import queue
+import time
+from pathlib import Path
+
+
+class ChapterExtractionManager:
+ """
+ Manages chapter extraction in a separate process to prevent GUI freezing
+ Similar to GlossaryManager but for chapter extraction
+ """
+
+ def __init__(self, log_callback=None):
+ """
+ Initialize the extraction manager
+
+ Args:
+ log_callback: Function to call with log messages (for GUI integration)
+ """
+ self.log_callback = log_callback
+ self.process = None
+ self.output_queue = queue.Queue()
+ self.error_queue = queue.Queue()
+ self.result = None
+ self.is_running = False
+ self.stop_requested = False
+
+ def extract_chapters_async(self, epub_path, output_dir, extraction_mode="smart",
+ progress_callback=None, completion_callback=None):
+ """
+ Start chapter extraction in a subprocess
+
+ Args:
+ epub_path: Path to EPUB file
+ output_dir: Output directory for extracted content
+ extraction_mode: Extraction mode (smart, comprehensive, full, enhanced)
+ progress_callback: Function to call with progress updates
+ completion_callback: Function to call when extraction completes
+ """
+ if self.is_running:
+ self._log("⚠️ Chapter extraction already in progress")
+ return False
+
+ self.is_running = True
+ self.stop_requested = False
+ self.result = None
+
+ # Start extraction in a thread that manages the subprocess
+ thread = threading.Thread(
+ target=self._run_extraction_subprocess,
+ args=(epub_path, output_dir, extraction_mode, progress_callback, completion_callback),
+ daemon=True
+ )
+ thread.start()
+
+ return True
+
+ def _run_extraction_subprocess(self, epub_path, output_dir, extraction_mode,
+ progress_callback, completion_callback):
+ """
+ Run the extraction subprocess and handle its output
+ """
+ try:
+ # Build command differently for frozen vs dev mode
+ if getattr(sys, 'frozen', False):
+ # In a frozen one-file build, sys.executable is our GUI .exe, not Python.
+ # Use an internal worker-mode flag handled by translator_gui.py to run the worker.
+ cmd = [
+ sys.executable,
+ '--run-chapter-extraction',
+ epub_path,
+ output_dir,
+ extraction_mode
+ ]
+ else:
+ # In dev mode, invoke the worker script with the Python interpreter
+ base_dir = Path(__file__).parent
+ worker_script = base_dir / "chapter_extraction_worker.py"
+ cmd = [
+ sys.executable,
+ str(worker_script),
+ epub_path,
+ output_dir,
+ extraction_mode
+ ]
+
+ # Set environment to force UTF-8 encoding
+ env = os.environ.copy()
+ env['PYTHONIOENCODING'] = 'utf-8'
+ env['PYTHONLEGACYWINDOWSSTDIO'] = '0' # Use new Windows console API
+
+ self._log(f"🚀 Starting chapter extraction subprocess...")
+ self._log(f"📚 EPUB: {os.path.basename(epub_path)}")
+ self._log(f"📂 Output: {output_dir}")
+ self._log(f"⚙️ Mode: {extraction_mode}")
+
+ # Start the subprocess with UTF-8 encoding
+ self.process = subprocess.Popen(
+ cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ encoding='utf-8',
+ errors='replace', # Replace invalid chars instead of failing
+ bufsize=1,
+ universal_newlines=True,
+ env=env # Pass the environment with UTF-8 settings
+ )
+
+ # Read output in real-time
+ while True:
+ if self.stop_requested:
+ self._terminate_process()
+ break
+
+ # Check if process is still running
+ if self.process.poll() is not None:
+ break
+
+ # Read stdout line by line with error handling
+ try:
+ line = self.process.stdout.readline()
+ if not line:
+ continue
+
+ line = line.strip()
+ if not line:
+ continue
+ except UnicodeDecodeError as e:
+ self._log(f"⚠️ Encoding error reading output: {e}")
+ continue
+
+ # Skip all processing if stop is requested to suppress logs
+ if self.stop_requested:
+ continue
+
+ # Parse output based on prefix
+ if line.startswith("[PROGRESS]"):
+ # Progress update
+ message = line[10:].strip()
+ if progress_callback:
+ progress_callback(message)
+ self._log(f"📊 {message}")
+
+ elif line.startswith("[INFO]"):
+ # Information message
+ message = line[6:].strip()
+ self._log(f"ℹ️ {message}")
+
+ elif line.startswith("[ERROR]"):
+ # Error message
+ message = line[7:].strip()
+ self._log(f"❌ {message}")
+ self.error_queue.put(message)
+
+ elif line.startswith("[RESULT]"):
+ # Final result as JSON
+ try:
+ json_str = line[8:].strip()
+ self.result = json.loads(json_str)
+
+ if self.result.get("success"):
+ self._log(f"✅ Extraction completed successfully!")
+ self._log(f"📚 Extracted {self.result.get('chapters', 0)} chapters")
+ else:
+ error = self.result.get("error", "Unknown error")
+ self._log(f"❌ Extraction failed: {error}")
+
+ except json.JSONDecodeError as e:
+ self._log(f"⚠️ Failed to parse result: {e}")
+
+ elif line.startswith("["):
+ # Other prefixed messages - skip
+ pass
+ else:
+ # Regular output - only log if not too verbose
+ if not any(skip in line for skip in ["📁 Searching for", "📁 Found", "📁 ✓", "📁 ✗"]):
+ self._log(line)
+
+ # Get any remaining output - but only process if not stopped
+ if not self.stop_requested:
+ remaining_output, remaining_error = self.process.communicate(timeout=1)
+
+ # Process any remaining output
+ if remaining_output:
+ for line in remaining_output.strip().split('\n'):
+ if line and not line.startswith("["):
+ self._log(line)
+
+ # Check for errors
+ if remaining_error:
+ for line in remaining_error.strip().split('\n'):
+ if line:
+ self._log(f"⚠️ {line}")
+
+ # Check final status
+ if self.process.returncode != 0:
+ self._log(f"⚠️ Process exited with code {self.process.returncode}")
+ else:
+ # If stopped, just clean up without processing output
+ try:
+ self.process.communicate(timeout=0.1)
+ except subprocess.TimeoutExpired:
+ pass # Ignore timeout when cleaning up
+
+ except subprocess.TimeoutExpired:
+ if not self.stop_requested:
+ self._log("⚠️ Subprocess communication timeout")
+ self._terminate_process()
+
+ except Exception as e:
+ # Only log errors if not stopping (unless it's a critical error)
+ if not self.stop_requested or "Subprocess error" in str(e):
+ self._log(f"❌ Subprocess error: {e}")
+ self.result = {
+ "success": False,
+ "error": str(e) if not self.stop_requested else "Extraction stopped by user"
+ }
+
+ finally:
+ self.is_running = False
+ # Store process reference before clearing it in case termination is needed
+ process_ref = self.process
+ self.process = None
+
+ # If process is still running, try to clean it up
+ if process_ref and process_ref.poll() is None:
+ try:
+ process_ref.terminate()
+ time.sleep(0.1) # Brief wait
+ if process_ref.poll() is None:
+ process_ref.kill()
+ except Exception:
+ pass # Ignore cleanup errors in finally block
+
+ # Ensure result is never None
+ if self.result is None:
+ if self.stop_requested:
+ self.result = {
+ "success": False,
+ "error": "Extraction stopped by user"
+ }
+ else:
+ self.result = {
+ "success": False,
+ "error": "Extraction process ended unexpectedly"
+ }
+
+ # Call completion callback
+ if completion_callback:
+ completion_callback(self.result)
+
+ def stop_extraction(self):
+ """Stop the extraction process"""
+ if not self.is_running:
+ return False
+
+ # Set stop flag first to suppress subsequent logs
+ self.stop_requested = True
+ self._log("🛑 Stopping chapter extraction...")
+
+ # Store process reference to avoid race condition
+ process_ref = self.process
+
+ # Give it a moment to stop gracefully
+ time.sleep(0.5)
+
+ # Force terminate if still running and process still exists
+ if process_ref:
+ self._terminate_process_ref(process_ref)
+
+ return True
+
+ def _terminate_process(self):
+ """Terminate the subprocess using current process reference"""
+ if self.process:
+ self._terminate_process_ref(self.process)
+
+ def _terminate_process_ref(self, process_ref):
+ """Terminate a specific process reference"""
+ if not process_ref:
+ return
+
+ try:
+ # Check if process is still alive before attempting termination
+ if process_ref.poll() is None:
+ process_ref.terminate()
+ # Give it a moment to terminate
+ time.sleep(0.5)
+
+ # Force kill if still running
+ if process_ref.poll() is None:
+ process_ref.kill()
+ time.sleep(0.1) # Brief wait after kill
+
+ # Only log termination if not stopping (user already knows they stopped it)
+ if not self.stop_requested:
+ self._log("✅ Process terminated")
+ else:
+ # Only log if not stopping
+ if not self.stop_requested:
+ self._log("✅ Process already terminated")
+ except Exception as e:
+ # Always log termination errors as they might indicate a problem
+ self._log(f"⚠️ Error terminating process: {e}")
+
+ def _log(self, message):
+ """Log a message using the callback if available"""
+ # Suppress logs when stop is requested (except for stop/termination messages)
+ if self.stop_requested and not any(keyword in message for keyword in ["🛑", "✅ Process terminated", "❌ Subprocess error"]):
+ return
+
+ if self.log_callback:
+ self.log_callback(message)
+ else:
+ print(message)
+
+ def is_extraction_running(self):
+ """Check if extraction is currently running"""
+ return self.is_running
+
+ def get_result(self):
+ """Get the extraction result if available"""
+ return self.result
+
+
+# Example usage
+if __name__ == "__main__":
+ import tkinter as tk
+ from tkinter import filedialog
+
+ def test_extraction():
+ """Test the extraction manager"""
+
+ # Create a simple GUI for testing
+ root = tk.Tk()
+ root.title("Chapter Extraction Test")
+ root.geometry("800x600")
+
+ # Text widget for logs
+ text = tk.Text(root, wrap=tk.WORD)
+ text.pack(fill=tk.BOTH, expand=True, padx=10, pady=10)
+
+ # Log callback
+ def log_message(msg):
+ text.insert(tk.END, msg + "\n")
+ text.see(tk.END)
+ root.update_idletasks()
+
+ # Progress callback
+ def progress_update(msg):
+ log_message(f"📊 Progress: {msg}")
+
+ # Completion callback
+ def extraction_complete(result):
+ if result and result.get("success"):
+ log_message(f"✅ Extraction completed!")
+ log_message(f" Chapters: {result.get('chapters', 0)}")
+ else:
+ log_message(f"❌ Extraction failed!")
+
+ # Create manager
+ manager = ChapterExtractionManager(log_callback=log_message)
+
+ # File selection
+ epub_path = filedialog.askopenfilename(
+ title="Select EPUB file",
+ filetypes=[("EPUB files", "*.epub"), ("All files", "*.*")]
+ )
+
+ if epub_path:
+ output_dir = os.path.splitext(os.path.basename(epub_path))[0]
+
+ # Start extraction
+ manager.extract_chapters_async(
+ epub_path,
+ output_dir,
+ extraction_mode="smart",
+ progress_callback=progress_update,
+ completion_callback=extraction_complete
+ )
+
+ # Button to stop
+ stop_btn = tk.Button(
+ root,
+ text="Stop Extraction",
+ command=lambda: manager.stop_extraction()
+ )
+ stop_btn.pack(pady=5)
+
+ root.mainloop()
+
+ # Run test
+ test_extraction()
\ No newline at end of file
diff --git a/chapter_extraction_worker.py b/chapter_extraction_worker.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2b0ecfebe96e3d26a62db65a4969570a683e9dc
--- /dev/null
+++ b/chapter_extraction_worker.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+"""
+Chapter Extraction Worker - Runs chapter extraction in a separate process to prevent GUI freezing
+"""
+
+import sys
+import os
+import io
+
+# Force UTF-8 encoding for stdout/stderr on Windows
+sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
+sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace')
+import json
+import zipfile
+import time
+import traceback
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+def run_chapter_extraction(epub_path, output_dir, extraction_mode="smart", progress_callback=None):
+ """
+ Run chapter extraction in this worker process
+
+ Args:
+ epub_path: Path to EPUB file
+ output_dir: Output directory for extracted content
+ extraction_mode: Extraction mode (smart, comprehensive, full, enhanced)
+ progress_callback: Callback function for progress updates (uses print for IPC)
+
+ Returns:
+ dict: Extraction results including chapters and metadata
+ """
+ try:
+ # Import here to avoid loading heavy modules until needed
+ from TransateKRtoEN import ChapterExtractor
+
+ # Create progress callback that prints to stdout for IPC
+ def worker_progress_callback(message):
+ # Use special prefix for progress messages
+ print(f"[PROGRESS] {message}", flush=True)
+
+ # Create extractor with progress callback
+ extractor = ChapterExtractor(progress_callback=worker_progress_callback)
+
+ # Set extraction mode
+ os.environ["EXTRACTION_MODE"] = extraction_mode
+
+ # Open EPUB and extract chapters
+ print(f"[INFO] Starting extraction of: {epub_path}", flush=True)
+ print(f"[INFO] Output directory: {output_dir}", flush=True)
+ print(f"[INFO] Extraction mode: {extraction_mode}", flush=True)
+
+ with zipfile.ZipFile(epub_path, 'r') as zf:
+ # Extract metadata first
+ metadata = extractor._extract_epub_metadata(zf)
+ print(f"[INFO] Extracted metadata: {list(metadata.keys())}", flush=True)
+
+ # Extract chapters
+ chapters = extractor.extract_chapters(zf, output_dir)
+
+ print(f"[INFO] Extracted {len(chapters)} chapters", flush=True)
+
+ # The extract_chapters method already handles OPF sorting internally
+ # Just log if OPF was used
+ opf_path = os.path.join(output_dir, 'content.opf')
+ if os.path.exists(opf_path):
+ print(f"[INFO] OPF file available for chapter ordering", flush=True)
+
+ # CRITICAL: Save the full chapters with body content!
+ # This is what the main process needs to load
+ chapters_full_path = os.path.join(output_dir, "chapters_full.json")
+ try:
+ with open(chapters_full_path, 'w', encoding='utf-8') as f:
+ json.dump(chapters, f, ensure_ascii=False)
+ print(f"[INFO] Saved full chapters data to: {chapters_full_path}", flush=True)
+ except Exception as e:
+ print(f"[WARNING] Could not save full chapters: {e}", flush=True)
+ # Fall back to saving individual files
+ for chapter in chapters:
+ try:
+ chapter_file = f"chapter_{chapter['num']:04d}_{chapter.get('filename', 'content').replace('/', '_')}.html"
+ chapter_path = os.path.join(output_dir, chapter_file)
+ with open(chapter_path, 'w', encoding='utf-8') as f:
+ f.write(chapter.get('body', ''))
+ print(f"[INFO] Saved chapter {chapter['num']} to {chapter_file}", flush=True)
+ except Exception as ce:
+ print(f"[WARNING] Could not save chapter {chapter.get('num')}: {ce}", flush=True)
+
+ # Return results as JSON for IPC
+ result = {
+ "success": True,
+ "chapters": len(chapters),
+ "metadata": metadata,
+ "chapter_info": [
+ {
+ "num": ch.get("num"),
+ "title": ch.get("title"),
+ "has_images": ch.get("has_images", False),
+ "file_size": ch.get("file_size", 0),
+ "content_hash": ch.get("content_hash", "")
+ }
+ for ch in chapters
+ ]
+ }
+
+ # Output result as JSON
+ print(f"[RESULT] {json.dumps(result)}", flush=True)
+ return result
+
+ except Exception as e:
+ # Send error information
+ error_info = {
+ "success": False,
+ "error": str(e),
+ "traceback": traceback.format_exc()
+ }
+ print(f"[ERROR] {str(e)}", flush=True)
+ print(f"[RESULT] {json.dumps(error_info)}", flush=True)
+ return error_info
+
+
+def main():
+ """Main entry point for worker process"""
+
+ # Parse command line arguments
+ if len(sys.argv) < 3:
+ print("[ERROR] Usage: chapter_extraction_worker.py [extraction_mode]", flush=True)
+ sys.exit(1)
+
+ epub_path = sys.argv[1]
+ output_dir = sys.argv[2]
+ extraction_mode = sys.argv[3] if len(sys.argv) > 3 else "smart"
+
+ # Validate inputs
+ if not os.path.exists(epub_path):
+ print(f"[ERROR] EPUB file not found: {epub_path}", flush=True)
+ sys.exit(1)
+
+ # Create output directory if needed
+ os.makedirs(output_dir, exist_ok=True)
+
+ # Run extraction
+ result = run_chapter_extraction(epub_path, output_dir, extraction_mode)
+
+ # Exit with appropriate code
+ sys.exit(0 if result.get("success", False) else 1)
+
+
+if __name__ == "__main__":
+ # Ensure freeze support for Windows frozen exe
+ try:
+ import multiprocessing
+ multiprocessing.freeze_support()
+ except Exception:
+ pass
+ main()
diff --git a/chapter_splitter.py b/chapter_splitter.py
new file mode 100644
index 0000000000000000000000000000000000000000..342330217c3f10a121e5478ea01d55001b735b63
--- /dev/null
+++ b/chapter_splitter.py
@@ -0,0 +1,195 @@
+import re
+from bs4 import BeautifulSoup
+import tiktoken
+
+class ChapterSplitter:
+ """Split large chapters into smaller chunks while preserving structure"""
+
+ def __init__(self, model_name="gpt-3.5-turbo", target_tokens=80000, compression_factor=1.0):
+ """
+ Initialize splitter with token counter
+ target_tokens: Target size for each chunk (leaving room for system prompt & history)
+ compression_factor: Expected compression ratio from source to target language (0.7-1.0)
+ """
+ try:
+ self.enc = tiktoken.encoding_for_model(model_name)
+ except:
+ self.enc = tiktoken.get_encoding("cl100k_base")
+ self.target_tokens = target_tokens
+ self.compression_factor = compression_factor
+
+ def count_tokens(self, text):
+ """Count tokens in text"""
+ try:
+ return len(self.enc.encode(text))
+ except:
+ # Fallback estimation
+ return len(text) // 4
+
+ def split_chapter(self, chapter_html, max_tokens=None):
+ """
+ Split a chapter into smaller chunks if it exceeds token limit
+ Returns: List of (chunk_html, chunk_index, total_chunks)
+ """
+ if max_tokens is None:
+ max_tokens = self.target_tokens
+
+ # Apply compression factor to output token limit
+ # If compression_factor is 0.7 and max_tokens is 4096,
+ # we expect output to be 4096 * 0.7 = 2867 tokens
+ effective_max_tokens = int(max_tokens * self.compression_factor)
+
+ # First check if splitting is needed
+ total_tokens = self.count_tokens(chapter_html)
+ if total_tokens <= effective_max_tokens:
+ return [(chapter_html, 1, 1)] # No split needed
+
+ # Parse HTML
+ soup = BeautifulSoup(chapter_html, 'html.parser')
+
+ # Try to find natural break points
+ chunks = []
+ current_chunk = []
+ current_tokens = 0
+
+ # Get all direct children of body, or all top-level elements
+ if soup.body:
+ elements = list(soup.body.children)
+ else:
+ elements = list(soup.children)
+
+ for element in elements:
+ if isinstance(element, str) and element.strip() == '':
+ continue
+
+ element_html = str(element)
+ element_tokens = self.count_tokens(element_html)
+
+ # If single element is too large, try to split it
+ if element_tokens > effective_max_tokens:
+ sub_chunks = self._split_large_element(element, effective_max_tokens)
+ for sub_chunk in sub_chunks:
+ chunks.append(sub_chunk)
+ else:
+ # Check if adding this element would exceed limit
+ if current_tokens + element_tokens > effective_max_tokens and current_chunk:
+ # Save current chunk
+ chunks.append(self._create_chunk_html(current_chunk))
+ current_chunk = [element_html]
+ current_tokens = element_tokens
+ else:
+ current_chunk.append(element_html)
+ current_tokens += element_tokens
+
+ # Don't forget the last chunk
+ if current_chunk:
+ chunks.append(self._create_chunk_html(current_chunk))
+
+ # Return chunks with metadata
+ total_chunks = len(chunks)
+ return [(chunk, i+1, total_chunks) for i, chunk in enumerate(chunks)]
+
+ def _split_large_element(self, element, max_tokens):
+ """Split a single large element (like a long paragraph)"""
+ chunks = []
+
+ if element.name == 'p' or not hasattr(element, 'children'):
+ # For paragraphs or text elements, split by sentences
+ text = element.get_text()
+ sentences = re.split(r'(?<=[.!?])\s+', text)
+
+ current_chunk = []
+ current_tokens = 0
+
+ for sentence in sentences:
+ sentence_tokens = self.count_tokens(sentence)
+
+ if current_tokens + sentence_tokens > max_tokens * 0.8 and current_chunk:
+ # Create paragraph with current sentences
+ chunk_text = ' '.join(current_chunk)
+ chunks.append(f"{chunk_text}
")
+ current_chunk = [sentence]
+ current_tokens = sentence_tokens
+ else:
+ current_chunk.append(sentence)
+ current_tokens += sentence_tokens
+
+ if current_chunk:
+ chunk_text = ' '.join(current_chunk)
+ chunks.append(f"{chunk_text}
")
+
+ else:
+ # For other elements, try to split by children
+ children = list(element.children)
+ current_chunk = []
+ current_tokens = 0
+
+ for child in children:
+ child_html = str(child)
+ child_tokens = self.count_tokens(child_html)
+
+ if current_tokens + child_tokens > max_tokens * 0.8 and current_chunk:
+ # Wrap in parent element type
+ wrapper = BeautifulSoup(f"<{element.name}>", 'html.parser')
+ wrapper_elem = wrapper.find(element.name)
+ for item in current_chunk:
+ wrapper_elem.append(BeautifulSoup(item, 'html.parser'))
+ chunks.append(str(wrapper))
+
+ current_chunk = [child_html]
+ current_tokens = child_tokens
+ else:
+ current_chunk.append(child_html)
+ current_tokens += child_tokens
+
+ if current_chunk:
+ wrapper = BeautifulSoup(f"<{element.name}>", 'html.parser')
+ wrapper_elem = wrapper.find(element.name)
+ for item in current_chunk:
+ wrapper_elem.append(BeautifulSoup(item, 'html.parser'))
+ chunks.append(str(wrapper))
+
+ return chunks
+
+ def _create_chunk_html(self, elements):
+ """Create a valid HTML chunk from list of elements"""
+ # Join elements and wrap in basic HTML structure if needed
+ content = '\n'.join(elements)
+
+ # Check if it already has body tags
+ if ' 1:
+ # Keep first body, move all content from others into it
+ main_body = bodies[0]
+ for extra_body in bodies[1:]:
+ for child in list(extra_body.children):
+ main_body.append(child)
+ extra_body.decompose()
+
+ return str(soup)
+
+ return merged
diff --git a/check_epub_directory.py b/check_epub_directory.py
new file mode 100644
index 0000000000000000000000000000000000000000..907074ed108151562ec7910a307a095ff11cab43
--- /dev/null
+++ b/check_epub_directory.py
@@ -0,0 +1,152 @@
+import os
+import re
+
+def diagnose_epub_directory(directory="."):
+ """Diagnose issues with EPUB output directory"""
+
+ print(f"\n{'='*60}")
+ print(f"EPUB Directory Diagnostic Tool")
+ print(f"{'='*60}\n")
+
+ # Get absolute path
+ abs_path = os.path.abspath(directory)
+ print(f"📁 Checking directory: {abs_path}")
+
+ # Check if directory exists
+ if not os.path.exists(abs_path):
+ print(f"❌ ERROR: Directory does not exist!")
+ return
+
+ if not os.path.isdir(abs_path):
+ print(f"❌ ERROR: Path is not a directory!")
+ return
+
+ # List contents
+ try:
+ contents = os.listdir(abs_path)
+ print(f"✅ Directory is accessible")
+ print(f"📊 Total items: {len(contents)}\n")
+ except Exception as e:
+ print(f"❌ ERROR: Cannot read directory: {e}")
+ return
+
+ # Categorize files
+ html_files = []
+ response_files = []
+ css_files = []
+ image_files = []
+ directories = []
+ other_files = []
+
+ for item in contents:
+ item_path = os.path.join(abs_path, item)
+
+ if os.path.isdir(item_path):
+ directories.append(item)
+ elif item.endswith('.html'):
+ html_files.append(item)
+ if item.startswith('response_'):
+ response_files.append(item)
+ elif item.endswith('.css'):
+ css_files.append(item)
+ elif item.lower().endswith(('.jpg', '.jpeg', '.png', '.gif', '.svg')):
+ image_files.append(item)
+ else:
+ other_files.append(item)
+
+ # Report findings
+ print("📋 Directory Contents Summary:")
+ print(f" • HTML files: {len(html_files)}")
+ print(f" • Response files (translated chapters): {len(response_files)}")
+ print(f" • CSS files: {len(css_files)}")
+ print(f" • Image files: {len(image_files)}")
+ print(f" • Subdirectories: {len(directories)}")
+ print(f" • Other files: {len(other_files)}")
+
+ # Check for required items
+ print(f"\n📍 Checking Required Items:")
+
+ # Check for metadata.json
+ if 'metadata.json' in contents:
+ print(" ✅ metadata.json found")
+ else:
+ print(" ❌ metadata.json NOT FOUND")
+
+ # Check for response files
+ if response_files:
+ print(f" ✅ {len(response_files)} translated chapter files found")
+
+ # Analyze chapter numbers
+ chapter_nums = []
+ for f in response_files:
+ m = re.match(r'response_(\d+)_', f)
+ if m:
+ chapter_nums.append(int(m.group(1)))
+
+ if chapter_nums:
+ chapter_nums.sort()
+ print(f" 📖 Chapter range: {min(chapter_nums)} to {max(chapter_nums)}")
+
+ # Check for missing chapters
+ expected = set(range(min(chapter_nums), max(chapter_nums) + 1))
+ actual = set(chapter_nums)
+ missing = expected - actual
+ if missing:
+ print(f" ⚠️ Missing chapters: {sorted(missing)}")
+ else:
+ print(" ❌ No response_*.html files found!")
+
+ if html_files:
+ print(f"\n 🔍 Found {len(html_files)} HTML files with different names:")
+ for i, f in enumerate(html_files[:5]):
+ print(f" {i+1}. {f}")
+ if len(html_files) > 5:
+ print(f" ... and {len(html_files) - 5} more")
+
+ # Check subdirectories
+ if directories:
+ print(f"\n📂 Subdirectories found:")
+ for d in directories:
+ print(f" • {d}/")
+
+ # Check contents of important subdirectories
+ if d in ['css', 'images', 'fonts']:
+ try:
+ sub_contents = os.listdir(os.path.join(abs_path, d))
+ print(f" Contains {len(sub_contents)} items")
+ except:
+ print(f" Cannot read contents")
+
+ # Sample file check
+ if response_files:
+ print(f"\n🔍 Checking a sample chapter file...")
+ sample_file = response_files[0]
+ sample_path = os.path.join(abs_path, sample_file)
+
+ try:
+ with open(sample_path, 'r', encoding='utf-8') as f:
+ content = f.read()
+ print(f" ✅ {sample_file} is readable")
+ print(f" 📏 File size: {len(content):,} characters")
+
+ # Check for basic HTML structure
+ if '' in content or ' 1:
+ diagnose_epub_directory(sys.argv[1])
+ else:
+ diagnose_epub_directory(".")
\ No newline at end of file
diff --git a/direct_imports.py b/direct_imports.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae1202fd39a7913499a6a056c629b50808b46302
--- /dev/null
+++ b/direct_imports.py
@@ -0,0 +1,38 @@
+import sys
+import os
+
+# Add the current directory to Python path so we can import our modules
+current_dir = os.path.dirname(os.path.abspath(__file__))
+if current_dir not in sys.path:
+ sys.path.insert(0, current_dir)
+
+# When running as executable, modules might be in _MEIPASS
+if hasattr(sys, '_MEIPASS'):
+ meipass_dir = sys._MEIPASS
+ if meipass_dir not in sys.path:
+ sys.path.insert(0, meipass_dir)
+
+# Now we can safely import our modules
+try:
+ from extract_glossary_from_epub import main as glossary_main
+except ImportError as e:
+ print(f"Failed to import glossary module: {e}")
+ glossary_main = None
+
+try:
+ from TransateKRtoEN import main as translation_main
+except ImportError as e:
+ print(f"Failed to import translation module: {e}")
+ translation_main = None
+
+try:
+ from epub_converter import fallback_compile_epub
+except ImportError as e:
+ print(f"Failed to import epub converter: {e}")
+ fallback_compile_epub = None
+
+try:
+ from scan_html_folder import scan_html_folder
+except ImportError as e:
+ print(f"Failed to import scanner: {e}")
+ scan_html_folder = None
diff --git a/enhanced_text_extractor.py b/enhanced_text_extractor.py
new file mode 100644
index 0000000000000000000000000000000000000000..31c02bdb17619366b59bf2fb0548a84d088ee7cf
--- /dev/null
+++ b/enhanced_text_extractor.py
@@ -0,0 +1,597 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Enhanced Text Extractor Module with CJK Support
+Provides superior text extraction from HTML with proper Unicode handling
+Optimized for Korean, Japanese, and Chinese content extraction
+"""
+
+import os
+import re
+import html
+import unicodedata
+from typing import Tuple, Optional
+import chardet
+
+# BEAUTIFUL SOUP IMPORT MONKEY FIX - Import BeautifulSoup BEFORE html2text
+# This prevents certain parser initialization issues
+try:
+ from bs4 import BeautifulSoup
+ # Force BeautifulSoup to initialize its parsers
+ _ = BeautifulSoup("", 'html.parser')
+except ImportError:
+ BeautifulSoup = None
+ raise ImportError("BeautifulSoup is required. Install with: pip install beautifulsoup4")
+
+# Now import html2text AFTER BeautifulSoup
+try:
+ import html2text
+except ImportError:
+ html2text = None
+ raise ImportError("html2text is required. Install with: pip install html2text")
+
+
+class EnhancedTextExtractor:
+ """Enhanced text extraction with proper Unicode and CJK handling"""
+
+ # Unicode preservation mappings
+ UNICODE_QUOTES = {
+ # Western quotes
+ '“': '\u201c', # Left double quotation mark
+ '”': '\u201d', # Right double quotation mark
+ '‘': '\u2018', # Left single quotation mark
+ '’': '\u2019', # Right single quotation mark
+ '"': '"', # Standard double quote
+ ''': "'", # Standard apostrophe
+
+ # CJK quotes and punctuation
+ '「': '「', # Japanese left corner bracket
+ '」': '」', # Japanese right corner bracket
+ '『': '『', # Japanese left white corner bracket
+ '』': '』', # Japanese right white corner bracket
+ '（': '（', # Fullwidth left parenthesis
+ '）': '）', # Fullwidth right parenthesis
+ '【': '【', # Left black lenticular bracket
+ '】': '】', # Right black lenticular bracket
+ '《': '《', # Left double angle bracket
+ '》': '》', # Right double angle bracket
+ '；': '；', # Fullwidth semicolon
+ '：': '：', # Fullwidth colon
+ '。': '。', # Ideographic full stop
+ '？': '？', # Fullwidth question mark
+ '！': '！', # Fullwidth exclamation mark
+ '、': '、', # Ideographic comma
+
+ # Numeric entities
+ '“': '\u201c', # Left double quote (numeric)
+ '”': '\u201d', # Right double quote (numeric)
+ '‘': '\u2018', # Left single quote (numeric)
+ '’': '\u2019', # Right single quote (numeric)
+
+ # Common CJK entities
+ '…': '…', # Horizontal ellipsis
+ '—': '—', # Em dash
+ '–': '–', # En dash
+ ' ': '\u00A0', # Non-breaking space
+ }
+
+ # CJK-specific punctuation to preserve
+ CJK_PUNCTUATION = {
+ '。', '、', '！', '？', '…', '—', '～', '・',
+ '「', '」', '『', '』', '（', '）', '【', '】',
+ '《', '》', '〈', '〉', '〔', '〕', '［', '］',
+ '：', '；', '"', '"', ''', ''',
+ '，', '．', '？', '！', '：', '；',
+ '"', '"', '‚', '„', '«', '»',
+ }
+
+ # Quote protection markers
+ QUOTE_MARKERS = {
+ '"': '␥', # Opening double quote marker
+ '"': '␦', # Closing double quote marker
+ '"': '␦', # Alternative closing quote
+ "'": '␣', # Opening single quote marker
+ "'": '', # Closing single quote marker
+ "'": '', # Alternative closing quote
+ }
+
+
+ def __init__(self, filtering_mode: str = "smart", preserve_structure: bool = True):
+ """Initialize the enhanced text extractor"""
+ if not html2text:
+ raise ImportError("html2text is required for enhanced extraction")
+
+ if not BeautifulSoup:
+ raise ImportError("BeautifulSoup is required for enhanced extraction")
+
+ self.filtering_mode = filtering_mode
+ self.preserve_structure = preserve_structure
+ self.h2t = None
+ self.detected_language = None
+
+ self._configure_html2text()
+
+ def _detect_encoding(self, content: bytes) -> str:
+ """Detect the encoding of the content"""
+ try:
+ # Try chardet detection
+ detected = chardet.detect(content)
+ if detected['confidence'] > 0.7:
+ return detected['encoding']
+ except Exception:
+ pass
+
+ # Try common CJK encodings in order
+ for encoding in ['utf-8', 'gb2312', 'gbk', 'gb18030', 'big5', 'shift_jis', 'euc-kr', 'euc-jp']:
+ try:
+ content.decode(encoding)
+ return encoding
+ except Exception:
+ continue
+
+ return 'utf-8' # Default fallback
+
+ def _detect_content_language(self, text: str) -> str:
+ """Detect the primary language of content"""
+ if not text:
+ return 'unknown'
+
+ # Take a sample of the text
+ sample = text[:5000]
+
+ # Count characters by script
+ korean_chars = sum(1 for char in sample if 0xAC00 <= ord(char) <= 0xD7AF)
+ japanese_kana = sum(1 for char in sample if (0x3040 <= ord(char) <= 0x309F) or (0x30A0 <= ord(char) <= 0x30FF))
+ chinese_chars = sum(1 for char in sample if 0x4E00 <= ord(char) <= 0x9FFF)
+ latin_chars = sum(1 for char in sample if 0x0041 <= ord(char) <= 0x007A)
+
+ # Determine primary language
+ if korean_chars > 50:
+ return 'korean'
+ elif japanese_kana > 20:
+ return 'japanese'
+ elif chinese_chars > 50 and japanese_kana < 10:
+ return 'chinese'
+ elif latin_chars > 100:
+ return 'english'
+ else:
+ return 'unknown'
+
+ def _configure_html2text(self):
+ """Configure html2text with optimal Unicode and CJK settings"""
+ self.h2t = html2text.HTML2Text()
+
+ # Core settings for Unicode preservation
+ self.h2t.unicode_snob = True
+ self.h2t.escape_snob = True
+ self.h2t.use_automatic_links = False
+
+ # Layout settings
+ self.h2t.body_width = 0
+ self.h2t.single_line_break = False
+
+ # Content filtering
+ self.h2t.ignore_links = False
+ self.h2t.ignore_images = False
+ self.h2t.ignore_anchors = False
+ self.h2t.skip_internal_links = False
+ self.h2t.ignore_tables = False
+
+ # Image handling - CRITICAL: Force html2text to preserve img tags as HTML
+ self.h2t.images_as_html = True # Keep images as ![]()
tags instead of ![]()
+ self.h2t.images_to_alt = False # Don't convert to alt text only
+ self.h2t.images_with_size = True # Include width/height attributes
+
+ # Additional settings
+ self.h2t.wrap_links = False
+ self.h2t.wrap_list_items = False
+ self.h2t.protect_links = True
+
+ # Structure preservation settings
+ if self.preserve_structure:
+ self.h2t.bypass_tables = False
+ self.h2t.ignore_emphasis = False
+ self.h2t.mark_code = True
+ self.h2t.ul_item_mark = '•'
+ else:
+ self.h2t.bypass_tables = True
+ self.h2t.ignore_emphasis = True
+ self.h2t.mark_code = False
+
+ def _decode_entities(self, text: str) -> str:
+ """Decode HTML entities to Unicode characters with CJK support"""
+ if not text:
+ return text
+
+ # First pass: Apply known CJK-aware replacements
+ for entity, unicode_char in self.UNICODE_QUOTES.items():
+ text = text.replace(entity, unicode_char)
+
+ # Second pass: standard HTML unescape
+ text = html.unescape(text)
+
+ # Third pass: handle numeric entities
+ def decode_decimal(match):
+ try:
+ code = int(match.group(1))
+ if code < 0x110000:
+ return chr(code)
+ except Exception:
+ pass
+ return match.group(0)
+
+ def decode_hex(match):
+ try:
+ code = int(match.group(1), 16)
+ if code < 0x110000:
+ return chr(code)
+ except Exception:
+ pass
+ return match.group(0)
+
+ text = re.sub(r'&#(\d+);?', decode_decimal, text)
+ text = re.sub(r'&#x([0-9a-fA-F]+);?', decode_hex, text)
+
+ # Fourth pass: handle special CJK entities
+ cjk_special_entities = {
+ '⟨': '〈', '⟩': '〉',
+ '⌈': '⌈', '⌉': '⌉',
+ '⌊': '⌊', '⌋': '⌋',
+ }
+
+ for entity, char in cjk_special_entities.items():
+ text = text.replace(entity, char)
+
+ return text
+
+ def _normalize_unicode(self, text: str) -> str:
+ """Normalize Unicode with CJK awareness"""
+ if self.detected_language in ['korean', 'japanese', 'chinese']:
+ return text
+ else:
+ return unicodedata.normalize('NFC', text)
+
+ def _protect_quotes(self, text: str) -> str:
+ """Protect quotes by replacing with special markers"""
+ for original, marker in self.QUOTE_MARKERS.items():
+ text = text.replace(original, marker)
+ return text
+
+ def _restore_quotes(self, text: str) -> str:
+ """Restore quotes from special markers"""
+ for original, marker in self.QUOTE_MARKERS.items():
+ text = text.replace(marker, original)
+ return text
+
+
+
+ def _preprocess_html_for_quotes(self, html_content: str) -> str:
+ """Pre-process HTML to protect quotes from conversion"""
+ def protect_quotes_in_text(match):
+ text = match.group(1)
+ return f'>{self._protect_quotes(text)}<'
+
+ # Apply to all text between tags
+ html_content = re.sub(r'>([^<]+)<', protect_quotes_in_text, html_content)
+ return html_content
+
+ def _protect_quotes_in_soup(self, soup: BeautifulSoup) -> None:
+ """Protect quotes in BeautifulSoup object before processing"""
+ for element in soup.find_all(string=True):
+ if element.parent.name not in ['script', 'style', 'noscript']:
+ original_text = str(element)
+ protected_text = self._protect_quotes(original_text)
+ element.replace_with(protected_text)
+
+ def _minimal_parser_fix(self, html_content: str) -> str:
+ """Apply minimal fixes only for parser errors"""
+ # Fix tags with ="" pattern
+ html_content = re.sub(r'<[^>]*?=\s*""\s*[^>]*?>', '', html_content)
+
+ # Fix malformed closing tags
+ html_content = re.sub(r'', r'', html_content)
+ html_content = re.sub(r'', '', html_content)
+ html_content = re.sub(r'', r'', html_content)
+
+ # Fix orphaned brackets
+ html_content = re.sub(r'<(?![a-zA-Z/!?])', '<', html_content)
+ html_content = re.sub(r'(?', '>', html_content)
+
+ # Fix unclosed tags at the end
+ if html_content.rstrip().endswith('<'):
+ html_content = html_content.rstrip()[:-1]
+
+ # Remove nested opening brackets
+ html_content = re.sub(r'<[^>]*?<[^>]*?>', '', html_content)
+
+ return html_content
+
+ def _clean_text_cjk_aware(self, text: str, preserve_structure: bool) -> str:
+ """Clean extracted text with CJK awareness"""
+ if not preserve_structure and self.detected_language not in ['korean', 'japanese', 'chinese']:
+ # Only do aggressive cleanup for non-CJK text
+ text = re.sub(r'^#+\s*', '', text, flags=re.MULTILINE)
+ text = re.sub(r'\*\*(.*?)\*\*', r'\1', text)
+ text = re.sub(r'\*(.*?)\*', r'\1', text)
+ text = re.sub(r'__(.*?)__', r'\1', text)
+ text = re.sub(r'_(.*?)_', r'\1', text)
+ text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', text)
+ text = re.sub(r'!\[([^\]]*)\]\([^)]+\)', '', text)
+ text = re.sub(r'`([^`]+)`', r'\1', text)
+ text = re.sub(r'```[^`]*```', '', text, flags=re.DOTALL)
+ text = re.sub(r'^[-*+]\s+', '', text, flags=re.MULTILINE)
+ text = re.sub(r'^\d+\.\s+', '', text, flags=re.MULTILINE)
+ text = re.sub(r'^>\s+', '', text, flags=re.MULTILINE)
+ text = re.sub(r'^[-_*]{3,}$', '', text, flags=re.MULTILINE)
+
+ # Clean whitespace
+ if self.detected_language in ['korean', 'japanese', 'chinese']:
+ text = re.sub(r'\n{3,}', '\n\n', text)
+ text = re.sub(r'[ ]{3,}', ' ', text)
+ else:
+ text = re.sub(r'\n{3,}', '\n\n', text)
+ text = re.sub(r' {2,}', ' ', text)
+
+ # Remove invisible characters
+ invisible_chars = ['\u200b', '\u200c', '\u200d', '\ufeff', '\u2060']
+ for char in invisible_chars:
+ text = text.replace(char, '')
+
+ return text.strip()
+
+ def _extract_title(self, soup: BeautifulSoup) -> Optional[str]:
+ """Extract chapter title from various sources"""
+ # Try title tag first
+ if soup.title and soup.title.string:
+ title = soup.title.string.strip()
+ title = self._decode_entities(title)
+ return title
+
+ # Try headers in order
+ for header_tag in ['h1', 'h2', 'h3', 'h4']:
+ headers = soup.find_all(header_tag)
+ for header in headers:
+ title = header.get_text(strip=True)
+ if title:
+ title = self._decode_entities(title)
+ if self._is_chapter_title(title):
+ return title
+
+ return None
+
+ def _is_chapter_title(self, text: str) -> bool:
+ """Check if text looks like a chapter title"""
+ if not text or len(text) > 200:
+ return False
+
+ # Common chapter patterns
+ patterns = [
+ r'第.{1,10}[章回話话]',
+ r'Chapter\s+\d+',
+ r'제\s*\d+\s*화',
+ r'第\d+話',
+ r'\d+\s*화',
+ r'EP\.?\s*\d+',
+ r'Part\s+\d+',
+ ]
+
+ for pattern in patterns:
+ if re.search(pattern, text, re.IGNORECASE):
+ return True
+
+ # Check if it's short and doesn't contain too much punctuation
+ if len(text) < 100:
+ punct_count = sum(1 for c in text if c in '.,;:!?。、！？')
+ if punct_count < len(text) * 0.2:
+ return True
+
+ return False
+
+ def _extract_body_content(self, soup: BeautifulSoup, full_html: str) -> str:
+ """Extract body content while preserving Unicode"""
+ # Remove script and style elements first
+ for element in soup(['script', 'style', 'noscript']):
+ element.decompose()
+
+ if soup.body:
+ return str(soup.body)
+ else:
+ return str(soup)
+
+ def extract_chapter_content(self, html_content: str, extraction_mode: str = None) -> Tuple[str, str, Optional[str]]:
+ """Extract chapter content with proper Unicode and CJK handling"""
+ try:
+ # Use instance filtering_mode if not overridden
+ if extraction_mode is None:
+ extraction_mode = self.filtering_mode
+
+ # Handle encoding if content is bytes
+ if isinstance(html_content, bytes):
+ encoding = self._detect_encoding(html_content)
+ html_content = html_content.decode(encoding, errors='replace')
+
+ # Pre-process HTML to protect quotes
+ html_content = self._preprocess_html_for_quotes(html_content)
+
+ # Pre-process HTML to decode all entities
+ html_content = self._decode_entities(html_content)
+
+ # Detect language early
+ self.detected_language = self._detect_content_language(html_content)
+ print(f"🌐 Detected language: {self.detected_language}")
+
+ # Parse with BeautifulSoup
+ parser = 'html.parser'
+ if self.detected_language in ['korean', 'japanese', 'chinese']:
+ # For CJK content, lxml might handle encoding better if available
+ try:
+ import lxml
+ parser = 'lxml'
+ except ImportError:
+ pass
+
+ soup = BeautifulSoup(html_content, parser)
+
+ # Protect quotes before any processing
+ self._protect_quotes_in_soup(soup)
+
+ # Extract title
+ chapter_title = self._extract_title(soup)
+
+ # Respect GUI toggles to exclude headers/titles BEFORE conversion
+ try:
+ batch_translate_active = os.getenv('BATCH_TRANSLATE_HEADERS', '0') == '1'
+ ignore_title_tag = os.getenv('IGNORE_TITLE', '0') == '1' and batch_translate_active
+ ignore_header_tags = os.getenv('IGNORE_HEADER', '0') == '1' and batch_translate_active
+ if ignore_title_tag and soup.title:
+ # Remove
so it isn't included when using full extraction
+ soup.title.decompose()
+ if ignore_header_tags:
+ # Remove visible headers from body prior to conversion
+ for tag_name in ['h1', 'h2', 'h3']:
+ for hdr in soup.find_all(tag_name):
+ hdr.decompose()
+ except Exception:
+ # Non-fatal – proceed with original soup if anything goes wrong
+ pass
+
+ # Determine content to convert (after removals)
+ if extraction_mode == "full":
+ content_to_convert = str(soup)
+ else:
+ content_to_convert = self._extract_body_content(soup, html_content)
+
+ # Convert using html2text
+ content_to_convert = self._decode_entities(content_to_convert)
+
+ # Convert to text with error handling
+ try:
+ clean_text = self.h2t.handle(content_to_convert)
+ except (AssertionError, UnboundLocalError) as e:
+ error_msg = str(e)
+ if "cannot access local variable" in error_msg or "we should not get here!" in error_msg or "unexpected call to parse_endtag" in error_msg or "unexpected call to parse_starttag" in error_msg:
+ print(f"⚠️ html2text encountered malformed HTML: {error_msg}")
+ print(f"⚠️ Applying minimal fixes...")
+ # Apply minimal fixes
+ content_to_convert = self._minimal_parser_fix(content_to_convert)
+ try:
+ clean_text = self.h2t.handle(content_to_convert)
+ print(f"✅ Successfully processed after minimal fixes")
+ except Exception as e2:
+ print(f"⚠️ html2text still failing: {e2}")
+ # Last resort fallback
+ clean_text = soup.get_text(separator='\n', strip=True)
+ print(f"✅ Used BeautifulSoup fallback")
+ else:
+ # Re-raise if it's a different error
+ raise
+ except Exception as e:
+ print(f"⚠️ Unexpected error in html2text: {e}")
+ # Fallback to BeautifulSoup
+ clean_text = soup.get_text(separator='\n', strip=True)
+
+ # Normalize only if appropriate
+ clean_text = self._normalize_unicode(clean_text)
+
+ # Clean based on settings and language
+ clean_text = self._clean_text_cjk_aware(clean_text, self.preserve_structure)
+
+ # Restore protected quotes
+ clean_text = self._restore_quotes(clean_text)
+
+ # For enhanced mode, both display and translation content are the same
+ return clean_text, clean_text, chapter_title
+
+ except Exception as e:
+ print(f"❌ Enhanced extraction failed: {e}")
+ raise
+
+
+# Test function
+def test_cjk_preservation():
+ """Test that CJK characters and quotes are properly preserved"""
+ test_cases = [
+ # Korean test with quotes
+ '''<html>
+ <head><title>제국의 붉은 사신
+
+ "왜 이러는 겁니까? 우리가 무슨 잘못을 했다고!"
+ "......"
+ "한 번만 살려주시오! 가족을 지키려면 어쩔 수 없었소!"
+ "응애! 응애! 응애!"
+ "미안하구나. 모든 죄는 내가 짊어지고 사마."
+
+
{chunk_text}
") + current_chunk = [sentence] + current_tokens = sentence_tokens + else: + current_chunk.append(sentence) + current_tokens += sentence_tokens + + if current_chunk: + chunk_text = ' '.join(current_chunk) + chunks.append(f"{chunk_text}
") + + else: + # For other elements, try to split by children + children = list(element.children) + current_chunk = [] + current_tokens = 0 + + for child in children: + child_html = str(child) + child_tokens = self.count_tokens(child_html) + + if current_tokens + child_tokens > max_tokens * 0.8 and current_chunk: + # Wrap in parent element type + wrapper = BeautifulSoup(f"<{element.name}>", 'html.parser') + wrapper_elem = wrapper.find(element.name) + for item in current_chunk: + wrapper_elem.append(BeautifulSoup(item, 'html.parser')) + chunks.append(str(wrapper)) + + current_chunk = [child_html] + current_tokens = child_tokens + else: + current_chunk.append(child_html) + current_tokens += child_tokens + + if current_chunk: + wrapper = BeautifulSoup(f"<{element.name}>", 'html.parser') + wrapper_elem = wrapper.find(element.name) + for item in current_chunk: + wrapper_elem.append(BeautifulSoup(item, 'html.parser')) + chunks.append(str(wrapper)) + + return chunks + + def _create_chunk_html(self, elements): + """Create a valid HTML chunk from list of elements""" + # Join elements and wrap in basic HTML structure if needed + content = '\n'.join(elements) + + # Check if it already has body tags + if ' 1:
+ diagnose_epub_directory(sys.argv[1])
+ else:
+ diagnose_epub_directory(".")
\ No newline at end of file
diff --git a/direct_imports.py b/direct_imports.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae1202fd39a7913499a6a056c629b50808b46302
--- /dev/null
+++ b/direct_imports.py
@@ -0,0 +1,38 @@
+import sys
+import os
+
+# Add the current directory to Python path so we can import our modules
+current_dir = os.path.dirname(os.path.abspath(__file__))
+if current_dir not in sys.path:
+ sys.path.insert(0, current_dir)
+
+# When running as executable, modules might be in _MEIPASS
+if hasattr(sys, '_MEIPASS'):
+ meipass_dir = sys._MEIPASS
+ if meipass_dir not in sys.path:
+ sys.path.insert(0, meipass_dir)
+
+# Now we can safely import our modules
+try:
+ from extract_glossary_from_epub import main as glossary_main
+except ImportError as e:
+ print(f"Failed to import glossary module: {e}")
+ glossary_main = None
+
+try:
+ from TransateKRtoEN import main as translation_main
+except ImportError as e:
+ print(f"Failed to import translation module: {e}")
+ translation_main = None
+
+try:
+ from epub_converter import fallback_compile_epub
+except ImportError as e:
+ print(f"Failed to import epub converter: {e}")
+ fallback_compile_epub = None
+
+try:
+ from scan_html_folder import scan_html_folder
+except ImportError as e:
+ print(f"Failed to import scanner: {e}")
+ scan_html_folder = None
diff --git a/enhanced_text_extractor.py b/enhanced_text_extractor.py
new file mode 100644
index 0000000000000000000000000000000000000000..31c02bdb17619366b59bf2fb0548a84d088ee7cf
--- /dev/null
+++ b/enhanced_text_extractor.py
@@ -0,0 +1,597 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Enhanced Text Extractor Module with CJK Support
+Provides superior text extraction from HTML with proper Unicode handling
+Optimized for Korean, Japanese, and Chinese content extraction
+"""
+
+import os
+import re
+import html
+import unicodedata
+from typing import Tuple, Optional
+import chardet
+
+# BEAUTIFUL SOUP IMPORT MONKEY FIX - Import BeautifulSoup BEFORE html2text
+# This prevents certain parser initialization issues
+try:
+ from bs4 import BeautifulSoup
+ # Force BeautifulSoup to initialize its parsers
+ _ = BeautifulSoup("", 'html.parser')
+except ImportError:
+ BeautifulSoup = None
+ raise ImportError("BeautifulSoup is required. Install with: pip install beautifulsoup4")
+
+# Now import html2text AFTER BeautifulSoup
+try:
+ import html2text
+except ImportError:
+ html2text = None
+ raise ImportError("html2text is required. Install with: pip install html2text")
+
+
+class EnhancedTextExtractor:
+ """Enhanced text extraction with proper Unicode and CJK handling"""
+
+ # Unicode preservation mappings
+ UNICODE_QUOTES = {
+ # Western quotes
+ '“': '\u201c', # Left double quotation mark
+ '”': '\u201d', # Right double quotation mark
+ '‘': '\u2018', # Left single quotation mark
+ '’': '\u2019', # Right single quotation mark
+ '"': '"', # Standard double quote
+ ''': "'", # Standard apostrophe
+
+ # CJK quotes and punctuation
+ '「': '「', # Japanese left corner bracket
+ '」': '」', # Japanese right corner bracket
+ '『': '『', # Japanese left white corner bracket
+ '』': '』', # Japanese right white corner bracket
+ '（': '（', # Fullwidth left parenthesis
+ '）': '）', # Fullwidth right parenthesis
+ '【': '【', # Left black lenticular bracket
+ '】': '】', # Right black lenticular bracket
+ '《': '《', # Left double angle bracket
+ '》': '》', # Right double angle bracket
+ '；': '；', # Fullwidth semicolon
+ '：': '：', # Fullwidth colon
+ '。': '。', # Ideographic full stop
+ '？': '？', # Fullwidth question mark
+ '！': '！', # Fullwidth exclamation mark
+ '、': '、', # Ideographic comma
+
+ # Numeric entities
+ '“': '\u201c', # Left double quote (numeric)
+ '”': '\u201d', # Right double quote (numeric)
+ '‘': '\u2018', # Left single quote (numeric)
+ '’': '\u2019', # Right single quote (numeric)
+
+ # Common CJK entities
+ '…': '…', # Horizontal ellipsis
+ '—': '—', # Em dash
+ '–': '–', # En dash
+ ' ': '\u00A0', # Non-breaking space
+ }
+
+ # CJK-specific punctuation to preserve
+ CJK_PUNCTUATION = {
+ '。', '、', '！', '？', '…', '—', '～', '・',
+ '「', '」', '『', '』', '（', '）', '【', '】',
+ '《', '》', '〈', '〉', '〔', '〕', '［', '］',
+ '：', '；', '"', '"', ''', ''',
+ '，', '．', '？', '！', '：', '；',
+ '"', '"', '‚', '„', '«', '»',
+ }
+
+ # Quote protection markers
+ QUOTE_MARKERS = {
+ '"': '␥', # Opening double quote marker
+ '"': '␦', # Closing double quote marker
+ '"': '␦', # Alternative closing quote
+ "'": '␣', # Opening single quote marker
+ "'": '', # Closing single quote marker
+ "'": '', # Alternative closing quote
+ }
+
+
+ def __init__(self, filtering_mode: str = "smart", preserve_structure: bool = True):
+ """Initialize the enhanced text extractor"""
+ if not html2text:
+ raise ImportError("html2text is required for enhanced extraction")
+
+ if not BeautifulSoup:
+ raise ImportError("BeautifulSoup is required for enhanced extraction")
+
+ self.filtering_mode = filtering_mode
+ self.preserve_structure = preserve_structure
+ self.h2t = None
+ self.detected_language = None
+
+ self._configure_html2text()
+
+ def _detect_encoding(self, content: bytes) -> str:
+ """Detect the encoding of the content"""
+ try:
+ # Try chardet detection
+ detected = chardet.detect(content)
+ if detected['confidence'] > 0.7:
+ return detected['encoding']
+ except Exception:
+ pass
+
+ # Try common CJK encodings in order
+ for encoding in ['utf-8', 'gb2312', 'gbk', 'gb18030', 'big5', 'shift_jis', 'euc-kr', 'euc-jp']:
+ try:
+ content.decode(encoding)
+ return encoding
+ except Exception:
+ continue
+
+ return 'utf-8' # Default fallback
+
+ def _detect_content_language(self, text: str) -> str:
+ """Detect the primary language of content"""
+ if not text:
+ return 'unknown'
+
+ # Take a sample of the text
+ sample = text[:5000]
+
+ # Count characters by script
+ korean_chars = sum(1 for char in sample if 0xAC00 <= ord(char) <= 0xD7AF)
+ japanese_kana = sum(1 for char in sample if (0x3040 <= ord(char) <= 0x309F) or (0x30A0 <= ord(char) <= 0x30FF))
+ chinese_chars = sum(1 for char in sample if 0x4E00 <= ord(char) <= 0x9FFF)
+ latin_chars = sum(1 for char in sample if 0x0041 <= ord(char) <= 0x007A)
+
+ # Determine primary language
+ if korean_chars > 50:
+ return 'korean'
+ elif japanese_kana > 20:
+ return 'japanese'
+ elif chinese_chars > 50 and japanese_kana < 10:
+ return 'chinese'
+ elif latin_chars > 100:
+ return 'english'
+ else:
+ return 'unknown'
+
+ def _configure_html2text(self):
+ """Configure html2text with optimal Unicode and CJK settings"""
+ self.h2t = html2text.HTML2Text()
+
+ # Core settings for Unicode preservation
+ self.h2t.unicode_snob = True
+ self.h2t.escape_snob = True
+ self.h2t.use_automatic_links = False
+
+ # Layout settings
+ self.h2t.body_width = 0
+ self.h2t.single_line_break = False
+
+ # Content filtering
+ self.h2t.ignore_links = False
+ self.h2t.ignore_images = False
+ self.h2t.ignore_anchors = False
+ self.h2t.skip_internal_links = False
+ self.h2t.ignore_tables = False
+
+ # Image handling - CRITICAL: Force html2text to preserve img tags as HTML
+ self.h2t.images_as_html = True # Keep images as tags instead of ![]()
+ self.h2t.images_to_alt = False # Don't convert to alt text only
+ self.h2t.images_with_size = True # Include width/height attributes
+
+ # Additional settings
+ self.h2t.wrap_links = False
+ self.h2t.wrap_list_items = False
+ self.h2t.protect_links = True
+
+ # Structure preservation settings
+ if self.preserve_structure:
+ self.h2t.bypass_tables = False
+ self.h2t.ignore_emphasis = False
+ self.h2t.mark_code = True
+ self.h2t.ul_item_mark = '•'
+ else:
+ self.h2t.bypass_tables = True
+ self.h2t.ignore_emphasis = True
+ self.h2t.mark_code = False
+
+ def _decode_entities(self, text: str) -> str:
+ """Decode HTML entities to Unicode characters with CJK support"""
+ if not text:
+ return text
+
+ # First pass: Apply known CJK-aware replacements
+ for entity, unicode_char in self.UNICODE_QUOTES.items():
+ text = text.replace(entity, unicode_char)
+
+ # Second pass: standard HTML unescape
+ text = html.unescape(text)
+
+ # Third pass: handle numeric entities
+ def decode_decimal(match):
+ try:
+ code = int(match.group(1))
+ if code < 0x110000:
+ return chr(code)
+ except Exception:
+ pass
+ return match.group(0)
+
+ def decode_hex(match):
+ try:
+ code = int(match.group(1), 16)
+ if code < 0x110000:
+ return chr(code)
+ except Exception:
+ pass
+ return match.group(0)
+
+ text = re.sub(r'&#(\d+);?', decode_decimal, text)
+ text = re.sub(r'&#x([0-9a-fA-F]+);?', decode_hex, text)
+
+ # Fourth pass: handle special CJK entities
+ cjk_special_entities = {
+ '⟨': '〈', '⟩': '〉',
+ '⌈': '⌈', '⌉': '⌉',
+ '⌊': '⌊', '⌋': '⌋',
+ }
+
+ for entity, char in cjk_special_entities.items():
+ text = text.replace(entity, char)
+
+ return text
+
+ def _normalize_unicode(self, text: str) -> str:
+ """Normalize Unicode with CJK awareness"""
+ if self.detected_language in ['korean', 'japanese', 'chinese']:
+ return text
+ else:
+ return unicodedata.normalize('NFC', text)
+
+ def _protect_quotes(self, text: str) -> str:
+ """Protect quotes by replacing with special markers"""
+ for original, marker in self.QUOTE_MARKERS.items():
+ text = text.replace(original, marker)
+ return text
+
+ def _restore_quotes(self, text: str) -> str:
+ """Restore quotes from special markers"""
+ for original, marker in self.QUOTE_MARKERS.items():
+ text = text.replace(marker, original)
+ return text
+
+
+
+ def _preprocess_html_for_quotes(self, html_content: str) -> str:
+ """Pre-process HTML to protect quotes from conversion"""
+ def protect_quotes_in_text(match):
+ text = match.group(1)
+ return f'>{self._protect_quotes(text)}<'
+
+ # Apply to all text between tags
+ html_content = re.sub(r'>([^<]+)<', protect_quotes_in_text, html_content)
+ return html_content
+
+ def _protect_quotes_in_soup(self, soup: BeautifulSoup) -> None:
+ """Protect quotes in BeautifulSoup object before processing"""
+ for element in soup.find_all(string=True):
+ if element.parent.name not in ['script', 'style', 'noscript']:
+ original_text = str(element)
+ protected_text = self._protect_quotes(original_text)
+ element.replace_with(protected_text)
+
+ def _minimal_parser_fix(self, html_content: str) -> str:
+ """Apply minimal fixes only for parser errors"""
+ # Fix tags with ="" pattern
+ html_content = re.sub(r'<[^>]*?=\s*""\s*[^>]*?>', '', html_content)
+
+ # Fix malformed closing tags
+ html_content = re.sub(r'', r'', html_content)
+ html_content = re.sub(r'', '', html_content)
+ html_content = re.sub(r'', r'', html_content)
+
+ # Fix orphaned brackets
+ html_content = re.sub(r'<(?![a-zA-Z/!?])', '<', html_content)
+ html_content = re.sub(r'(?', '>', html_content)
+
+ # Fix unclosed tags at the end
+ if html_content.rstrip().endswith('<'):
+ html_content = html_content.rstrip()[:-1]
+
+ # Remove nested opening brackets
+ html_content = re.sub(r'<[^>]*?<[^>]*?>', '', html_content)
+
+ return html_content
+
+ def _clean_text_cjk_aware(self, text: str, preserve_structure: bool) -> str:
+ """Clean extracted text with CJK awareness"""
+ if not preserve_structure and self.detected_language not in ['korean', 'japanese', 'chinese']:
+ # Only do aggressive cleanup for non-CJK text
+ text = re.sub(r'^#+\s*', '', text, flags=re.MULTILINE)
+ text = re.sub(r'\*\*(.*?)\*\*', r'\1', text)
+ text = re.sub(r'\*(.*?)\*', r'\1', text)
+ text = re.sub(r'__(.*?)__', r'\1', text)
+ text = re.sub(r'_(.*?)_', r'\1', text)
+ text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', text)
+ text = re.sub(r'!\[([^\]]*)\]\([^)]+\)', '', text)
+ text = re.sub(r'`([^`]+)`', r'\1', text)
+ text = re.sub(r'```[^`]*```', '', text, flags=re.DOTALL)
+ text = re.sub(r'^[-*+]\s+', '', text, flags=re.MULTILINE)
+ text = re.sub(r'^\d+\.\s+', '', text, flags=re.MULTILINE)
+ text = re.sub(r'^>\s+', '', text, flags=re.MULTILINE)
+ text = re.sub(r'^[-_*]{3,}$', '', text, flags=re.MULTILINE)
+
+ # Clean whitespace
+ if self.detected_language in ['korean', 'japanese', 'chinese']:
+ text = re.sub(r'\n{3,}', '\n\n', text)
+ text = re.sub(r'[ ]{3,}', ' ', text)
+ else:
+ text = re.sub(r'\n{3,}', '\n\n', text)
+ text = re.sub(r' {2,}', ' ', text)
+
+ # Remove invisible characters
+ invisible_chars = ['\u200b', '\u200c', '\u200d', '\ufeff', '\u2060']
+ for char in invisible_chars:
+ text = text.replace(char, '')
+
+ return text.strip()
+
+ def _extract_title(self, soup: BeautifulSoup) -> Optional[str]:
+ """Extract chapter title from various sources"""
+ # Try title tag first
+ if soup.title and soup.title.string:
+ title = soup.title.string.strip()
+ title = self._decode_entities(title)
+ return title
+
+ # Try headers in order
+ for header_tag in ['h1', 'h2', 'h3', 'h4']:
+ headers = soup.find_all(header_tag)
+ for header in headers:
+ title = header.get_text(strip=True)
+ if title:
+ title = self._decode_entities(title)
+ if self._is_chapter_title(title):
+ return title
+
+ return None
+
+ def _is_chapter_title(self, text: str) -> bool:
+ """Check if text looks like a chapter title"""
+ if not text or len(text) > 200:
+ return False
+
+ # Common chapter patterns
+ patterns = [
+ r'第.{1,10}[章回話话]',
+ r'Chapter\s+\d+',
+ r'제\s*\d+\s*화',
+ r'第\d+話',
+ r'\d+\s*화',
+ r'EP\.?\s*\d+',
+ r'Part\s+\d+',
+ ]
+
+ for pattern in patterns:
+ if re.search(pattern, text, re.IGNORECASE):
+ return True
+
+ # Check if it's short and doesn't contain too much punctuation
+ if len(text) < 100:
+ punct_count = sum(1 for c in text if c in '.,;:!?。、！？')
+ if punct_count < len(text) * 0.2:
+ return True
+
+ return False
+
+ def _extract_body_content(self, soup: BeautifulSoup, full_html: str) -> str:
+ """Extract body content while preserving Unicode"""
+ # Remove script and style elements first
+ for element in soup(['script', 'style', 'noscript']):
+ element.decompose()
+
+ if soup.body:
+ return str(soup.body)
+ else:
+ return str(soup)
+
+ def extract_chapter_content(self, html_content: str, extraction_mode: str = None) -> Tuple[str, str, Optional[str]]:
+ """Extract chapter content with proper Unicode and CJK handling"""
+ try:
+ # Use instance filtering_mode if not overridden
+ if extraction_mode is None:
+ extraction_mode = self.filtering_mode
+
+ # Handle encoding if content is bytes
+ if isinstance(html_content, bytes):
+ encoding = self._detect_encoding(html_content)
+ html_content = html_content.decode(encoding, errors='replace')
+
+ # Pre-process HTML to protect quotes
+ html_content = self._preprocess_html_for_quotes(html_content)
+
+ # Pre-process HTML to decode all entities
+ html_content = self._decode_entities(html_content)
+
+ # Detect language early
+ self.detected_language = self._detect_content_language(html_content)
+ print(f"🌐 Detected language: {self.detected_language}")
+
+ # Parse with BeautifulSoup
+ parser = 'html.parser'
+ if self.detected_language in ['korean', 'japanese', 'chinese']:
+ # For CJK content, lxml might handle encoding better if available
+ try:
+ import lxml
+ parser = 'lxml'
+ except ImportError:
+ pass
+
+ soup = BeautifulSoup(html_content, parser)
+
+ # Protect quotes before any processing
+ self._protect_quotes_in_soup(soup)
+
+ # Extract title
+ chapter_title = self._extract_title(soup)
+
+ # Respect GUI toggles to exclude headers/titles BEFORE conversion
+ try:
+ batch_translate_active = os.getenv('BATCH_TRANSLATE_HEADERS', '0') == '1'
+ ignore_title_tag = os.getenv('IGNORE_TITLE', '0') == '1' and batch_translate_active
+ ignore_header_tags = os.getenv('IGNORE_HEADER', '0') == '1' and batch_translate_active
+ if ignore_title_tag and soup.title:
+ # Remove
"왜 이러는 겁니까? 우리가 무슨 잘못을 했다고!"
+"......"
+"한 번만 살려주시오! 가족을 지키려면 어쩔 수 없었소!"
+"응애! 응애! 응애!"
+"미안하구나. 모든 죄는 내가 짊어지고 사마."
+ +