Spaces:
Running
Running
Commit
·
8e753b3
1
Parent(s):
190ea81
Rm image as citation
Browse files- api/chatbot.py +67 -32
- search/engines/image.py +42 -13
api/chatbot.py
CHANGED
|
@@ -18,7 +18,7 @@ class GeminiClient:
|
|
| 18 |
logger.warning("FlashAPI not set - Gemini client will use fallback responses")
|
| 19 |
self.client = None
|
| 20 |
else:
|
| 21 |
-
|
| 22 |
|
| 23 |
def generate_content(self, prompt: str, model: str = "gemini-2.5-flash", temperature: float = 0.7) -> str:
|
| 24 |
"""Generate content using Gemini API"""
|
|
@@ -205,8 +205,8 @@ class CookingTutorChatbot:
|
|
| 205 |
images = source_aggregation['images']
|
| 206 |
if images:
|
| 207 |
logger.info(f"Found {len(images)} images from search")
|
| 208 |
-
# Create enhanced image data with better frontend integration
|
| 209 |
-
enhanced_images = self._enhance_images_for_frontend(images[:
|
| 210 |
response_data['images'] = enhanced_images
|
| 211 |
|
| 212 |
# Create structured content with image placement suggestions
|
|
@@ -446,7 +446,7 @@ class CookingTutorChatbot:
|
|
| 446 |
return ''.join(enhanced_sections)
|
| 447 |
|
| 448 |
def _create_structured_content(self, text: str, images: List[Dict]) -> List[Dict]:
|
| 449 |
-
"""Create structured content blocks for optimal frontend rendering"""
|
| 450 |
if not images:
|
| 451 |
return [{'type': 'text', 'content': text}]
|
| 452 |
|
|
@@ -457,49 +457,84 @@ class CookingTutorChatbot:
|
|
| 457 |
image_index = 0
|
| 458 |
|
| 459 |
for section in sections:
|
| 460 |
-
#
|
| 461 |
-
|
| 462 |
-
'type': 'text',
|
| 463 |
-
'content': section['content'].strip(),
|
| 464 |
-
'section_type': section['type']
|
| 465 |
-
})
|
| 466 |
|
| 467 |
-
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
placement_context = image['placement_context']
|
| 471 |
-
|
| 472 |
-
should_add_image = (
|
| 473 |
-
(section['type'] == 'ingredients' and placement_context == 'after_ingredients') or
|
| 474 |
-
(section['type'] == 'instructions' and placement_context == 'after_instructions') or
|
| 475 |
-
(section['type'] == 'tips' and placement_context == 'after_tips') or
|
| 476 |
-
(section['type'] == 'intro' and placement_context == 'after_intro')
|
| 477 |
-
)
|
| 478 |
-
|
| 479 |
-
if should_add_image:
|
| 480 |
structured_blocks.append({
|
| 481 |
-
'type': '
|
| 482 |
-
'
|
| 483 |
-
'placement': 'after_section',
|
| 484 |
'section_type': section['type']
|
| 485 |
})
|
| 486 |
-
|
| 487 |
-
|
| 488 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 489 |
while image_index < len(images):
|
| 490 |
image = images[image_index]
|
| 491 |
structured_blocks.append({
|
| 492 |
'type': 'image',
|
| 493 |
'image_data': image,
|
| 494 |
-
'placement': '
|
| 495 |
})
|
| 496 |
image_index += 1
|
| 497 |
|
| 498 |
return structured_blocks
|
| 499 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 500 |
def _process_citations(self, response: str, url_mapping: Dict[int, str]) -> str:
|
| 501 |
"""Replace citation tags with actual URLs, handling various citation formats flexibly"""
|
| 502 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 503 |
# More flexible pattern to match various citation formats
|
| 504 |
citation_patterns = [
|
| 505 |
r'<#([^>]+)>', # Standard format: <#1>, <#1,2,3>
|
|
@@ -551,8 +586,8 @@ class CookingTutorChatbot:
|
|
| 551 |
doc_id = extract_numeric_id(citation_id)
|
| 552 |
|
| 553 |
if doc_id is not None and doc_id in url_mapping:
|
| 554 |
-
|
| 555 |
-
|
| 556 |
logger.info(f"[CITATION] Replacing <#{citation_id}> with {url}")
|
| 557 |
else:
|
| 558 |
if doc_id is None:
|
|
|
|
| 18 |
logger.warning("FlashAPI not set - Gemini client will use fallback responses")
|
| 19 |
self.client = None
|
| 20 |
else:
|
| 21 |
+
self.client = genai.Client(api_key=gemini_flash_api_key)
|
| 22 |
|
| 23 |
def generate_content(self, prompt: str, model: str = "gemini-2.5-flash", temperature: float = 0.7) -> str:
|
| 24 |
"""Generate content using Gemini API"""
|
|
|
|
| 205 |
images = source_aggregation['images']
|
| 206 |
if images:
|
| 207 |
logger.info(f"Found {len(images)} images from search")
|
| 208 |
+
# Create enhanced image data with better frontend integration - get more images
|
| 209 |
+
enhanced_images = self._enhance_images_for_frontend(images[:6], user_query)
|
| 210 |
response_data['images'] = enhanced_images
|
| 211 |
|
| 212 |
# Create structured content with image placement suggestions
|
|
|
|
| 446 |
return ''.join(enhanced_sections)
|
| 447 |
|
| 448 |
def _create_structured_content(self, text: str, images: List[Dict]) -> List[Dict]:
|
| 449 |
+
"""Create structured content blocks for optimal frontend rendering with inline image placement"""
|
| 450 |
if not images:
|
| 451 |
return [{'type': 'text', 'content': text}]
|
| 452 |
|
|
|
|
| 457 |
image_index = 0
|
| 458 |
|
| 459 |
for section in sections:
|
| 460 |
+
# Split section content into paragraphs for better inline placement
|
| 461 |
+
paragraphs = section['content'].strip().split('\n\n')
|
|
|
|
|
|
|
|
|
|
|
|
|
| 462 |
|
| 463 |
+
for i, paragraph in enumerate(paragraphs):
|
| 464 |
+
if paragraph.strip():
|
| 465 |
+
# Add paragraph as text block
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 466 |
structured_blocks.append({
|
| 467 |
+
'type': 'text',
|
| 468 |
+
'content': paragraph.strip(),
|
|
|
|
| 469 |
'section_type': section['type']
|
| 470 |
})
|
| 471 |
+
|
| 472 |
+
# Check if we should add an image after this paragraph
|
| 473 |
+
if image_index < len(images):
|
| 474 |
+
image = images[image_index]
|
| 475 |
+
placement_context = image['placement_context']
|
| 476 |
+
|
| 477 |
+
# More aggressive inline placement
|
| 478 |
+
should_add_image = (
|
| 479 |
+
# Add images more frequently for better visual flow
|
| 480 |
+
(section['type'] == 'ingredients' and placement_context == 'after_ingredients' and i == 0) or
|
| 481 |
+
(section['type'] == 'instructions' and placement_context == 'after_instructions' and i == 0) or
|
| 482 |
+
(section['type'] == 'tips' and placement_context == 'after_tips' and i == 0) or
|
| 483 |
+
(section['type'] == 'intro' and placement_context == 'after_intro' and i == 0) or
|
| 484 |
+
# Add images between paragraphs for better distribution
|
| 485 |
+
(i == 1 and image_index < len(images) - 1) or # Second paragraph gets an image
|
| 486 |
+
(i == 2 and image_index < len(images) - 2) # Third paragraph gets an image
|
| 487 |
+
)
|
| 488 |
+
|
| 489 |
+
if should_add_image:
|
| 490 |
+
structured_blocks.append({
|
| 491 |
+
'type': 'image',
|
| 492 |
+
'image_data': image,
|
| 493 |
+
'placement': 'inline',
|
| 494 |
+
'section_type': section['type']
|
| 495 |
+
})
|
| 496 |
+
image_index += 1
|
| 497 |
+
|
| 498 |
+
# Add any remaining images at strategic points
|
| 499 |
while image_index < len(images):
|
| 500 |
image = images[image_index]
|
| 501 |
structured_blocks.append({
|
| 502 |
'type': 'image',
|
| 503 |
'image_data': image,
|
| 504 |
+
'placement': 'inline'
|
| 505 |
})
|
| 506 |
image_index += 1
|
| 507 |
|
| 508 |
return structured_blocks
|
| 509 |
|
| 510 |
+
def _remove_image_urls_from_text(self, text: str) -> str:
|
| 511 |
+
"""Remove image URLs from text to prevent them from being processed as citations"""
|
| 512 |
+
import re
|
| 513 |
+
|
| 514 |
+
# Remove common image URL patterns that might appear in text
|
| 515 |
+
image_url_patterns = [
|
| 516 |
+
r'https?://[^\s]+\.(jpg|jpeg|png|gif|webp|svg)(\?[^\s]*)?', # Direct image URLs
|
| 517 |
+
r'<img[^>]*src=["\']([^"\']+)["\'][^>]*>', # HTML img tags
|
| 518 |
+
r'!\[[^\]]*\]\([^)]+\)', # Markdown image syntax
|
| 519 |
+
]
|
| 520 |
+
|
| 521 |
+
cleaned_text = text
|
| 522 |
+
for pattern in image_url_patterns:
|
| 523 |
+
cleaned_text = re.sub(pattern, '', cleaned_text, flags=re.IGNORECASE)
|
| 524 |
+
|
| 525 |
+
# Clean up any extra whitespace left behind
|
| 526 |
+
cleaned_text = re.sub(r'\n\s*\n\s*\n', '\n\n', cleaned_text)
|
| 527 |
+
cleaned_text = cleaned_text.strip()
|
| 528 |
+
|
| 529 |
+
return cleaned_text
|
| 530 |
+
|
| 531 |
def _process_citations(self, response: str, url_mapping: Dict[int, str]) -> str:
|
| 532 |
"""Replace citation tags with actual URLs, handling various citation formats flexibly"""
|
| 533 |
|
| 534 |
+
# First, remove any image URLs from the response to prevent them from being processed as citations
|
| 535 |
+
# This prevents image URLs from appearing as citations in the text
|
| 536 |
+
response = self._remove_image_urls_from_text(response)
|
| 537 |
+
|
| 538 |
# More flexible pattern to match various citation formats
|
| 539 |
citation_patterns = [
|
| 540 |
r'<#([^>]+)>', # Standard format: <#1>, <#1,2,3>
|
|
|
|
| 586 |
doc_id = extract_numeric_id(citation_id)
|
| 587 |
|
| 588 |
if doc_id is not None and doc_id in url_mapping:
|
| 589 |
+
url = url_mapping[doc_id]
|
| 590 |
+
urls.append(f'<{url}>')
|
| 591 |
logger.info(f"[CITATION] Replacing <#{citation_id}> with {url}")
|
| 592 |
else:
|
| 593 |
if doc_id is None:
|
search/engines/image.py
CHANGED
|
@@ -88,37 +88,57 @@ class ImageSearchEngine:
|
|
| 88 |
'query': final_dish_query,
|
| 89 |
'context': 'final_dish',
|
| 90 |
'type': 'final_dish',
|
| 91 |
-
'max_results': max(
|
| 92 |
})
|
| 93 |
|
| 94 |
-
# 2. Ingredients query - more specific
|
| 95 |
if any(keyword in query_lower for keyword in ['pad thai', 'noodles', 'pasta']):
|
| 96 |
ingredients_query = f"pad thai ingredients rice noodles shrimp"
|
|
|
|
| 97 |
elif any(keyword in query_lower for keyword in ['fusion', 'western']):
|
| 98 |
ingredients_query = f"fusion cooking ingredients fresh"
|
|
|
|
| 99 |
else:
|
| 100 |
ingredients_query = f"{clean_query} ingredients fresh"
|
|
|
|
| 101 |
|
| 102 |
queries.append({
|
| 103 |
'query': ingredients_query,
|
| 104 |
'context': 'ingredients',
|
| 105 |
'type': 'ingredients',
|
| 106 |
-
'max_results': max(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
})
|
| 108 |
|
| 109 |
# 3. Cooking technique/process query - more specific
|
| 110 |
if any(keyword in query_lower for keyword in ['pad thai', 'noodles', 'pasta']):
|
| 111 |
technique_query = f"pad thai cooking technique wok stir fry"
|
|
|
|
| 112 |
elif any(keyword in query_lower for keyword in ['fusion', 'western']):
|
| 113 |
technique_query = f"fusion cooking technique western"
|
|
|
|
| 114 |
else:
|
| 115 |
technique_query = f"{clean_query} cooking technique"
|
|
|
|
| 116 |
|
| 117 |
queries.append({
|
| 118 |
'query': technique_query,
|
| 119 |
'context': 'technique',
|
| 120 |
'type': 'technique',
|
| 121 |
-
'max_results': max(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
})
|
| 123 |
|
| 124 |
return queries
|
|
@@ -140,24 +160,33 @@ class ImageSearchEngine:
|
|
| 140 |
else:
|
| 141 |
type_groups['other'].append(result)
|
| 142 |
|
| 143 |
-
# Select diverse results
|
| 144 |
diverse_results = []
|
| 145 |
|
| 146 |
-
# Prioritize:
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
|
|
|
| 153 |
|
| 154 |
# Fill remaining slots with other results
|
| 155 |
all_remaining = []
|
| 156 |
for group in type_groups.values():
|
| 157 |
-
all_remaining.extend(group
|
|
|
|
|
|
|
|
|
|
| 158 |
|
| 159 |
diverse_results.extend(all_remaining[:num_results - len(diverse_results)])
|
| 160 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
return diverse_results[:num_results]
|
| 162 |
|
| 163 |
def _validate_image_results(self, results: List[Dict]) -> List[Dict]:
|
|
|
|
| 88 |
'query': final_dish_query,
|
| 89 |
'context': 'final_dish',
|
| 90 |
'type': 'final_dish',
|
| 91 |
+
'max_results': max(2, num_results // 4) # More final dish images
|
| 92 |
})
|
| 93 |
|
| 94 |
+
# 2. Ingredients query - more specific and diverse
|
| 95 |
if any(keyword in query_lower for keyword in ['pad thai', 'noodles', 'pasta']):
|
| 96 |
ingredients_query = f"pad thai ingredients rice noodles shrimp"
|
| 97 |
+
ingredients_query2 = f"pad thai fresh vegetables herbs"
|
| 98 |
elif any(keyword in query_lower for keyword in ['fusion', 'western']):
|
| 99 |
ingredients_query = f"fusion cooking ingredients fresh"
|
| 100 |
+
ingredients_query2 = f"western cooking ingredients vegetables"
|
| 101 |
else:
|
| 102 |
ingredients_query = f"{clean_query} ingredients fresh"
|
| 103 |
+
ingredients_query2 = f"{clean_query} raw ingredients vegetables"
|
| 104 |
|
| 105 |
queries.append({
|
| 106 |
'query': ingredients_query,
|
| 107 |
'context': 'ingredients',
|
| 108 |
'type': 'ingredients',
|
| 109 |
+
'max_results': max(2, num_results // 4) # More ingredient images
|
| 110 |
+
})
|
| 111 |
+
|
| 112 |
+
queries.append({
|
| 113 |
+
'query': ingredients_query2,
|
| 114 |
+
'context': 'ingredients',
|
| 115 |
+
'type': 'ingredients',
|
| 116 |
+
'max_results': max(1, num_results // 6) # Additional ingredient variety
|
| 117 |
})
|
| 118 |
|
| 119 |
# 3. Cooking technique/process query - more specific
|
| 120 |
if any(keyword in query_lower for keyword in ['pad thai', 'noodles', 'pasta']):
|
| 121 |
technique_query = f"pad thai cooking technique wok stir fry"
|
| 122 |
+
technique_query2 = f"pad thai preparation cooking process"
|
| 123 |
elif any(keyword in query_lower for keyword in ['fusion', 'western']):
|
| 124 |
technique_query = f"fusion cooking technique western"
|
| 125 |
+
technique_query2 = f"fusion cooking preparation method"
|
| 126 |
else:
|
| 127 |
technique_query = f"{clean_query} cooking technique"
|
| 128 |
+
technique_query2 = f"{clean_query} preparation method"
|
| 129 |
|
| 130 |
queries.append({
|
| 131 |
'query': technique_query,
|
| 132 |
'context': 'technique',
|
| 133 |
'type': 'technique',
|
| 134 |
+
'max_results': max(2, num_results // 4) # More technique images
|
| 135 |
+
})
|
| 136 |
+
|
| 137 |
+
queries.append({
|
| 138 |
+
'query': technique_query2,
|
| 139 |
+
'context': 'technique',
|
| 140 |
+
'type': 'technique',
|
| 141 |
+
'max_results': max(1, num_results // 6) # Additional technique variety
|
| 142 |
})
|
| 143 |
|
| 144 |
return queries
|
|
|
|
| 160 |
else:
|
| 161 |
type_groups['other'].append(result)
|
| 162 |
|
| 163 |
+
# Select diverse results with emphasis on ingredients and techniques
|
| 164 |
diverse_results = []
|
| 165 |
|
| 166 |
+
# Prioritize: 2 ingredients, 2 techniques, 2 final dishes for better diversity
|
| 167 |
+
for _ in range(2): # Get 2 of each type
|
| 168 |
+
if type_groups['ingredients'] and len(diverse_results) < num_results:
|
| 169 |
+
diverse_results.append(type_groups['ingredients'].pop(0))
|
| 170 |
+
if type_groups['technique'] and len(diverse_results) < num_results:
|
| 171 |
+
diverse_results.append(type_groups['technique'].pop(0))
|
| 172 |
+
if type_groups['final_dish'] and len(diverse_results) < num_results:
|
| 173 |
+
diverse_results.append(type_groups['final_dish'].pop(0))
|
| 174 |
|
| 175 |
# Fill remaining slots with other results
|
| 176 |
all_remaining = []
|
| 177 |
for group in type_groups.values():
|
| 178 |
+
all_remaining.extend(group) # Include all remaining results
|
| 179 |
+
|
| 180 |
+
# Sort by quality score if available
|
| 181 |
+
all_remaining.sort(key=lambda x: x.get('quality_score', 0), reverse=True)
|
| 182 |
|
| 183 |
diverse_results.extend(all_remaining[:num_results - len(diverse_results)])
|
| 184 |
|
| 185 |
+
logger.info(f"Prioritized {len(diverse_results)} diverse images: "
|
| 186 |
+
f"ingredients={len([r for r in diverse_results if r.get('image_type') == 'ingredients'])}, "
|
| 187 |
+
f"technique={len([r for r in diverse_results if r.get('image_type') == 'technique'])}, "
|
| 188 |
+
f"final_dish={len([r for r in diverse_results if r.get('image_type') == 'final_dish'])}")
|
| 189 |
+
|
| 190 |
return diverse_results[:num_results]
|
| 191 |
|
| 192 |
def _validate_image_results(self, results: List[Dict]) -> List[Dict]:
|