LiamKhoaLe commited on
Commit
8e753b3
·
1 Parent(s): 190ea81

Rm image as citation

Browse files
Files changed (2) hide show
  1. api/chatbot.py +67 -32
  2. search/engines/image.py +42 -13
api/chatbot.py CHANGED
@@ -18,7 +18,7 @@ class GeminiClient:
18
  logger.warning("FlashAPI not set - Gemini client will use fallback responses")
19
  self.client = None
20
  else:
21
- self.client = genai.Client(api_key=gemini_flash_api_key)
22
 
23
  def generate_content(self, prompt: str, model: str = "gemini-2.5-flash", temperature: float = 0.7) -> str:
24
  """Generate content using Gemini API"""
@@ -205,8 +205,8 @@ class CookingTutorChatbot:
205
  images = source_aggregation['images']
206
  if images:
207
  logger.info(f"Found {len(images)} images from search")
208
- # Create enhanced image data with better frontend integration
209
- enhanced_images = self._enhance_images_for_frontend(images[:3], user_query)
210
  response_data['images'] = enhanced_images
211
 
212
  # Create structured content with image placement suggestions
@@ -446,7 +446,7 @@ class CookingTutorChatbot:
446
  return ''.join(enhanced_sections)
447
 
448
  def _create_structured_content(self, text: str, images: List[Dict]) -> List[Dict]:
449
- """Create structured content blocks for optimal frontend rendering"""
450
  if not images:
451
  return [{'type': 'text', 'content': text}]
452
 
@@ -457,49 +457,84 @@ class CookingTutorChatbot:
457
  image_index = 0
458
 
459
  for section in sections:
460
- # Add text section
461
- structured_blocks.append({
462
- 'type': 'text',
463
- 'content': section['content'].strip(),
464
- 'section_type': section['type']
465
- })
466
 
467
- # Check if we should add an image after this section
468
- if image_index < len(images):
469
- image = images[image_index]
470
- placement_context = image['placement_context']
471
-
472
- should_add_image = (
473
- (section['type'] == 'ingredients' and placement_context == 'after_ingredients') or
474
- (section['type'] == 'instructions' and placement_context == 'after_instructions') or
475
- (section['type'] == 'tips' and placement_context == 'after_tips') or
476
- (section['type'] == 'intro' and placement_context == 'after_intro')
477
- )
478
-
479
- if should_add_image:
480
  structured_blocks.append({
481
- 'type': 'image',
482
- 'image_data': image,
483
- 'placement': 'after_section',
484
  'section_type': section['type']
485
  })
486
- image_index += 1
487
-
488
- # Add any remaining images at the end
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
489
  while image_index < len(images):
490
  image = images[image_index]
491
  structured_blocks.append({
492
  'type': 'image',
493
  'image_data': image,
494
- 'placement': 'end'
495
  })
496
  image_index += 1
497
 
498
  return structured_blocks
499
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
500
  def _process_citations(self, response: str, url_mapping: Dict[int, str]) -> str:
501
  """Replace citation tags with actual URLs, handling various citation formats flexibly"""
502
 
 
 
 
 
503
  # More flexible pattern to match various citation formats
504
  citation_patterns = [
505
  r'<#([^>]+)>', # Standard format: <#1>, <#1,2,3>
@@ -551,8 +586,8 @@ class CookingTutorChatbot:
551
  doc_id = extract_numeric_id(citation_id)
552
 
553
  if doc_id is not None and doc_id in url_mapping:
554
- url = url_mapping[doc_id]
555
- urls.append(f'<{url}>')
556
  logger.info(f"[CITATION] Replacing <#{citation_id}> with {url}")
557
  else:
558
  if doc_id is None:
 
18
  logger.warning("FlashAPI not set - Gemini client will use fallback responses")
19
  self.client = None
20
  else:
21
+ self.client = genai.Client(api_key=gemini_flash_api_key)
22
 
23
  def generate_content(self, prompt: str, model: str = "gemini-2.5-flash", temperature: float = 0.7) -> str:
24
  """Generate content using Gemini API"""
 
205
  images = source_aggregation['images']
206
  if images:
207
  logger.info(f"Found {len(images)} images from search")
208
+ # Create enhanced image data with better frontend integration - get more images
209
+ enhanced_images = self._enhance_images_for_frontend(images[:6], user_query)
210
  response_data['images'] = enhanced_images
211
 
212
  # Create structured content with image placement suggestions
 
446
  return ''.join(enhanced_sections)
447
 
448
  def _create_structured_content(self, text: str, images: List[Dict]) -> List[Dict]:
449
+ """Create structured content blocks for optimal frontend rendering with inline image placement"""
450
  if not images:
451
  return [{'type': 'text', 'content': text}]
452
 
 
457
  image_index = 0
458
 
459
  for section in sections:
460
+ # Split section content into paragraphs for better inline placement
461
+ paragraphs = section['content'].strip().split('\n\n')
 
 
 
 
462
 
463
+ for i, paragraph in enumerate(paragraphs):
464
+ if paragraph.strip():
465
+ # Add paragraph as text block
 
 
 
 
 
 
 
 
 
 
466
  structured_blocks.append({
467
+ 'type': 'text',
468
+ 'content': paragraph.strip(),
 
469
  'section_type': section['type']
470
  })
471
+
472
+ # Check if we should add an image after this paragraph
473
+ if image_index < len(images):
474
+ image = images[image_index]
475
+ placement_context = image['placement_context']
476
+
477
+ # More aggressive inline placement
478
+ should_add_image = (
479
+ # Add images more frequently for better visual flow
480
+ (section['type'] == 'ingredients' and placement_context == 'after_ingredients' and i == 0) or
481
+ (section['type'] == 'instructions' and placement_context == 'after_instructions' and i == 0) or
482
+ (section['type'] == 'tips' and placement_context == 'after_tips' and i == 0) or
483
+ (section['type'] == 'intro' and placement_context == 'after_intro' and i == 0) or
484
+ # Add images between paragraphs for better distribution
485
+ (i == 1 and image_index < len(images) - 1) or # Second paragraph gets an image
486
+ (i == 2 and image_index < len(images) - 2) # Third paragraph gets an image
487
+ )
488
+
489
+ if should_add_image:
490
+ structured_blocks.append({
491
+ 'type': 'image',
492
+ 'image_data': image,
493
+ 'placement': 'inline',
494
+ 'section_type': section['type']
495
+ })
496
+ image_index += 1
497
+
498
+ # Add any remaining images at strategic points
499
  while image_index < len(images):
500
  image = images[image_index]
501
  structured_blocks.append({
502
  'type': 'image',
503
  'image_data': image,
504
+ 'placement': 'inline'
505
  })
506
  image_index += 1
507
 
508
  return structured_blocks
509
 
510
+ def _remove_image_urls_from_text(self, text: str) -> str:
511
+ """Remove image URLs from text to prevent them from being processed as citations"""
512
+ import re
513
+
514
+ # Remove common image URL patterns that might appear in text
515
+ image_url_patterns = [
516
+ r'https?://[^\s]+\.(jpg|jpeg|png|gif|webp|svg)(\?[^\s]*)?', # Direct image URLs
517
+ r'<img[^>]*src=["\']([^"\']+)["\'][^>]*>', # HTML img tags
518
+ r'!\[[^\]]*\]\([^)]+\)', # Markdown image syntax
519
+ ]
520
+
521
+ cleaned_text = text
522
+ for pattern in image_url_patterns:
523
+ cleaned_text = re.sub(pattern, '', cleaned_text, flags=re.IGNORECASE)
524
+
525
+ # Clean up any extra whitespace left behind
526
+ cleaned_text = re.sub(r'\n\s*\n\s*\n', '\n\n', cleaned_text)
527
+ cleaned_text = cleaned_text.strip()
528
+
529
+ return cleaned_text
530
+
531
  def _process_citations(self, response: str, url_mapping: Dict[int, str]) -> str:
532
  """Replace citation tags with actual URLs, handling various citation formats flexibly"""
533
 
534
+ # First, remove any image URLs from the response to prevent them from being processed as citations
535
+ # This prevents image URLs from appearing as citations in the text
536
+ response = self._remove_image_urls_from_text(response)
537
+
538
  # More flexible pattern to match various citation formats
539
  citation_patterns = [
540
  r'<#([^>]+)>', # Standard format: <#1>, <#1,2,3>
 
586
  doc_id = extract_numeric_id(citation_id)
587
 
588
  if doc_id is not None and doc_id in url_mapping:
589
+ url = url_mapping[doc_id]
590
+ urls.append(f'<{url}>')
591
  logger.info(f"[CITATION] Replacing <#{citation_id}> with {url}")
592
  else:
593
  if doc_id is None:
search/engines/image.py CHANGED
@@ -88,37 +88,57 @@ class ImageSearchEngine:
88
  'query': final_dish_query,
89
  'context': 'final_dish',
90
  'type': 'final_dish',
91
- 'max_results': max(1, num_results // 3)
92
  })
93
 
94
- # 2. Ingredients query - more specific
95
  if any(keyword in query_lower for keyword in ['pad thai', 'noodles', 'pasta']):
96
  ingredients_query = f"pad thai ingredients rice noodles shrimp"
 
97
  elif any(keyword in query_lower for keyword in ['fusion', 'western']):
98
  ingredients_query = f"fusion cooking ingredients fresh"
 
99
  else:
100
  ingredients_query = f"{clean_query} ingredients fresh"
 
101
 
102
  queries.append({
103
  'query': ingredients_query,
104
  'context': 'ingredients',
105
  'type': 'ingredients',
106
- 'max_results': max(1, num_results // 3)
 
 
 
 
 
 
 
107
  })
108
 
109
  # 3. Cooking technique/process query - more specific
110
  if any(keyword in query_lower for keyword in ['pad thai', 'noodles', 'pasta']):
111
  technique_query = f"pad thai cooking technique wok stir fry"
 
112
  elif any(keyword in query_lower for keyword in ['fusion', 'western']):
113
  technique_query = f"fusion cooking technique western"
 
114
  else:
115
  technique_query = f"{clean_query} cooking technique"
 
116
 
117
  queries.append({
118
  'query': technique_query,
119
  'context': 'technique',
120
  'type': 'technique',
121
- 'max_results': max(1, num_results // 3)
 
 
 
 
 
 
 
122
  })
123
 
124
  return queries
@@ -140,24 +160,33 @@ class ImageSearchEngine:
140
  else:
141
  type_groups['other'].append(result)
142
 
143
- # Select diverse results
144
  diverse_results = []
145
 
146
- # Prioritize: 1 final dish, 1 ingredients, 1 technique, then fill with others
147
- if type_groups['final_dish']:
148
- diverse_results.append(type_groups['final_dish'][0])
149
- if type_groups['ingredients'] and len(diverse_results) < num_results:
150
- diverse_results.append(type_groups['ingredients'][0])
151
- if type_groups['technique'] and len(diverse_results) < num_results:
152
- diverse_results.append(type_groups['technique'][0])
 
153
 
154
  # Fill remaining slots with other results
155
  all_remaining = []
156
  for group in type_groups.values():
157
- all_remaining.extend(group[1:]) # Skip first item (already used)
 
 
 
158
 
159
  diverse_results.extend(all_remaining[:num_results - len(diverse_results)])
160
 
 
 
 
 
 
161
  return diverse_results[:num_results]
162
 
163
  def _validate_image_results(self, results: List[Dict]) -> List[Dict]:
 
88
  'query': final_dish_query,
89
  'context': 'final_dish',
90
  'type': 'final_dish',
91
+ 'max_results': max(2, num_results // 4) # More final dish images
92
  })
93
 
94
+ # 2. Ingredients query - more specific and diverse
95
  if any(keyword in query_lower for keyword in ['pad thai', 'noodles', 'pasta']):
96
  ingredients_query = f"pad thai ingredients rice noodles shrimp"
97
+ ingredients_query2 = f"pad thai fresh vegetables herbs"
98
  elif any(keyword in query_lower for keyword in ['fusion', 'western']):
99
  ingredients_query = f"fusion cooking ingredients fresh"
100
+ ingredients_query2 = f"western cooking ingredients vegetables"
101
  else:
102
  ingredients_query = f"{clean_query} ingredients fresh"
103
+ ingredients_query2 = f"{clean_query} raw ingredients vegetables"
104
 
105
  queries.append({
106
  'query': ingredients_query,
107
  'context': 'ingredients',
108
  'type': 'ingredients',
109
+ 'max_results': max(2, num_results // 4) # More ingredient images
110
+ })
111
+
112
+ queries.append({
113
+ 'query': ingredients_query2,
114
+ 'context': 'ingredients',
115
+ 'type': 'ingredients',
116
+ 'max_results': max(1, num_results // 6) # Additional ingredient variety
117
  })
118
 
119
  # 3. Cooking technique/process query - more specific
120
  if any(keyword in query_lower for keyword in ['pad thai', 'noodles', 'pasta']):
121
  technique_query = f"pad thai cooking technique wok stir fry"
122
+ technique_query2 = f"pad thai preparation cooking process"
123
  elif any(keyword in query_lower for keyword in ['fusion', 'western']):
124
  technique_query = f"fusion cooking technique western"
125
+ technique_query2 = f"fusion cooking preparation method"
126
  else:
127
  technique_query = f"{clean_query} cooking technique"
128
+ technique_query2 = f"{clean_query} preparation method"
129
 
130
  queries.append({
131
  'query': technique_query,
132
  'context': 'technique',
133
  'type': 'technique',
134
+ 'max_results': max(2, num_results // 4) # More technique images
135
+ })
136
+
137
+ queries.append({
138
+ 'query': technique_query2,
139
+ 'context': 'technique',
140
+ 'type': 'technique',
141
+ 'max_results': max(1, num_results // 6) # Additional technique variety
142
  })
143
 
144
  return queries
 
160
  else:
161
  type_groups['other'].append(result)
162
 
163
+ # Select diverse results with emphasis on ingredients and techniques
164
  diverse_results = []
165
 
166
+ # Prioritize: 2 ingredients, 2 techniques, 2 final dishes for better diversity
167
+ for _ in range(2): # Get 2 of each type
168
+ if type_groups['ingredients'] and len(diverse_results) < num_results:
169
+ diverse_results.append(type_groups['ingredients'].pop(0))
170
+ if type_groups['technique'] and len(diverse_results) < num_results:
171
+ diverse_results.append(type_groups['technique'].pop(0))
172
+ if type_groups['final_dish'] and len(diverse_results) < num_results:
173
+ diverse_results.append(type_groups['final_dish'].pop(0))
174
 
175
  # Fill remaining slots with other results
176
  all_remaining = []
177
  for group in type_groups.values():
178
+ all_remaining.extend(group) # Include all remaining results
179
+
180
+ # Sort by quality score if available
181
+ all_remaining.sort(key=lambda x: x.get('quality_score', 0), reverse=True)
182
 
183
  diverse_results.extend(all_remaining[:num_results - len(diverse_results)])
184
 
185
+ logger.info(f"Prioritized {len(diverse_results)} diverse images: "
186
+ f"ingredients={len([r for r in diverse_results if r.get('image_type') == 'ingredients'])}, "
187
+ f"technique={len([r for r in diverse_results if r.get('image_type') == 'technique'])}, "
188
+ f"final_dish={len([r for r in diverse_results if r.get('image_type') == 'final_dish'])}")
189
+
190
  return diverse_results[:num_results]
191
 
192
  def _validate_image_results(self, results: List[Dict]) -> List[Dict]: