Enh search strats

search/engines/cooking.py

@@ -16,56 +16,133 @@ class CookingSearchEngine:
         })
         self.timeout = timeout
         
-        # Curated cooking sources
+        # Comprehensive cooking sources with enhanced search strategies
         self.cooking_sources = {
             'allrecipes': {
                 'base_url': 'https://www.allrecipes.com',
                 'search_url': 'https://www.allrecipes.com/search',
-                'domains': ['allrecipes.com']
+                'domains': ['allrecipes.com'],
+                'search_params': ['q', 'query', 'search'],
+                'priority': 1
             },
             'food_network': {
                 'base_url': 'https://www.foodnetwork.com',
                 'search_url': 'https://www.foodnetwork.com/search',
-                'domains': ['foodnetwork.com']
+                'domains': ['foodnetwork.com'],
+                'search_params': ['q', 'query', 'search'],
+                'priority': 1
             },
             'epicurious': {
                 'base_url': 'https://www.epicurious.com',
                 'search_url': 'https://www.epicurious.com/search',
-                'domains': ['epicurious.com']
+                'domains': ['epicurious.com'],
+                'search_params': ['q', 'query', 'search'],
+                'priority': 1
             },
             'serious_eats': {
                 'base_url': 'https://www.seriouseats.com',
                 'search_url': 'https://www.seriouseats.com/search',
-                'domains': ['seriouseats.com']
+                'domains': ['seriouseats.com'],
+                'search_params': ['q', 'query', 'search'],
+                'priority': 1
             },
             'bon_appetit': {
                 'base_url': 'https://www.bonappetit.com',
                 'search_url': 'https://www.bonappetit.com/search',
-                'domains': ['bonappetit.com']
+                'domains': ['bonappetit.com'],
+                'search_params': ['q', 'query', 'search'],
+                'priority': 1
+            },
+            'taste_of_home': {
+                'base_url': 'https://www.tasteofhome.com',
+                'search_url': 'https://www.tasteofhome.com/search',
+                'domains': ['tasteofhome.com'],
+                'search_params': ['q', 'query', 'search'],
+                'priority': 2
+            },
+            'food_com': {
+                'base_url': 'https://www.food.com',
+                'search_url': 'https://www.food.com/search',
+                'domains': ['food.com'],
+                'search_params': ['q', 'query', 'search'],
+                'priority': 2
+            },
+            'bbc_good_food': {
+                'base_url': 'https://www.bbcgoodfood.com',
+                'search_url': 'https://www.bbcgoodfood.com/search',
+                'domains': ['bbcgoodfood.com'],
+                'search_params': ['q', 'query', 'search'],
+                'priority': 2
+            },
+            'martha_stewart': {
+                'base_url': 'https://www.marthastewart.com',
+                'search_url': 'https://www.marthastewart.com/search',
+                'domains': ['marthastewart.com'],
+                'search_params': ['q', 'query', 'search'],
+                'priority': 2
+            },
+            'king_arthur_baking': {
+                'base_url': 'https://www.kingarthurbaking.com',
+                'search_url': 'https://www.kingarthurbaking.com/search',
+                'domains': ['kingarthurbaking.com'],
+                'search_params': ['q', 'query', 'search'],
+                'priority': 2
             }
         }
     
     def search(self, query: str, num_results: int = 10) -> List[Dict]:
-        """Search cooking sources for relevant information"""
+        """Search cooking sources for relevant information with enhanced strategies"""
         results = []
         
-        # Strategy 1: Direct cooking source searches
-        for source_name, source_config in self.cooking_sources.items():
+        # Enhanced query processing
+        enhanced_queries = self._create_enhanced_queries(query)
+        logger.info(f"Enhanced queries for cooking search: {enhanced_queries}")
+        
+        # Strategy 1: Priority-based source searches
+        priority_sources = self._get_priority_sources()
+        
+        for priority_level in [1, 2]:  # Search priority 1 sources first, then priority 2
             if len(results) >= num_results:
                 break
                 
-            source_results = self._search_cooking_source(query, source_name, source_config)
-            results.extend(source_results)
-            
-            # Add delay between requests
-            time.sleep(0.5)
+            for source_name in priority_sources.get(priority_level, []):
+                if len(results) >= num_results:
+                    break
+                    
+                source_config = self.cooking_sources[source_name]
+                
+                # Try multiple query variations for each source
+                for query_variant in enhanced_queries:
+                    if len(results) >= num_results:
+                        break
+                        
+                    source_results = self._search_cooking_source(query_variant, source_name, source_config)
+                    if source_results:
+                        results.extend(source_results)
+                        logger.info(f"{source_name} found {len(source_results)} results for query: {query_variant}")
+                        break  # Move to next source if we found results
+                
+                # Add delay between requests
+                time.sleep(0.3)
+        
+        # Strategy 2: Recipe-specific searches if we need more results
+        if len(results) < num_results:
+            recipe_results = self._search_recipe_specific(query, num_results - len(results))
+            results.extend(recipe_results)
         
-        # Strategy 2: Cooking fallback sources
+        # Strategy 3: Technique-specific searches
+        if len(results) < num_results:
+            technique_results = self._search_technique_specific(query, num_results - len(results))
+            results.extend(technique_results)
+        
+        # Strategy 4: Cooking fallback sources
         if len(results) < num_results:
             fallback_results = self._get_fallback_sources(query, num_results - len(results))
             results.extend(fallback_results)
         
-        return results[:num_results]
+        # Remove duplicates and return top results
+        unique_results = self._remove_duplicates(results)
+        return unique_results[:num_results]
     
     def _search_cooking_source(self, query: str, source_name: str, source_config: Dict) -> List[Dict]:
         """Search a specific cooking source"""
@@ -195,3 +272,120 @@ class CookingSearchEngine:
         ]
         
         return fallback_sources[:num_results]
+    
+    def _create_enhanced_queries(self, query: str) -> List[str]:
+        """Create enhanced query variations for better cooking search results"""
+        import re
+        
+        # Clean the base query
+        base_query = re.sub(r'[^\w\s\-\.]', ' ', query).strip()
+        base_query = re.sub(r'\s+', ' ', base_query)
+        
+        enhanced_queries = [base_query]
+        
+        # Add cooking-specific enhancements
+        cooking_enhancements = [
+            f"{base_query} recipe",
+            f"{base_query} cooking method",
+            f"{base_query} how to cook",
+            f"{base_query} ingredients",
+            f"{base_query} technique",
+            f"{base_query} tutorial"
+        ]
+        
+        # Add technique-specific queries
+        cooking_techniques = ['bake', 'roast', 'grill', 'fry', 'boil', 'steam', 'sauté', 'braise', 'poach']
+        for technique in cooking_techniques:
+            if technique in base_query.lower():
+                enhanced_queries.append(f"{base_query} {technique} method")
+                enhanced_queries.append(f"how to {technique} {base_query}")
+        
+        # Add cuisine-specific enhancements
+        cuisines = ['italian', 'chinese', 'mexican', 'french', 'indian', 'thai', 'japanese', 'mediterranean']
+        for cuisine in cuisines:
+            if cuisine in base_query.lower():
+                enhanced_queries.append(f"{cuisine} {base_query} recipe")
+                enhanced_queries.append(f"authentic {cuisine} {base_query}")
+        
+        # Remove duplicates and limit
+        unique_queries = list(dict.fromkeys(enhanced_queries))
+        return unique_queries[:5]  # Limit to 5 query variations
+    
+    def _get_priority_sources(self) -> Dict[int, List[str]]:
+        """Get sources organized by priority"""
+        priority_sources = {1: [], 2: []}
+        
+        for source_name, config in self.cooking_sources.items():
+            priority = config.get('priority', 2)
+            priority_sources[priority].append(source_name)
+        
+        return priority_sources
+    
+    def _search_recipe_specific(self, query: str, num_results: int) -> List[Dict]:
+        """Search for recipe-specific content"""
+        recipe_queries = [
+            f"{query} recipe ingredients",
+            f"{query} recipe instructions",
+            f"{query} recipe steps",
+            f"how to make {query}",
+            f"{query} cooking recipe"
+        ]
+        
+        results = []
+        for recipe_query in recipe_queries:
+            if len(results) >= num_results:
+                break
+                
+            # Search top priority sources for recipe content
+            priority_sources = self._get_priority_sources()
+            for source_name in priority_sources.get(1, []):
+                if len(results) >= num_results:
+                    break
+                    
+                source_config = self.cooking_sources[source_name]
+                source_results = self._search_cooking_source(recipe_query, source_name, source_config)
+                results.extend(source_results)
+                time.sleep(0.2)
+        
+        return results[:num_results]
+    
+    def _search_technique_specific(self, query: str, num_results: int) -> List[Dict]:
+        """Search for cooking technique-specific content"""
+        technique_queries = [
+            f"{query} cooking technique",
+            f"{query} cooking method",
+            f"how to cook {query}",
+            f"{query} preparation method",
+            f"{query} cooking tips"
+        ]
+        
+        results = []
+        for technique_query in technique_queries:
+            if len(results) >= num_results:
+                break
+                
+            # Search priority sources for technique content
+            priority_sources = self._get_priority_sources()
+            for source_name in priority_sources.get(1, []):
+                if len(results) >= num_results:
+                    break
+                    
+                source_config = self.cooking_sources[source_name]
+                source_results = self._search_cooking_source(technique_query, source_name, source_config)
+                results.extend(source_results)
+                time.sleep(0.2)
+        
+        return results[:num_results]
+    
+    def _remove_duplicates(self, results: List[Dict]) -> List[Dict]:
+        """Remove duplicate results based on URL"""
+        seen_urls = set()
+        unique_results = []
+        
+        for result in results:
+            url = result.get('url', '')
+            if url and url not in seen_urls:
+                seen_urls.add(url)
+                unique_results.append(result)
+        
+        return unique_results

search/extractors/content.py

@@ -22,15 +22,19 @@ class ContentExtractor:
         })
         self.timeout = timeout
         
-        # Medical content indicators
-        self.medical_indicators = [
-            'symptom', 'treatment', 'diagnosis', 'medicine', 'medication',
-            'therapy', 'condition', 'disease', 'health', 'medical',
-            'doctor', 'physician', 'patient', 'clinical', 'study'
+        # Cooking content indicators
+        self.cooking_indicators = [
+            'recipe', 'ingredients', 'instructions', 'cooking', 'baking', 'roasting',
+            'grilling', 'frying', 'boiling', 'steaming', 'sautéing', 'braising',
+            'seasoning', 'spices', 'herbs', 'sauce', 'marinade', 'dressing',
+            'temperature', 'timing', 'preparation', 'technique', 'method',
+            'oven', 'stovetop', 'grill', 'pan', 'pot', 'skillet', 'knife',
+            'cutting', 'chopping', 'dicing', 'slicing', 'mixing', 'stirring',
+            'servings', 'cook time', 'prep time', 'total time', 'difficulty'
         ]
     
     def extract(self, url: str, max_length: int = 2000) -> Optional[str]:
-        """Extract content from a URL with medical focus"""
+        """Extract content from a URL with cooking focus"""
         try:
             response = self.session.get(url, timeout=self.timeout)
             response.raise_for_status()
@@ -49,11 +53,11 @@ class ContentExtractor:
             # Clean and process content
             cleaned_content = self._clean_content(content)
             
-            # Focus on medical content if possible
-            medical_content = self._extract_medical_content(cleaned_content)
+            # Focus on cooking content if possible
+            cooking_content = self._extract_cooking_content(cleaned_content)
             
             # Truncate to max length
-            final_content = self._truncate_content(medical_content or cleaned_content, max_length)
+            final_content = self._truncate_content(cooking_content or cleaned_content, max_length)
             
             return final_content if final_content else None
             
@@ -145,29 +149,29 @@ class ContentExtractor:
         
         return content.strip()
     
-    def _extract_medical_content(self, content: str) -> Optional[str]:
-        """Extract medical-focused content from the text"""
+    def _extract_cooking_content(self, content: str) -> Optional[str]:
+        """Extract cooking-focused content from the text"""
         if not content:
             return None
         
         # Split content into sentences
         sentences = re.split(r'[.!?]+', content)
-        medical_sentences = []
+        cooking_sentences = []
         
         for sentence in sentences:
             sentence = sentence.strip()
             if len(sentence) < 20:  # Skip very short sentences
                 continue
             
-            # Check if sentence contains medical indicators
+            # Check if sentence contains cooking indicators
             sentence_lower = sentence.lower()
-            if any(indicator in sentence_lower for indicator in self.medical_indicators):
-                medical_sentences.append(sentence)
+            if any(indicator in sentence_lower for indicator in self.cooking_indicators):
+                cooking_sentences.append(sentence)
         
-        if medical_sentences:
-            # Return medical sentences, prioritizing longer ones
-            medical_sentences.sort(key=len, reverse=True)
-            return '. '.join(medical_sentences[:10]) + '.'
+        if cooking_sentences:
+            # Return cooking sentences, prioritizing longer ones
+            cooking_sentences.sort(key=len, reverse=True)
+            return '. '.join(cooking_sentences[:15]) + '.'  # More sentences for cooking content
         
         return None
     

search/processors/cooking.py

@@ -9,17 +9,47 @@ class CookingSearchProcessor:
     """Process and enhance cooking search results"""
     
     def __init__(self):
-        self.cooking_keywords = [
-            'recipe', 'cooking', 'baking', 'roasting', 'grilling', 'frying', 'boiling', 'steaming',
-            'ingredients', 'seasoning', 'spices', 'herbs', 'sauce', 'marinade', 'dressing',
-            'technique', 'method', 'temperature', 'timing', 'preparation', 'cooking time',
-            'oven', 'stovetop', 'grill', 'pan', 'pot', 'skillet', 'knife', 'cutting',
-            'vegetarian', 'vegan', 'gluten-free', 'dairy-free', 'keto', 'paleo', 'diet',
-            'appetizer', 'main course', 'dessert', 'breakfast', 'lunch', 'dinner',
-            'cuisine', 'italian', 'chinese', 'mexican', 'french', 'indian', 'thai',
-            'substitution', 'alternative', 'variation', 'modification', 'adaptation',
-            'troubleshooting', 'tips', 'tricks', 'hacks', 'mistakes', 'common errors'
-        ]
+        # Enhanced cooking keywords with categories
+        self.cooking_keywords = {
+            'primary': [
+                'recipe', 'cooking', 'baking', 'roasting', 'grilling', 'frying', 'boiling', 'steaming',
+                'sautéing', 'braising', 'poaching', 'broiling', 'searing', 'simmering'
+            ],
+            'ingredients': [
+                'ingredients', 'seasoning', 'spices', 'herbs', 'sauce', 'marinade', 'dressing',
+                'oil', 'butter', 'flour', 'sugar', 'salt', 'pepper', 'garlic', 'onion',
+                'vegetables', 'meat', 'chicken', 'beef', 'pork', 'fish', 'seafood'
+            ],
+            'techniques': [
+                'technique', 'method', 'temperature', 'timing', 'preparation', 'cooking time',
+                'prep time', 'total time', 'servings', 'difficulty', 'skill level'
+            ],
+            'equipment': [
+                'oven', 'stovetop', 'grill', 'pan', 'pot', 'skillet', 'knife', 'cutting',
+                'mixing', 'stirring', 'chopping', 'dicing', 'slicing', 'whisking'
+            ],
+            'dietary': [
+                'vegetarian', 'vegan', 'gluten-free', 'dairy-free', 'keto', 'paleo', 'diet',
+                'healthy', 'low-carb', 'low-fat', 'protein', 'fiber'
+            ],
+            'meal_types': [
+                'appetizer', 'main course', 'dessert', 'breakfast', 'lunch', 'dinner',
+                'snack', 'side dish', 'soup', 'salad', 'pasta', 'pizza'
+            ],
+            'cuisines': [
+                'italian', 'chinese', 'mexican', 'french', 'indian', 'thai', 'japanese',
+                'mediterranean', 'american', 'asian', 'european', 'fusion'
+            ],
+            'modifications': [
+                'substitution', 'alternative', 'variation', 'modification', 'adaptation',
+                'troubleshooting', 'tips', 'tricks', 'hacks', 'mistakes', 'common errors'
+            ]
+        }
+        
+        # Flatten all keywords for easy lookup
+        self.all_cooking_keywords = []
+        for category, keywords in self.cooking_keywords.items():
+            self.all_cooking_keywords.extend(keywords)
     
     def process_results(self, results: List[Dict], user_query: str) -> Tuple[str, Dict[int, str]]:
         """Process search results and create comprehensive cooking summary"""
@@ -62,37 +92,67 @@ class CookingSearchProcessor:
         return relevant_results[:10]
     
     def _calculate_relevance_score(self, result: Dict, user_query: str) -> float:
-        """Calculate cooking relevance score for a result"""
+        """Calculate enhanced cooking relevance score for a result"""
         score = 0.0
         
-        # Check title relevance
+        # Check title and content relevance
         title = result.get('title', '').lower()
+        content = result.get('content', '').lower()
         query_lower = user_query.lower()
         
-        # Direct query match in title
-        if any(word in title for word in query_lower.split()):
-            score += 0.4
-        
-        # Cooking keyword match in title
-        cooking_matches = sum(1 for keyword in self.cooking_keywords if keyword in title)
-        score += min(cooking_matches * 0.1, 0.3)
-        
-        # Domain credibility for cooking sources
+        # Direct query match in title (highest priority)
+        query_words = query_lower.split()
+        title_matches = sum(1 for word in query_words if word in title)
+        if title_matches > 0:
+            score += min(title_matches * 0.15, 0.4)
+        
+        # Direct query match in content
+        content_matches = sum(1 for word in query_words if word in content)
+        if content_matches > 0:
+            score += min(content_matches * 0.05, 0.2)
+        
+        # Enhanced cooking keyword scoring by category
+        for category, keywords in self.cooking_keywords.items():
+            category_matches = sum(1 for keyword in keywords if keyword in title)
+            if category_matches > 0:
+                # Different weights for different categories
+                if category == 'primary':
+                    score += min(category_matches * 0.08, 0.25)
+                elif category == 'ingredients':
+                    score += min(category_matches * 0.06, 0.2)
+                elif category == 'techniques':
+                    score += min(category_matches * 0.07, 0.2)
+                elif category == 'cuisines':
+                    score += min(category_matches * 0.05, 0.15)
+                else:
+                    score += min(category_matches * 0.04, 0.1)
+        
+        # Domain credibility for cooking sources (enhanced list)
         url = result.get('url', '').lower()
         credible_domains = [
             'allrecipes.com', 'foodnetwork.com', 'epicurious.com', 'seriouseats.com',
             'bonappetit.com', 'cooking.nytimes.com', 'tasteofhome.com', 'food.com',
             'bbcgoodfood.com', 'jamieoliver.com', 'gordonramsay.com', 'marthastewart.com',
-            'kingarthurbaking.com', 'sallysbakingaddiction.com', 'smittenkitchen.com'
+            'kingarthurbaking.com', 'sallysbakingaddiction.com', 'smittenkitchen.com',
+            'food52.com', 'cookinglight.com', 'eatingwell.com', 'delish.com',
+            'tasty.co', 'buzzfeed.com/food', 'foodandwine.com', 'saveur.com'
         ]
         
         if any(domain in url for domain in credible_domains):
-            score += 0.3
+            score += 0.25
         
         # Source type bonus for cooking
         source = result.get('source', '')
         if 'cooking' in source or 'recipe' in source or any(domain in source for domain in credible_domains):
-            score += 0.2
+            score += 0.15
+        
+        # Recipe-specific content bonus
+        if any(word in title for word in ['recipe', 'how to', 'tutorial', 'guide']):
+            score += 0.1
+        
+        # URL path analysis for cooking content
+        if any(path in url for path in ['/recipe/', '/recipes/', '/cooking/', '/food/']):
+            score += 0.1
         
         return min(score, 1.0)
     
@@ -174,11 +234,13 @@ class CookingSearchProcessor:
         return combined_summary
     
     def _group_by_topic(self, results: List[Dict]) -> Dict[str, List[Dict]]:
-        """Group results by cooking topic"""
+        """Group results by enhanced cooking topics"""
         topics = {
             'recipes': [],
             'techniques': [],
             'ingredients': [],
+            'equipment': [],
+            'tips_tricks': [],
             'general': []
         }
         
@@ -187,13 +249,17 @@ class CookingSearchProcessor:
             summary_lower = result.get('summary', '').lower()
             content_lower = f"{title_lower} {summary_lower}"
             
-            # Categorize by content
-            if any(word in content_lower for word in ['recipe', 'ingredients', 'instructions', 'steps']):
+            # Enhanced categorization by content
+            if any(word in content_lower for word in ['recipe', 'ingredients', 'instructions', 'steps', 'how to make']):
                 topics['recipes'].append(result)
-            elif any(word in content_lower for word in ['technique', 'method', 'how to', 'cooking']):
+            elif any(word in content_lower for word in ['technique', 'method', 'how to cook', 'cooking method', 'preparation']):
                 topics['techniques'].append(result)
-            elif any(word in content_lower for word in ['ingredients', 'substitution', 'alternative', 'variation']):
+            elif any(word in content_lower for word in ['ingredients', 'substitution', 'alternative', 'variation', 'seasoning', 'spices']):
                 topics['ingredients'].append(result)
+            elif any(word in content_lower for word in ['equipment', 'tools', 'knife', 'pan', 'pot', 'oven', 'grill']):
+                topics['equipment'].append(result)
+            elif any(word in content_lower for word in ['tips', 'tricks', 'hacks', 'mistakes', 'troubleshooting', 'advice']):
+                topics['tips_tricks'].append(result)
             else:
                 topics['general'].append(result)
         
@@ -206,10 +272,12 @@ class CookingSearchProcessor:
         
         # Add topic header
         topic_headers = {
-            'recipes': "**Recipes and Instructions:**",
-            'techniques': "**Cooking Techniques:**",
-            'ingredients': "**Ingredients and Substitutions:**",
-            'general': "**General Information:**"
+            'recipes': "**🍳 Recipes and Instructions:**",
+            'techniques': "**👨‍🍳 Cooking Techniques:**",
+            'ingredients': "**🥘 Ingredients and Substitutions:**",
+            'equipment': "**🔪 Equipment and Tools:**",
+            'tips_tricks': "**💡 Tips and Tricks:**",
+            'general': "**📚 General Information:**"
         }
         
         header = topic_headers.get(topic, "**Information:**")