Update entire src directory with fixed __init__.py

second_brain_online/application/__init__.py

@@ -0,0 +1,9 @@
+from . import agents, rag
+
+# Optional import for evaluation - may cause issues in some environments
+try:
+    from .evaluation import evaluate
+    __all__ = ["rag", "agents", "evaluate"]
+except ImportError as e:
+    print(f"Warning: Could not import evaluation module: {e}")
+    __all__ = ["rag", "agents"]

second_brain_online/application/agents/__init__.py

@@ -0,0 +1,3 @@
+from .agents import extract_tool_responses, get_agent
+
+__all__ = ["get_agent", "extract_tool_responses"]

second_brain_online/application/agents/agents.py

@@ -0,0 +1,144 @@
+from pathlib import Path
+from typing import Any
+
+import opik
+from loguru import logger
+from opik import opik_context
+from smolagents import LiteLLMModel, MessageRole, MultiStepAgent, ToolCallingAgent
+
+from second_brain_online.config import settings
+
+from .tools import (
+    HuggingFaceEndpointSummarizerTool,
+    MongoDBRetrieverTool,
+    OpenAISummarizerTool,
+    what_can_i_do,
+)
+
+
+def get_agent(retriever_config_path: Path) -> "AgentWrapper":
+    agent = AgentWrapper.build_from_smolagents(
+        retriever_config_path=retriever_config_path
+    )
+
+    return agent
+
+
+class AgentWrapper:
+    def __init__(self, agent: MultiStepAgent) -> None:
+        self.__agent = agent
+
+    @property
+    def input_messages(self) -> list[dict]:
+        return self.__agent.input_messages
+
+    @property
+    def agent_name(self) -> str:
+        return self.__agent.agent_name
+
+    @property
+    def max_steps(self) -> str:
+        return self.__agent.max_steps
+
+    @classmethod
+    def build_from_smolagents(cls, retriever_config_path: Path) -> "AgentWrapper":
+        retriever_tool = MongoDBRetrieverTool(config_path=retriever_config_path)
+        if settings.USE_HUGGINGFACE_DEDICATED_ENDPOINT:
+            logger.warning(
+                f"Using Hugging Face dedicated endpoint as the summarizer with URL: {settings.HUGGINGFACE_DEDICATED_ENDPOINT}"
+            )
+            summarizer_tool = HuggingFaceEndpointSummarizerTool()
+        else:
+            logger.warning(
+                f"Using OpenAI as the summarizer with model: {settings.OPENAI_MODEL_ID}"
+            )
+            summarizer_tool = OpenAISummarizerTool(stream=False)
+
+        model = LiteLLMModel(
+            model_id=settings.OPENAI_MODEL_ID,
+            api_base="https://api.openai.com/v1",
+            api_key=settings.OPENAI_API_KEY,
+        )
+
+        agent = ToolCallingAgent(
+            tools=[what_can_i_do, retriever_tool],  # Remove summarizer - it's redundant
+            model=model,
+            max_steps=2,  # Reduce steps since we removed summarizer
+            verbosity_level=2,
+        )
+
+        return cls(agent)
+
+    @opik.track(name="Agent.run")
+    def run(self, task: str, **kwargs) -> Any:
+        result = self.__agent.run(task, **kwargs)
+
+        model = self.__agent.model
+        metadata = {
+            "system_prompt": self.__agent.system_prompt,
+            "system_prompt_template": self.__agent.system_prompt_template,
+            "tool_description_template": self.__agent.tool_description_template,
+            "tools": self.__agent.tools,
+            "model_id": self.__agent.model.model_id,
+            "api_base": self.__agent.model.api_base,
+            "input_token_count": model.last_input_token_count,
+            "output_token_count": model.last_output_token_count,
+        }
+        if hasattr(self.__agent, "step_number"):
+            metadata["step_number"] = self.__agent.step_number
+        opik_context.update_current_trace(
+            tags=["agent"],
+            metadata=metadata,
+        )
+
+        return result
+
+
+def extract_tool_responses(agent: ToolCallingAgent) -> str:
+    """
+    Extracts and concatenates all tool response contents with numbered observation delimiters.
+
+    Args:
+        input_messages (List[Dict]): List of message dictionaries containing 'role' and 'content' keys
+
+    Returns:
+        str: Tool response contents separated by numbered observation delimiters
+
+    Example:
+        >>> messages = [
+        ...     {"role": MessageRole.TOOL_RESPONSE, "content": "First response"},
+        ...     {"role": MessageRole.USER, "content": "Question"},
+        ...     {"role": MessageRole.TOOL_RESPONSE, "content": "Second response"}
+        ... ]
+        >>> extract_tool_responses(messages)
+        "-------- OBSERVATION 1 --------\nFirst response\n-------- OBSERVATION 2 --------\nSecond response"
+    """
+
+    tool_responses = [
+        msg["content"]
+        for msg in agent.input_messages
+        if msg["role"] == MessageRole.TOOL_RESPONSE
+    ]
+
+    return "\n".join(
+        f"-------- OBSERVATION {i + 1} --------\n{response}"
+        for i, response in enumerate(tool_responses)
+    )
+
+
+class OpikAgentMonitorCallback:
+    def __init__(self) -> None:
+        self.output_state: dict = {}
+
+    def __call__(self, step_log) -> None:
+        input_state = {
+            "agent_memory": step_log.agent_memory,
+            "tool_calls": step_log.tool_calls,
+        }
+        self.output_state = {"observations": step_log.observations}
+
+        self.trace(input_state)
+
+    @opik.track(name="Callback.agent_step")
+    def trace(self, step_log) -> dict:
+        return self.output_state

second_brain_online/application/agents/tools/__init__.py

@@ -0,0 +1,10 @@
+from .mongodb_retriever import MongoDBRetrieverTool
+from .summarizer import HuggingFaceEndpointSummarizerTool, OpenAISummarizerTool
+from .what_can_i_do import what_can_i_do
+
+__all__ = [
+    "what_can_i_do",
+    "MongoDBRetrieverTool",
+    "HuggingFaceEndpointSummarizerTool",
+    "OpenAISummarizerTool",
+]

second_brain_online/application/agents/tools/mongodb_retriever.py

@@ -0,0 +1,148 @@
+import json
+from pathlib import Path
+
+import yaml
+from loguru import logger
+from opik import opik_context, track
+from smolagents import Tool
+
+from second_brain_online.application.rag import get_retriever
+
+
+class MongoDBRetrieverTool(Tool):
+    name = "mongodb_vector_search_retriever"
+    description = """Use this tool to search and retrieve relevant documents from a knowledge base using semantic search.
+    This tool performs similarity-based search to find the most relevant documents matching the query.
+    Best used when you need to:
+    - Find specific information from stored documents
+    - Get context about a topic
+    - Research historical data or documentation
+    The tool will return multiple relevant document snippets."""
+
+    inputs = {
+        "query": {
+            "type": "string",
+            "description": """The search query to find relevant documents for using semantic search.
+            Should be a clear, specific question or statement about the information you're looking for.""",
+        }
+    }
+    output_type = "string"
+
+    def __init__(self, config_path: Path, **kwargs):
+        super().__init__(**kwargs)
+
+        self.config_path = config_path
+        self.retriever = self.__load_retriever(config_path)
+
+    def __load_retriever(self, config_path: Path):
+        config = yaml.safe_load(config_path.read_text())
+        config = config["parameters"]
+
+        return get_retriever(
+            embedding_model_id=config["embedding_model_id"],
+            embedding_model_type=config["embedding_model_type"],
+            retriever_type=config["retriever_type"],
+            k=5,
+            device=config["device"],
+        )
+
+    @track(name="MongoDBRetrieverTool.forward")
+    def forward(self, query: str) -> str:
+        if hasattr(self.retriever, "search_kwargs"):
+            search_kwargs = self.retriever.search_kwargs
+        else:
+            try:
+                search_kwargs = {
+                    "fulltext_penalty": self.retriever.fulltext_penalty,
+                    "vector_score_penalty": self.retriever.vector_penalty,
+                    "top_k": self.retriever.top_k,
+                }
+            except AttributeError:
+                logger.warning("Could not extract search kwargs from retriever.")
+
+                search_kwargs = {}
+
+        opik_context.update_current_trace(
+            tags=["agent"],
+            metadata={
+                "search": search_kwargs,
+                "embedding_model_id": self.retriever.vectorstore.embeddings.model,
+            },
+        )
+
+        try:
+            query = self.__parse_query(query)
+            relevant_docs = self.retriever.invoke(query)
+
+            formatted_docs = []
+            for i, doc in enumerate(relevant_docs, 1):
+                # Extract metadata
+                title = doc.metadata.get("title", "Untitled")
+                datetime = doc.metadata.get("datetime", "unknown")
+                contextual_summary = doc.metadata.get("contextual_summary", "")
+                marketing_insights = doc.metadata.get("marketing_insights", {})
+                content = doc.page_content.strip()
+                
+                # Format marketing insights if available
+                marketing_insights_text = ""
+                if marketing_insights:
+                    marketing_insights_text = "\n<marketing_insights>\n"
+                    
+                    # Add quotes
+                    quotes = marketing_insights.get("quotes", [])
+                    if quotes:
+                        marketing_insights_text += "<quotes>\n"
+                        for quote in quotes:
+                            marketing_insights_text += f"- \"{quote.get('quote', '')}\" (Sentiment: {quote.get('sentiment', 'Unknown')})\n"
+                        marketing_insights_text += "</quotes>\n"
+                    
+                    # Add key findings
+                    findings = marketing_insights.get("key_findings", [])
+                    if findings:
+                        marketing_insights_text += "<key_findings>\n"
+                        for finding in findings:
+                            marketing_insights_text += f"- {finding.get('finding', '')} (Impact: {finding.get('impact', 'Unknown')})\n"
+                        marketing_insights_text += "</key_findings>\n"
+                    
+                    marketing_insights_text += "</marketing_insights>\n"
+                
+                # Create optimized document structure - truncate content to avoid token overload
+                content_preview = content[:500] + "..." if len(content) > 500 else content
+                formatted_docs.append(
+                    f"""
+<document id="{i}">
+<title>{title}</title>
+<date>{datetime}</date>
+<contextual_summary>
+{contextual_summary}
+</contextual_summary>
+{marketing_insights_text}
+<content>
+{content_preview}
+</content>
+</document>
+"""
+                )
+
+            result = "\n".join(formatted_docs)
+            result = f"""
+<search_results>
+{result}
+</search_results>
+When using context from any document, reference the document title and date for attribution.
+"""
+            return result
+        except Exception:
+            logger.opt(exception=True).debug("Error retrieving documents.")
+
+            return "Error retrieving documents."
+
+    @track(name="MongoDBRetrieverTool.parse_query")
+    def __parse_query(self, query: str) -> str:
+        try:
+            # Try to parse as JSON first
+            query_dict = json.loads(query)
+            return query_dict["query"]
+        except (json.JSONDecodeError, KeyError):
+            # If JSON parsing fails, return the query as-is
+            return query

second_brain_online/application/agents/tools/summarizer.py

@@ -0,0 +1,126 @@
+from openai import OpenAI
+from opik import track
+from smolagents import Tool
+
+from second_brain_online.config import settings
+
+
+class HuggingFaceEndpointSummarizerTool(Tool):
+    name = "huggingface_summarizer"
+    description = """Use this tool to summarize a piece of text. Especially useful when you need to summarize a document."""
+
+    inputs = {
+        "text": {
+            "type": "string",
+            "description": """The text to summarize.""",
+        }
+    }
+    output_type = "string"
+
+    SYSTEM_PROMPT = """
+    
+    Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
+
+    ### Instruction:
+    You are a helpful assistant specialized in summarizing documents. Generate a concise TL;DR summary in markdown format having a maximum of 512 characters of the key findings from the provided documents, highlighting the most significant insights
+
+    ### Input:
+    {content}
+
+    ### Response:
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+        assert settings.HUGGINGFACE_ACCESS_TOKEN is not None, (
+            "HUGGINGFACE_ACCESS_TOKEN is required to use the dedicated endpoint. Add it to the .env file."
+        )
+        assert settings.HUGGINGFACE_DEDICATED_ENDPOINT is not None, (
+            "HUGGINGFACE_DEDICATED_ENDPOINT is required to use the dedicated endpoint. Add it to the .env file."
+        )
+
+        self.__client = OpenAI(
+            base_url=settings.HUGGINGFACE_DEDICATED_ENDPOINT,
+            api_key=settings.HUGGINGFACE_ACCESS_TOKEN,
+        )
+
+    @track
+    def forward(self, text: str) -> str:
+        result = self.__client.chat.completions.create(
+            model="tgi",
+            messages=[
+                {
+                    "role": "user",
+                    "content": self.SYSTEM_PROMPT.format(content=text),
+                },
+            ],
+        )
+
+        return result.choices[0].message.content
+
+
+class OpenAISummarizerTool(Tool):
+    name = "openai_summarizer"
+    description = """Use this tool to summarize search results in XML format. This tool is especially useful when you need to analyze multiple documents from search results. The tool will parse XML search results, identify topics that are directly relevant to the user's query, and create a focused summary with document references. It filters out irrelevant topics to ensure the summary directly answers the user's question."""
+
+    inputs = {
+        "text": {
+            "type": "string",
+            "description": """The text to summarize.""",
+        }
+    }
+    output_type = "string"
+
+    SYSTEM_PROMPT = """You are an expert document analyst specialized in query-focused summarization.
+
+Your task is to analyze search results and create a focused summary that directly answers the user's question.
+
+When you receive XML search results, you should:
+1. Parse ALL documents from the XML structure
+2. Identify topics that are directly relevant to the user's query
+3. Filter out irrelevant topics that don't relate to the question
+4. Group related information by relevant topics
+5. Extract key insights that directly answer the user's question
+6. Include document references with titles and dates when available
+
+Analysis Guidelines:
+- Focus on information that directly answers the user's question
+- Only include topics that are relevant to the query
+- Use specific document titles and dates from the XML metadata when available
+- Ignore irrelevant information like cookie policies, privacy policies, HTTP errors, etc.
+- Create a well-structured, readable summary
+- Group similar topics together when appropriate
+
+Document content:
+{content}
+
+Generate a focused summary that directly answers the user's question, organized by relevant topics with document references. Exclude any topics that don't directly relate to the question."""
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+        self.__client = OpenAI(
+            base_url="https://api.openai.com/v1",
+            api_key=settings.OPENAI_API_KEY,
+        )
+
+    @track
+    def forward(self, text: str) -> str:
+        result = self.__client.chat.completions.create(
+            model=settings.OPENAI_MODEL_ID,
+            messages=[
+                {
+                    "role": "system",
+                    "content": "You are an expert document analyst specialized in query-focused topic-based summarization. You excel at parsing XML search results, identifying relevant topics, and creating structured summaries with proper document references."
+                },
+                {
+                    "role": "user",
+                    "content": self.SYSTEM_PROMPT.format(content=text),
+                },
+            ],
+            temperature=0.1,  # Lower temperature for more consistent, focused output
+            max_tokens=2000,  # Increased token limit for more detailed summaries
+        )
+
+        return result.choices[0].message.content

second_brain_online/application/agents/tools/what_can_i_do.py

@@ -0,0 +1,60 @@
+import opik
+from smolagents import tool
+
+
+@opik.track(name="what_can_i_do")
+@tool
+def what_can_i_do(question: str) -> str:
+    """Returns a comprehensive list of available capabilities and topics in the Second Brain system.
+
+    This tool should be used when:
+    - The user explicitly asks what the system can do
+    - The user asks about available features or capabilities
+    - The user seems unsure about what questions they can ask
+    - The user wants to explore the system's knowledge areas
+
+    This tool should NOT be used when:
+    - The user asks a specific technical question
+    - The user already knows what they want to learn about
+    - The question is about a specific topic covered in the knowledge base
+
+    Args:
+        question: The user's query about system capabilities. While this parameter is required,
+                 the function returns a standard capability list regardless of the specific question.
+
+    Returns:
+        str: A formatted string containing categorized lists of example questions and topics
+             that users can explore within the Second Brain system.
+
+    Examples:
+        >>> what_can_i_do("What can this system do?")
+        >>> what_can_i_do("What kind of questions can I ask?")
+        >>> what_can_i_do("Help me understand what I can learn here")
+    """
+
+    return """
+You can ask questions about the content in your Second Brain, such as:
+
+Architecture and Systems:
+- What is the feature/training/inference (FTI) architecture?
+- How do agentic systems work?
+- Detail how does agent memory work in agentic applications?
+
+LLM Technology:
+- What are LLMs?
+- What is BERT (Bidirectional Encoder Representations from Transformers)?
+- Detail how does RLHF (Reinforcement Learning from Human Feedback) work?
+- What are the top LLM frameworks for building applications?
+- Write me a paragraph on how can I optimize LLMs during inference?
+
+RAG and Document Processing:
+- What tools are available for processing PDFs for LLMs and RAG?
+- What's the difference between vector databases and vector indices?
+- How does document chunk overlap affect RAG performance?
+- What is chunk reranking and why is it important?
+- What are advanced RAG techniques for optimization?
+- How can RAG pipelines be evaluated?
+
+Learning Resources:
+- Can you recommend courses on LLMs and RAG?
+"""

second_brain_online/application/evaluation/__init__.py

@@ -0,0 +1,5 @@
+from .evaluate import evaluate_agent
+from .summary_density_heuristic import SummaryDensityHeuristic
+from .summary_density_judge import SummaryDensityJudge
+
+__all__ = ["evaluate_agent", "SummaryDensityHeuristic", "SummaryDensityJudge"]

second_brain_online/application/evaluation/evaluate.py

@@ -0,0 +1,73 @@
+from pathlib import Path
+
+from loguru import logger
+from opik.evaluation import evaluate
+from opik.evaluation.metrics import AnswerRelevance, Hallucination, Moderation
+
+from second_brain_online import opik_utils
+from second_brain_online.application.agents import agents, extract_tool_responses
+from second_brain_online.config import settings
+
+from .summary_density_heuristic import SummaryDensityHeuristic
+from .summary_density_judge import SummaryDensityJudge
+
+opik_utils.configure()
+
+
+def evaluate_agent(prompts: list[str], retriever_config_path: Path) -> None:
+    assert settings.COMET_API_KEY, (
+        "COMET_API_KEY is not set. We need it to track the experiment with Opik."
+    )
+
+    logger.info("Starting evaluation...")
+    logger.info(f"Evaluating agent with {len(prompts)} prompts.")
+
+    def evaluation_task(x: dict) -> dict:
+        """Call agentic app logic to evaluate."""
+
+        agent = agents.get_agent(retriever_config_path=retriever_config_path)
+        response = agent.run(x["input"])
+        context = extract_tool_responses(agent)
+
+        return {
+            "input": x["input"],
+            "context": context,
+            "output": response,
+        }
+
+    # Get or create dataset
+    dataset_name = "second_brain_rag_agentic_app_evaluation_dataset"
+    dataset = opik_utils.get_or_create_dataset(name=dataset_name, prompts=prompts)
+
+    # Evaluate
+    agent = agents.get_agent(retriever_config_path=retriever_config_path)
+    experiment_config = {
+        "model_id": settings.OPENAI_MODEL_ID,
+        "retriever_config_path": retriever_config_path,
+        "agent_config": {
+            "max_steps": agent.max_steps,
+            "agent_name": agent.agent_name,
+        },
+    }
+    scoring_metrics = [
+        Hallucination(),
+        AnswerRelevance(),
+        Moderation(),
+        SummaryDensityHeuristic(),
+        SummaryDensityJudge(),
+    ]
+
+    if dataset:
+        logger.info("Evaluation details:")
+        logger.info(f"Dataset: {dataset_name}")
+        logger.info(f"Metrics: {[m.__class__.__name__ for m in scoring_metrics]}")
+
+        evaluate(
+            dataset=dataset,
+            task=evaluation_task,
+            scoring_metrics=scoring_metrics,
+            experiment_config=experiment_config,
+            task_threads=2,
+        )
+    else:
+        logger.error("Can't run the evaluation as the dataset items are empty.")

second_brain_online/application/evaluation/summary_density_heuristic.py

@@ -0,0 +1,87 @@
+from typing import Any
+
+from opik.evaluation.metrics import base_metric, score_result
+
+
+class SummaryDensityHeuristic(base_metric.BaseMetric):
+    """
+    A metric that evaluates whether an LLM's output has appropriate length and density.
+
+    This metric uses an heuristic to determine if the output length is appropriate for the given instruction.
+    It returns a normalized score between 0 and 1, where:
+    - 0.0 (Poor): Output is either too short and incomplete, or too long with unnecessary information
+    - 0.5 (Good): Output has decent length balance but still slightly too short or too long
+    - 1.0 (Excellent): Output length is appropriate, answering the question concisely without being verbose
+    """
+
+    def __init__(
+        self,
+        name: str = "summary_density_heuristic",
+        min_length: int = 128,
+        max_length: int = 1024,
+    ) -> None:
+        self.name = name
+        self.min_length = min_length
+        self.max_length = max_length
+
+    def score(
+        self, input: str, output: str, **ignored_kwargs: Any
+    ) -> score_result.ScoreResult:
+        """
+        Score the output of an LLM.
+
+        Args:
+            input: The input prompt given to the LLM.
+            output: The output of an LLM to score.
+            **ignored_kwargs: Any additional keyword arguments.
+
+        Returns:
+            ScoreResult: The computed score with explanation.
+        """
+
+        length_score = self._compute_length_score(output)
+
+        reason = f"Output length: {len(output)} chars. "
+        if length_score == 1.0:
+            reason += "Length is within ideal range."
+        elif length_score >= 0.5:
+            reason += "Length is slightly outside ideal range."
+        else:
+            reason += "Length is significantly outside ideal range."
+
+        return score_result.ScoreResult(
+            name=self.name,
+            value=length_score,
+            reason=reason,
+        )
+
+    def _compute_length_score(self, text: str) -> float:
+        """
+        Compute a score based on text length relative to min and max boundaries.
+
+        Args:
+            text: The text to evaluate.
+
+        Returns:
+            float: A score between 0 and 1, where:
+                - 0.0: Text length is significantly outside the boundaries
+                - 0.5: Text length is slightly outside the boundaries
+                - 1.0: Text length is within the ideal range
+        """
+        length = len(text)
+
+        # If length is within bounds, return perfect score
+        if self.min_length <= length <= self.max_length:
+            return 1.0
+
+        if length < self.min_length:
+            deviation = (self.min_length - length) / self.min_length
+        else:
+            deviation = (length - self.max_length) / self.max_length
+
+        # Convert deviation to a score between 0 and 1
+        # deviation <= 0.5 -> score between 0.5 and 1.0
+        # deviation > 0.5 -> score between 0.0 and 0.5
+        score = max(0.0, 1.0 - deviation)
+
+        return score

second_brain_online/application/evaluation/summary_density_judge.py

@@ -0,0 +1,117 @@
+import json
+from typing import Any
+
+from opik.evaluation.metrics import base_metric, exceptions, score_result
+from opik.evaluation.models import LiteLLMChatModel
+from pydantic import BaseModel
+
+from second_brain_online.config import settings
+
+
+class LLMJudgeStyleOutputResult(BaseModel):
+    score: int
+    reason: str
+
+
+class SummaryDensityJudge(base_metric.BaseMetric):
+    """
+    A metric that evaluates whether an LLM's output has appropriate length and density.
+
+    This metric uses another LLM to judge if the output length is appropriate for the given instruction.
+    It returns a normalized score between 0 and 1, where:
+    - 0.0 (Poor): Output is either too short and incomplete, or too long with unnecessary information
+    - 0.5 (Good): Output has decent length balance but still slightly too short or too long
+    - 1.0 (Excellent): Output length is appropriate, answering the question concisely without being verbose
+    """
+
+    def __init__(
+        self,
+        name: str = "summary_density_judge",
+        model_name: str = settings.OPENAI_MODEL_ID,
+    ) -> None:
+        self.name = name
+        self.llm_client = LiteLLMChatModel(model_name=model_name)
+        self.prompt_template = """
+        You are an impartial expert judge. Evaluate the quality of a given answer to an instruction based on how long the answer it is. 
+
+How to decide whether the lengths of the answer is appropriate:
+1 (Poor): Too short, does not answer the question OR too long, it contains too much noise and unrequired information, where the answer could be more concise.
+2 (Good): Good lengthbalance of the answer, but the answer is still too short OR too long.
+3 (Excellent): The length of the answer is appropriate, it answers the question and is not too long or too short.
+
+Example of bad answer that is too short: 
+<answer>
+LangChain, LlamaIndex, Haystack
+</answer>
+
+Example of bad answer that is too long:
+<answer>
+LangChain is a powerful and versatile framework designed specifically for building sophisticated LLM applications. It provides comprehensive abstractions for essential components like prompting, memory management, agent behaviors, and chain orchestration. The framework boasts an impressive ecosystem with extensive integrations across various tools and services, making it highly flexible for diverse use cases. However, this extensive functionality comes with a steeper learning curve that might require dedicated time to master.
+
+LlamaIndex (which was formerly known as GPTIndex) has carved out a specialized niche in the LLM tooling landscape, focusing primarily on data ingestion and advanced indexing capabilities for Large Language Models. It offers a rich set of sophisticated mechanisms to structure and query your data, including vector stores for semantic similarity search, keyword indices for traditional text matching, and tree indices for hierarchical data organization. While it particularly shines in Retrieval-Augmented Generation (RAG) applications, its comprehensive feature set might be excessive for more straightforward implementation needs.
+
+Haystack stands out as a robust end-to-end framework that places particular emphasis on question-answering systems and semantic search capabilities. It provides a comprehensive suite of document processing tools and comes equipped with production-ready pipelines that can be deployed with minimal configuration. The framework includes advanced features like multi-stage retrieval, document ranking, and reader-ranker architectures. While these capabilities make it powerful for complex information retrieval tasks, new users might find the initial configuration and architecture decisions somewhat challenging to navigate.
+
+Each of these frameworks brings unique strengths to the table while sharing some overlapping functionality. The choice between them often depends on specific use cases, technical requirements, and team expertise. LangChain offers the broadest general-purpose toolkit, LlamaIndex excels in data handling and RAG, while Haystack provides the most streamlined experience for question-answering systems.
+</answer>
+
+Example of excellent answer that is appropriate:
+<answer>
+1. LangChain is a powerful framework for building LLM applications that provides abstractions for prompting, memory, agents, and chains. It has extensive integrations with various tools and services, making it highly flexible but potentially complex to learn. 
+2. LlamaIndex specializes in data ingestion and indexing for LLMs, offering sophisticated ways to structure and query your data through vector stores, keyword indices, and tree indices. It excels at RAG applications but may be overkill for simpler use cases. 
+3. Haystack is an end-to-end framework focused on question-answering and semantic search, with strong document processing capabilities and ready-to-use pipelines. While powerful, its learning curve can be steep for beginners. 
+</answer>
+
+Instruction: {input}
+
+Answer: {output}
+
+Provide your evaluation in JSON format with the following structure:
+{{
+    "accuracy": {{
+        "reason": "...",
+        "score": 0
+    }},
+    "style": {{
+        "reason": "...",
+        "score": 0
+    }}
+}}
+"""
+
+    def score(self, input: str, output: str, **ignored_kwargs: Any):
+        """
+        Score the output of an LLM.
+
+        Args:
+            output: The output of an LLM to score.
+            **ignored_kwargs: Any additional keyword arguments. This is important so that the metric can be used in the `evaluate` function.
+        """
+
+        prompt = self.prompt_template.format(input=input, output=output)
+
+        model_output = self.llm_client.generate_string(
+            input=prompt, response_format=LLMJudgeStyleOutputResult
+        )
+
+        return self._parse_model_output(model_output)
+
+    def _parse_model_output(self, content: str) -> score_result.ScoreResult:
+        try:
+            dict_content = json.loads(content)
+        except Exception:
+            raise exceptions.MetricComputationError("Failed to parse the model output.")
+
+        score = dict_content["score"]
+        try:
+            assert 1 <= score <= 3, f"Invalid score value: {score}"
+        except AssertionError as e:
+            raise exceptions.MetricComputationError(str(e))
+
+        score = (score - 1) / 2.0  # Normalize the score to be between 0 and 1
+
+        return score_result.ScoreResult(
+            name=self.name,
+            value=score,
+            reason=dict_content["reason"],
+        )

second_brain_online/application/rag/__init__.py

@@ -0,0 +1,11 @@
+from .embeddings import EmbeddingModelType, get_embedding_model
+from .retrievers import RetrieverType, get_retriever
+from .splitters import get_splitter
+
+__all__ = [
+    "get_retriever",
+    "get_splitter",
+    "EmbeddingModelType",
+    "get_embedding_model",
+    "RetrieverType",
+]

second_brain_online/application/rag/embeddings.py

@@ -0,0 +1,74 @@
+from typing import Literal, Union
+
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_openai import OpenAIEmbeddings
+
+EmbeddingModelType = Literal["openai", "huggingface"]
+EmbeddingsModel = Union[OpenAIEmbeddings, HuggingFaceEmbeddings]
+
+
+def get_embedding_model(
+    model_id: str,
+    model_type: EmbeddingModelType = "huggingface",
+    device: str = "cpu",
+) -> EmbeddingsModel:
+    """Gets an instance of the configured embedding model.
+
+    The function returns either an OpenAI or HuggingFace embedding model based on the
+    provided model type.
+
+    Args:
+        model_id (str): The ID/name of the embedding model to use
+        model_type (EmbeddingModelType): The type of embedding model to use.
+            Must be either "openai" or "huggingface". Defaults to "huggingface"
+        device (str): The device to use for the embedding model. Defaults to "cpu"
+
+    Returns:
+        EmbeddingsModel: An embedding model instance based on the configuration settings
+
+    Raises:
+        ValueError: If model_type is not "openai" or "huggingface"
+    """
+
+    if model_type == "openai":
+        return get_openai_embedding_model(model_id)
+    elif model_type == "huggingface":
+        return get_huggingface_embedding_model(model_id, device)
+    else:
+        raise ValueError(f"Invalid embedding model type: {model_type}")
+
+
+def get_openai_embedding_model(model_id: str) -> OpenAIEmbeddings:
+    """Gets an OpenAI embedding model instance.
+
+    Args:
+        model_id (str): The ID/name of the OpenAI embedding model to use
+
+    Returns:
+        OpenAIEmbeddings: A configured OpenAI embeddings model instance with
+            special token handling enabled
+    """
+    return OpenAIEmbeddings(
+        model=model_id,
+        allowed_special={"<|endoftext|>"},
+    )
+
+
+def get_huggingface_embedding_model(
+    model_id: str, device: str
+) -> HuggingFaceEmbeddings:
+    """Gets a HuggingFace embedding model instance.
+
+    Args:
+        model_id (str): The ID/name of the HuggingFace embedding model to use
+        device (str): The compute device to run the model on (e.g. "cpu", "cuda")
+
+    Returns:
+        HuggingFaceEmbeddings: A configured HuggingFace embeddings model instance
+            with remote code trust enabled and embedding normalization disabled
+    """
+    return HuggingFaceEmbeddings(
+        model_name=model_id,
+        model_kwargs={"device": device, "trust_remote_code": True},
+        encode_kwargs={"normalize_embeddings": False},
+    )

second_brain_online/application/rag/retrievers.py

@@ -0,0 +1,83 @@
+from typing import Literal, Union
+
+from langchain_mongodb import MongoDBAtlasVectorSearch
+from langchain_mongodb.retrievers import (
+    MongoDBAtlasHybridSearchRetriever,
+    MongoDBAtlasParentDocumentRetriever,
+)
+from loguru import logger
+
+from second_brain_online.config import settings
+
+from .embeddings import EmbeddingModelType, EmbeddingsModel, get_embedding_model
+from .splitters import get_splitter
+
+# Add these type definitions at the top of the file
+RetrieverType = Literal["contextual", "parent"]
+RetrieverModel = Union[
+    MongoDBAtlasHybridSearchRetriever, MongoDBAtlasParentDocumentRetriever
+]
+
+
+def get_retriever(
+    embedding_model_id: str,
+    embedding_model_type: EmbeddingModelType = "huggingface",
+    retriever_type: RetrieverType = "contextual",
+    k: int = 3,
+    device: str = "cpu",
+) -> RetrieverModel:
+    logger.info(
+        f"Getting '{retriever_type}' retriever for '{embedding_model_type}' - '{embedding_model_id}' on '{device}' "
+        f"with {k} top results"
+    )
+
+    embedding_model = get_embedding_model(
+        embedding_model_id, embedding_model_type, device
+    )
+
+    if retriever_type == "contextual":
+        return get_hybrid_search_retriever(embedding_model, k)
+    elif retriever_type == "parent":
+        return get_parent_document_retriever(embedding_model, k)
+    else:
+        raise ValueError(f"Invalid retriever type: {retriever_type}")
+
+
+def get_hybrid_search_retriever(
+    embedding_model: EmbeddingsModel, k: int
+) -> MongoDBAtlasHybridSearchRetriever:
+    vectorstore = MongoDBAtlasVectorSearch.from_connection_string(
+        connection_string=settings.MONGODB_URI,
+        embedding=embedding_model,
+        namespace=f"{settings.MONGODB_DATABASE_NAME}.{settings.MONGODB_COLLECTION_NAME}",
+        text_key="chunk",
+        embedding_key="embedding",
+        relevance_score_fn="dotProduct",
+    )
+
+    retriever = MongoDBAtlasHybridSearchRetriever(
+        vectorstore=vectorstore,
+        search_index_name="chunk_text_search",
+        top_k=k,
+        vector_penalty=50,
+        fulltext_penalty=50,
+    )
+
+    return retriever
+
+
+def get_parent_document_retriever(
+    embedding_model: EmbeddingsModel, k: int = 3
+) -> MongoDBAtlasParentDocumentRetriever:
+    retriever = MongoDBAtlasParentDocumentRetriever.from_connection_string(
+        connection_string=settings.MONGODB_URI,
+        embedding_model=embedding_model,
+        child_splitter=get_splitter(200),
+        parent_splitter=get_splitter(800),
+        database_name=settings.MONGODB_DATABASE_NAME,
+        collection_name=settings.MONGODB_COLLECTION_NAME,
+        text_key="chunk",
+        search_kwargs={"k": k},
+    )
+
+    return retriever

second_brain_online/application/rag/splitters.py

@@ -0,0 +1,28 @@
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from loguru import logger
+
+
+def get_splitter(chunk_size: int) -> RecursiveCharacterTextSplitter:
+    """Returns a token-based text splitter with overlap.
+
+    Args:
+        chunk_size: Number of tokens for each text chunk.
+        summarization_type: Type of summarization to use ("contextual" or "simple").
+        **kwargs: Additional keyword arguments passed to the summarization agent.
+
+    Returns:
+        RecursiveCharacterTextSplitter: A configured text splitter instance with
+            summarization capabilities.
+    """
+
+    chunk_overlap = int(0.15 * chunk_size)
+
+    logger.info(
+        f"Getting splitter with chunk size: {chunk_size} and overlap: {chunk_overlap}"
+    )
+
+    return RecursiveCharacterTextSplitter.from_tiktoken_encoder(
+        encoding_name="cl100k_base",
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+    )

second_brain_online/application/ui/__init__.py

@@ -0,0 +1,3 @@
+from .custom_gradio_ui import CustomGradioUI
+
+__all__ = ["CustomGradioUI"]

second_brain_online/application/ui/custom_gradio_ui.py

@@ -0,0 +1,287 @@
+import json
+import re
+from typing import Any, Dict, List, Tuple
+
+import gradio as gr
+from smolagents import ToolCallingAgent
+
+
+class CustomGradioUI:
+    """Custom Gradio UI for better formatting of agent responses with source attribution."""
+    
+    def __init__(self, agent: ToolCallingAgent):
+        self.agent = agent
+        self.setup_ui()
+    
+    def setup_ui(self):
+        """Setup the Gradio interface with custom components."""
+        with gr.Blocks(
+            title="Second Brain AI Assistant",
+            theme=gr.themes.Soft(),
+            css="""
+            .source-card {
+                border: 1px solid #e0e0e0;
+                border-radius: 8px;
+                padding: 12px;
+                margin: 8px 0;
+                background-color: #f8f9fa;
+            }
+            .source-title {
+                font-weight: bold;
+                color: #2c3e50;
+                margin-bottom: 4px;
+            }
+            .source-date {
+                font-size: 0.9em;
+                color: #6c757d;
+                margin-bottom: 8px;
+            }
+            .answer-section {
+                background-color: #ffffff;
+                border: 1px solid #dee2e6;
+                border-radius: 8px;
+                padding: 16px;
+                margin-bottom: 16px;
+            }
+            .tool-usage {
+                background-color: #e3f2fd;
+                border-left: 4px solid #2196f3;
+                padding: 8px 12px;
+                margin: 8px 0;
+                border-radius: 4px;
+                font-size: 0.9em;
+            }
+            """
+        ) as self.interface:
+            
+            gr.Markdown("# 🧠 Second Brain AI Assistant")
+            gr.Markdown("Ask questions about your documents and get AI-powered insights with source attribution.")
+            
+            with gr.Row():
+                with gr.Column(scale=4):
+                    self.query_input = gr.Textbox(
+                        label="Ask a question",
+                        placeholder="What pricing objections were raised in the meetings?",
+                        lines=2
+                    )
+                with gr.Column(scale=1):
+                    self.submit_btn = gr.Button("Ask", variant="primary", size="lg")
+            
+            with gr.Row():
+                with gr.Column():
+                    self.answer_output = gr.HTML(label="Answer")
+                    self.sources_output = gr.HTML(label="Sources")
+                    self.tools_output = gr.HTML(label="Tools Used")
+            
+            with gr.Accordion("🔍 Debug: Raw Response", open=False):
+                self.debug_output = gr.Textbox(
+                    label="Raw Agent Response",
+                    lines=10,
+                    max_lines=20,
+                    interactive=False
+                )
+            
+            # Event handlers
+            self.submit_btn.click(
+                fn=self.process_query,
+                inputs=[self.query_input],
+                outputs=[self.answer_output, self.sources_output, self.tools_output, self.debug_output]
+            )
+            
+            self.query_input.submit(
+                fn=self.process_query,
+                inputs=[self.query_input],
+                outputs=[self.answer_output, self.sources_output, self.tools_output, self.debug_output]
+            )
+    
+    def process_query(self, query: str) -> Tuple[str, str, str, str]:
+        """Process the user query and return formatted response components."""
+        if not query.strip():
+            return "", "", "", ""
+        
+        try:
+            # Run the agent
+            result = self.agent.run(query)
+            
+            # Parse the result
+            answer, sources, tools_used = self.parse_agent_response(result)
+            
+            # Debug information
+            print(f"DEBUG - Raw result: {str(result)[:200]}...")
+            print(f"DEBUG - Parsed answer: {answer[:100]}...")
+            print(f"DEBUG - Sources found: {len(sources)}")
+            print(f"DEBUG - Tools found: {tools_used}")
+            
+            # Format outputs
+            answer_html = self.format_answer(answer)
+            sources_html = self.format_sources(sources)
+            tools_html = self.format_tools(tools_used)
+            debug_text = str(result)
+            
+            return answer_html, sources_html, tools_html, debug_text
+            
+        except Exception as e:
+            error_msg = f"<div style='color: #dc3545; padding: 12px; border: 1px solid #f5c6cb; border-radius: 4px; background-color: #f8d7da;'>Error: {str(e)}</div>"
+            return error_msg, "", "", str(e)
+    
+    def parse_agent_response(self, result: Any) -> Tuple[str, List[Dict], List[str]]:
+        """Parse the agent response to extract answer, sources, and tools used."""
+        answer = ""
+        sources = []
+        tools_used = []
+        
+        # Convert result to string if it's not already
+        result_str = str(result)
+        
+        # Extract tool usage from the result first
+        # Pattern 1: 🛠️ Used tool toolname
+        tool_pattern1 = r'🛠️ Used tool (\w+)'
+        tool_matches1 = re.findall(tool_pattern1, result_str)
+        
+        # Pattern 2: Calling tool: 'toolname'
+        tool_pattern2 = r"Calling tool:\s*'([^']+)'"
+        tool_matches2 = re.findall(tool_pattern2, result_str)
+        
+        # Combine both patterns
+        all_tool_matches = tool_matches1 + tool_matches2
+        tools_used = list(set(all_tool_matches))  # Remove duplicates
+        
+        # Try multiple patterns to extract the answer
+        # Pattern 1: JSON format with "answer" key
+        json_match = re.search(r'{"answer":\s*"([^"]+)"}', result_str)
+        if json_match:
+            answer = json_match.group(1)
+            # Unescape the JSON string
+            answer = answer.replace('\\n', '\n').replace('\\"', '"')
+        else:
+            # Pattern 2: Look for "Final answer:" followed by content
+            final_answer_match = re.search(r'Final answer:\s*(.+?)(?=\n\n|\Z)', result_str, re.DOTALL)
+            if final_answer_match:
+                answer = final_answer_match.group(1).strip()
+                # Try to extract JSON from final answer
+                json_in_final = re.search(r'{"answer":\s*"([^"]+)"}', answer)
+                if json_in_final:
+                    answer = json_in_final.group(1).replace('\\n', '\n').replace('\\"', '"')
+            else:
+                # Pattern 3: Use the entire result as answer if no specific pattern matches
+                answer = result_str
+        
+        # Extract sources from the answer text using multiple patterns
+        # Pattern 1: (Document: "Title", Date)
+        source_pattern1 = r'\(Document:\s*"([^"]+)",\s*([^)]+)\)'
+        source_matches1 = re.findall(source_pattern1, answer)
+        
+        # Pattern 2: (Document: Title, Date) - without quotes
+        source_pattern2 = r'\(Document:\s*([^,]+),\s*([^)]+)\)'
+        source_matches2 = re.findall(source_pattern2, answer)
+        
+        # Pattern 3: (Document 1, Date) - numbered format
+        source_pattern3 = r'\(Document\s+(\d+),\s*([^)]+)\)'
+        source_matches3 = re.findall(source_pattern3, answer)
+        
+        # Pattern 4: (from "Title" on Date) - new format seen in output
+        source_pattern4 = r'\(from\s+"([^"]+)"\s+on\s+([^)]+)\)'
+        source_matches4 = re.findall(source_pattern4, answer)
+        
+        # Pattern 5: (from "Title" on Date) - without quotes
+        source_pattern5 = r'\(from\s+([^"]+)\s+on\s+([^)]+)\)'
+        source_matches5 = re.findall(source_pattern5, answer)
+        
+        # Combine all patterns
+        all_source_matches = source_matches1 + source_matches2 + source_matches3 + source_matches4 + source_matches5
+        
+        for doc_title, doc_date in all_source_matches:
+            # Clean up the title and date
+            clean_title = doc_title.strip().strip('"')
+            clean_date = doc_date.strip()
+            
+            # Handle numbered documents (Document 1, Document 2, etc.)
+            if clean_title.isdigit():
+                clean_title = f"Document {clean_title}"
+            
+            sources.append({
+                "title": clean_title,
+                "date": clean_date
+            })
+        
+        # Remove duplicates based on title and date
+        unique_sources = []
+        seen = set()
+        for source in sources:
+            key = (source["title"], source["date"])
+            if key not in seen:
+                seen.add(key)
+                unique_sources.append(source)
+        
+        return answer, unique_sources, tools_used
+    
+    def format_answer(self, answer: str) -> str:
+        """Format the answer with proper HTML structure."""
+        if not answer:
+            return "<div class='answer-section'><p>No answer provided.</p></div>"
+        
+        # Remove source references from the answer text for cleaner display
+        answer = re.sub(r'\(Document:[^)]+\)', '', answer)
+        
+        # Clean up extra whitespace
+        answer = re.sub(r'\s+', ' ', answer).strip()
+        
+        # Format numbered lists and bullet points
+        answer = re.sub(r'\n\s*\d+\.\s*', '<br><br><strong>', answer)  # Numbered lists
+        answer = re.sub(r'\n\s*•\s*', '<br>• ', answer)  # Bullet points
+        answer = re.sub(r'\n\s*-\s*', '<br>• ', answer)  # Dash points
+        
+        # Format bold text (markdown style)
+        answer = re.sub(r'\*\*(.*?)\*\*', r'<strong>\1</strong>', answer)
+        
+        # Format line breaks
+        answer = answer.replace('\n', '<br>')
+        
+        # Clean up multiple line breaks
+        answer = re.sub(r'(<br>){3,}', '<br><br>', answer)
+        
+        return f"""
+        <div class='answer-section'>
+            <h3>📝 Answer</h3>
+            <div style='line-height: 1.6; font-size: 16px;'>{answer}</div>
+        </div>
+        """
+    
+    def format_sources(self, sources: List[Dict]) -> str:
+        """Format the sources with proper HTML structure."""
+        if not sources:
+            return "<div><h3>📚 Sources</h3><p>No sources found.</p></div>"
+        
+        sources_html = "<div><h3>📚 Sources</h3>"
+        
+        for i, source in enumerate(sources, 1):
+            sources_html += f"""
+            <div class='source-card'>
+                <div class='source-title'>{i}. {source['title']}</div>
+                <div class='source-date'>📅 {source['date']}</div>
+            </div>
+            """
+        
+        sources_html += "</div>"
+        return sources_html
+    
+    def format_tools(self, tools_used: List[str]) -> str:
+        """Format the tools used with proper HTML structure."""
+        if not tools_used:
+            return "<div><h3>🛠️ Tools Used</h3><p>No tools used.</p></div>"
+        
+        tools_html = "<div><h3>🛠️ Tools Used</h3>"
+        
+        for tool in tools_used:
+            tools_html += f"""
+            <div class='tool-usage'>
+                🔧 {tool.replace('_', ' ').title()}
+            </div>
+            """
+        
+        tools_html += "</div>"
+        return tools_html
+    
+    def launch(self, **kwargs):
+        """Launch the Gradio interface."""
+        return self.interface.launch(**kwargs)

second_brain_online/config.py

@@ -0,0 +1,76 @@
+from loguru import logger
+from pydantic import Field, field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """
+    A Pydantic-based settings class for managing application configurations.
+    """
+
+    # --- Pydantic Settings ---
+    model_config: SettingsConfigDict = SettingsConfigDict(
+        env_file=".env", env_file_encoding="utf-8"
+    )
+
+    # --- Comet ML & Opik Configuration ---
+    COMET_API_KEY: str | None = Field(
+        default=None, description="API key for Comet ML and Opik services."
+    )
+    COMET_PROJECT: str = Field(
+        default="second_brain_course",
+        description="Project name for Comet ML and Opik tracking.",
+    )
+
+    # --- Hugging Face Configuration ---
+    HUGGINGFACE_ACCESS_TOKEN: str | None = Field(
+        default=None, description="Access token for Hugging Face API authentication."
+    )
+    USE_HUGGINGFACE_DEDICATED_ENDPOINT: bool = Field(
+        default=False,
+        description="Whether to use the dedicated endpoint for summarizing responses. If True, we will use the dedicated endpoint instead of OpenAI.",
+    )
+    HUGGINGFACE_DEDICATED_ENDPOINT: str | None = Field(
+        default=None,
+        description="Dedicated endpoint URL for real-time inference. "
+        "If provided, we will use the dedicated endpoint instead of OpenAI. "
+        "For example, https://um18v2aeit3f6g1b.eu-west-1.aws.endpoints.huggingface.cloud/v1/, "
+        "with /v1 after the endpoint URL.",
+    )
+
+    # --- MongoDB Atlas Configuration ---
+    MONGODB_DATABASE_NAME: str = Field(
+        default="second_brain_course",
+        description="Name of the MongoDB database.",
+    )
+    MONGODB_COLLECTION_NAME: str = Field(
+        default="rag",
+        description="Name of the MongoDB collection for RAG documents.",
+    )
+    MONGODB_URI: str = Field(
+        default="mongodb+srv://contextdb:HOqIgSH01CoEiMb1@cluster0.d9cmff.mongodb.net/",
+        description="Connection URI for the MongoDB Atlas instance.",
+    )
+
+    # --- OpenAI API Configuration ---
+    OPENAI_API_KEY: str = Field(
+        description="API key for OpenAI service authentication.",
+    )
+    OPENAI_MODEL_ID: str = Field(
+        default="gpt-4o", description="Identifier for the OpenAI model to be used."
+    )
+
+    @field_validator("OPENAI_API_KEY")
+    @classmethod
+    def check_not_empty(cls, value: str, info) -> str:
+        if not value or value.strip() == "":
+            logger.error(f"{info.field_name} cannot be empty.")
+            raise ValueError(f"{info.field_name} cannot be empty.")
+        return value
+
+
+try:
+    settings = Settings()
+except Exception as e:
+    logger.error(f"Failed to load configuration: {e}")
+    raise SystemExit(e)

second_brain_online/opik_utils.py

@@ -0,0 +1,77 @@
+import os
+
+import opik
+from loguru import logger
+from opik.configurator.configure import OpikConfigurator
+
+from second_brain_online.config import settings
+
+
+def configure() -> None:
+    if settings.COMET_API_KEY and settings.COMET_PROJECT:
+        try:
+            client = OpikConfigurator(api_key=settings.COMET_API_KEY)
+            default_workspace = client._get_default_workspace()
+        except Exception:
+            logger.warning(
+                "Default workspace not found. Setting workspace to None and enabling interactive mode."
+            )
+            default_workspace = None
+
+        os.environ["OPIK_PROJECT_NAME"] = settings.COMET_PROJECT
+
+        opik.configure(
+            api_key=settings.COMET_API_KEY,
+            workspace=default_workspace,
+            use_local=False,
+            force=True,
+        )
+        logger.info(
+            f"Opik configured successfully using workspace '{default_workspace}'"
+        )
+    else:
+        logger.warning(
+            "COMET_API_KEY and COMET_PROJECT are not set. Set them to enable prompt monitoring with Opik (powered by Comet ML)."
+        )
+
+
+def get_or_create_dataset(name: str, prompts: list[str]) -> opik.Dataset | None:
+    client = opik.Opik()
+    try:
+        dataset = client.get_dataset(name=name)
+    except Exception:
+        dataset = None
+
+    if dataset:
+        logger.warning(f"Dataset '{name}' already exists. Skipping dataset creation.")
+
+        return dataset
+
+    assert prompts, "Prompts are required to create a dataset."
+
+    dataset_items = []
+    for prompt in prompts:
+        dataset_items.append(
+            {
+                "input": prompt,
+            }
+        )
+
+    dataset = create_dataset(
+        name=name,
+        description="Dataset for evaluating the agentic app.",
+        items=dataset_items,
+    )
+
+    return dataset
+
+
+def create_dataset(name: str, description: str, items: list[dict]) -> opik.Dataset:
+    client = opik.Opik()
+
+    dataset = client.get_or_create_dataset(name=name, description=description)
+    dataset.insert(items)
+
+    dataset = client.get_dataset(name=name)
+
+    return dataset