| import gradio as gr | |
| import evaluate | |
| l3score = evaluate.load("nhop/L3Score") | |
| def compute_l3score(api_key, provider, model, questions, predictions, references): | |
| try: | |
| result = l3score.compute( | |
| questions=[q.strip() for q in questions.split("\n") if q.strip()], | |
| predictions=[p.strip() for p in predictions.split("\n") if p.strip()], | |
| references=[r.strip() for r in references.split("\n") if r.strip()], | |
| api_key=api_key, | |
| provider=provider, | |
| model=model | |
| ) | |
| return result | |
| except Exception as e: | |
| return {"error": str(e)} | |
| with gr.Blocks() as demo: | |
| gr.Markdown(r""" | |
| # Metric: L3Score | |
| """) | |
| with gr.Row(): | |
| api_key = gr.Textbox(label="API Key", type="password") | |
| provider = gr.Dropdown(label="Provider", choices=["openai", "deepseek", "xai"], value="openai") | |
| model = gr.Textbox(label="Model", value="gpt-4o-mini") | |
| with gr.Row(): | |
| questions = gr.Textbox(label="Questions (one per line)", lines=4, placeholder="What is the capital of France?") | |
| predictions = gr.Textbox(label="Predictions (one per line)", lines=4, placeholder="Paris") | |
| references = gr.Textbox(label="References (one per line)", lines=4, placeholder="Paris") | |
| compute_button = gr.Button("Compute L3Score") | |
| output = gr.JSON(label="L3Score Result") | |
| compute_button.click( | |
| fn=compute_l3score, | |
| inputs=[api_key, provider, model, questions, predictions, references], | |
| outputs=output | |
| ) | |
| gr.Markdown(r""" | |
| ## 📌 Description | |
| **L3Score** evaluates how semantically close a model-generated answer is to a reference answer for a given question. It prompts a **language model as a judge** using: | |
| ```text | |
| You are given a question, ground-truth answer, and a candidate answer. | |
| Question: {{question}} | |
| Ground-truth answer: {{gt}} | |
| Candidate answer: {{answer}} | |
| Is the semantic meaning of the ground-truth and candidate answers similar? | |
| Answer in one word - Yes or No. | |
| ``` | |
| The model's **log-probabilities** for "Yes" and "No" tokens are used to compute the score. | |
| ---""") | |
| gr.Markdown(""" ## 🧮 Scoring Logic""") | |
| gr.Markdown(r"""Let $l_{\text{yes}}$ and $l_{\text{no}}$ be the log-probabilities of 'Yes' and 'No', respectively.""",latex_delimiters=[ {"left": "$", "right": "$", "display": False }]) | |
| gr.Markdown(r""" | |
| - If neither token is in the top-5: | |
| $$ | |
| \text{L3Score} = 0 | |
| $$ | |
| - If both are present: | |
| $$ | |
| \text{L3Score} = \frac{\exp(l_{\text{yes}})}{\exp(l_{\text{yes}}) + \exp(l_{\text{no}})} | |
| $$ | |
| - If only one is present, the missing token’s probability is estimated using the minimum of: | |
| - remaining probability mass apart from the top-5 tokens | |
| - the least likely top-5 token | |
| The score ranges from 0 to 1, where 1 indicates the highest confidence by the LLM that the predicted and reference answers are semantically equivalent. | |
| See [SPIQA paper](https://arxiv.org/pdf/2407.09413) for details. | |
| --- | |
| ## 🚀 How to Use | |
| ```python | |
| import evaluate | |
| l3score = evaluate.load("nhop/L3Score") | |
| questions = ["What is the capital of France?", "What is the capital of Germany?"] | |
| predictions = ["Paris", "Moscow"] | |
| references = ["Paris", "Berlin"] | |
| score = l3score.compute( | |
| questions=questions, | |
| predictions=predictions, | |
| references=references, | |
| api_key="your-openai-api-key", | |
| provider="openai", | |
| model="gpt-4o-mini" | |
| ) | |
| print(score) | |
| # {'L3Score': 0.49..., 'Cost': ...} | |
| ``` | |
| --- | |
| ## 🔠 Inputs | |
| | Name | Type | Description | | |
| |--------------|--------------|-----------------------------------------------------------------------------| | |
| | `questions` | `list[str]` | The list of input questions. | | |
| | `predictions`| `list[str]` | Generated answers by the model being evaluated. | | |
| | `references` | `list[str]` | Ground-truth or reference answers. | | |
| | `api_key` | `str` | API key for the selected LLM provider. | | |
| | `provider` | `str` | Must support top-n token log-probabilities. **Default**: openai | | |
| | `model` | `str` | Name of the evaluation LLM. **Default**: gpt-4o-mini | | |
| ## 📄 Output | |
| Calling the `compute` method returns a dictionary containing the L3Score: | |
| ```python | |
| {"L3Score": float, "Cost": float} | |
| ``` | |
| The value is the **average score** over all (question, prediction, reference) triplets and the total cost of all API calls. | |
| --- | |
| ## 📊 Examples | |
| ```python | |
| l3score = evaluate.load("nhop/L3Score") | |
| score = l3score.compute( | |
| questions=["What is the capital of France?"], | |
| predictions=["Paris"], | |
| references=["Paris"], | |
| api_key="your-openai-api-key", | |
| provider="openai", | |
| model="gpt-4o-mini" | |
| ) | |
| # {'L3Score': 0.99..., 'Cost': ...} | |
| score = l3score.compute( | |
| questions=["What is the capital of Germany?"], | |
| predictions=["Moscow"], | |
| references=["Berlin"], | |
| api_key="your-openai-api-key", | |
| provider="openai", | |
| model="gpt-4o-mini" | |
| ) | |
| # {'L3Score': 0.00..., 'Cost': ...} | |
| ``` | |
| --- | |
| ## ⚠️ Limitations and Bias | |
| - Requires models that expose **top-n token log-probabilities** (e.g., OpenAI, DeepSeek, Groq). | |
| - Scores are **only comparable when using the same judge model**. | |
| ## 📖 Citation | |
| ```bibtex | |
| @article{pramanick2024spiqa, | |
| title={SPIQA: A Dataset for Multimodal Question Answering on Scientific Papers}, | |
| author={Pramanick, Shraman and Chellappa, Rama and Venugopalan, Subhashini}, | |
| journal={arXiv preprint arXiv:2407.09413}, | |
| year={2024} | |
| } | |
| ``` | |
| """) | |
| demo.launch() | |