Spaces:
Configuration error
Configuration error
Add my new, shiny module.
Browse files
llm_harness_mistral_arc.py
CHANGED
|
@@ -27,7 +27,13 @@ class llm_harness_mistral_arc(evaluate.Metric):
|
|
| 27 |
citation=_CITATION,
|
| 28 |
inputs_description=_KWARGS_DESCRIPTION,
|
| 29 |
# This defines the format of each prediction and reference
|
| 30 |
-
features=
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
# Homepage of the module for documentation
|
| 32 |
homepage="http://module.homepage",
|
| 33 |
# Additional links to the codebase or references
|
|
@@ -35,7 +41,7 @@ class llm_harness_mistral_arc(evaluate.Metric):
|
|
| 35 |
reference_urls=["http://path.to.reference.url/new_module"]
|
| 36 |
)
|
| 37 |
|
| 38 |
-
def _compute(self, pretrained
|
| 39 |
outputs = lm_eval.simple_evaluate(
|
| 40 |
model="hf",
|
| 41 |
model_args={"pretrained":pretrained},
|
|
|
|
| 27 |
citation=_CITATION,
|
| 28 |
inputs_description=_KWARGS_DESCRIPTION,
|
| 29 |
# This defines the format of each prediction and reference
|
| 30 |
+
features=[
|
| 31 |
+
datasets.Features(
|
| 32 |
+
{
|
| 33 |
+
"pretrained": datasets.Value("string", id="sequence"),
|
| 34 |
+
"tasks": datasets.Sequence(datasets.Value("string", id="sequence"), id="tasks"),
|
| 35 |
+
}
|
| 36 |
+
)],
|
| 37 |
# Homepage of the module for documentation
|
| 38 |
homepage="http://module.homepage",
|
| 39 |
# Additional links to the codebase or references
|
|
|
|
| 41 |
reference_urls=["http://path.to.reference.url/new_module"]
|
| 42 |
)
|
| 43 |
|
| 44 |
+
def _compute(self, pretrained, tasks):
|
| 45 |
outputs = lm_eval.simple_evaluate(
|
| 46 |
model="hf",
|
| 47 |
model_args={"pretrained":pretrained},
|