Update bias_auc.py
Browse files- bias_auc.py +2 -10
bias_auc.py
CHANGED
|
@@ -9,12 +9,9 @@ _DESCRIPTION = """\
|
|
| 9 |
Suite of threshold-agnostic metrics that provide a nuanced view
|
| 10 |
of this unintended bias, by considering the various ways that a
|
| 11 |
classifier’s score distribution can vary across designated groups.
|
| 12 |
-
|
| 13 |
The following are computed:
|
| 14 |
-
|
| 15 |
- BNSP (Background Negative, Subgroup Positive); and
|
| 16 |
- BPSN (Background Positive, Subgroup Negative) AUC
|
| 17 |
-
|
| 18 |
"""
|
| 19 |
|
| 20 |
_CITATION = """\
|
|
@@ -28,18 +25,14 @@ _CITATION = """\
|
|
| 28 |
"""
|
| 29 |
|
| 30 |
_KWARGS_DESCRIPTION = """\
|
| 31 |
-
|
| 32 |
Args:
|
| 33 |
target list[list[str]]: list containing list of group targeted for each item
|
| 34 |
label list[int]: list containing label index for each item
|
| 35 |
output list[list[float]]: list of model output values for each
|
| 36 |
-
|
| 37 |
Returns (for each subgroup in target):
|
| 38 |
'Subgroup' : Subgroup AUC score,
|
| 39 |
'BPSN' : BPSN (Background Positive, Subgroup Negative) AUC,
|
| 40 |
'BNSP' : BNSP (Background Negative, Subgroup Positive) AUC score,
|
| 41 |
-
|
| 42 |
-
|
| 43 |
Example:
|
| 44 |
>>> from evaluate import load
|
| 45 |
|
|
@@ -54,7 +47,6 @@ Example:
|
|
| 54 |
... [0.4341845214366913, 0.5658154487609863],
|
| 55 |
... [0.400595098733902, 0.5994048714637756],
|
| 56 |
... [0.3840397894382477, 0.6159601807594299]]
|
| 57 |
-
|
| 58 |
>>> metric = load('Intel/bias_auc')
|
| 59 |
>>> metric.add_batch(target=target,
|
| 60 |
label=label,
|
|
@@ -68,7 +60,7 @@ Example:
|
|
| 68 |
|
| 69 |
class BiasAUC(evaluate.Metric):
|
| 70 |
def _info(self):
|
| 71 |
-
return
|
| 72 |
description=_DESCRIPTION,
|
| 73 |
citation=_CITATION,
|
| 74 |
inputs_description=_KWARGS_DESCRIPTION,
|
|
@@ -165,4 +157,4 @@ class BiasAUC(evaluate.Metric):
|
|
| 165 |
for subgroup in subgroups}
|
| 166 |
result = self._update_overall(result, label, output)
|
| 167 |
return result
|
| 168 |
-
|
|
|
|
| 9 |
Suite of threshold-agnostic metrics that provide a nuanced view
|
| 10 |
of this unintended bias, by considering the various ways that a
|
| 11 |
classifier’s score distribution can vary across designated groups.
|
|
|
|
| 12 |
The following are computed:
|
|
|
|
| 13 |
- BNSP (Background Negative, Subgroup Positive); and
|
| 14 |
- BPSN (Background Positive, Subgroup Negative) AUC
|
|
|
|
| 15 |
"""
|
| 16 |
|
| 17 |
_CITATION = """\
|
|
|
|
| 25 |
"""
|
| 26 |
|
| 27 |
_KWARGS_DESCRIPTION = """\
|
|
|
|
| 28 |
Args:
|
| 29 |
target list[list[str]]: list containing list of group targeted for each item
|
| 30 |
label list[int]: list containing label index for each item
|
| 31 |
output list[list[float]]: list of model output values for each
|
|
|
|
| 32 |
Returns (for each subgroup in target):
|
| 33 |
'Subgroup' : Subgroup AUC score,
|
| 34 |
'BPSN' : BPSN (Background Positive, Subgroup Negative) AUC,
|
| 35 |
'BNSP' : BNSP (Background Negative, Subgroup Positive) AUC score,
|
|
|
|
|
|
|
| 36 |
Example:
|
| 37 |
>>> from evaluate import load
|
| 38 |
|
|
|
|
| 47 |
... [0.4341845214366913, 0.5658154487609863],
|
| 48 |
... [0.400595098733902, 0.5994048714637756],
|
| 49 |
... [0.3840397894382477, 0.6159601807594299]]
|
|
|
|
| 50 |
>>> metric = load('Intel/bias_auc')
|
| 51 |
>>> metric.add_batch(target=target,
|
| 52 |
label=label,
|
|
|
|
| 60 |
|
| 61 |
class BiasAUC(evaluate.Metric):
|
| 62 |
def _info(self):
|
| 63 |
+
return evaluate.MetricInfo(
|
| 64 |
description=_DESCRIPTION,
|
| 65 |
citation=_CITATION,
|
| 66 |
inputs_description=_KWARGS_DESCRIPTION,
|
|
|
|
| 157 |
for subgroup in subgroups}
|
| 158 |
result = self._update_overall(result, label, output)
|
| 159 |
return result
|
| 160 |
+
|