jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Update] Bbeh harmony summarizer added (#1951)

Browse Source 
* bbeh

* bbeh

* fix_smallbugs_bbeh

* removeprint

* harmonic

* update_summerizer

* harmonic-tested

* harmonic-tested

* clean

* clean

* cleaned_rebased

---------

Co-authored-by: yufeng zhao <zhaoyufeng@pjlab.org.cn>
This commit is contained in:
Yufeng Zhao 2025-03-17 17:19:56 +08:00 committed by GitHub
parent 854c6bf025
commit 15c825a51a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 43 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
opencompass/configs/summarizers/bbeh.py Normal file
View File

@ -0,0 +1,30 @@
from mmengine.config import read_base
with read_base():
from .groups.bbeh import bbeh_summary_groups
# Get all the BBEH subset names from the imported bbeh_summary_groups
bbeh_subsets = []
for group in bbeh_summary_groups:
if group['name'] == 'bbeh':
bbeh_subsets = group['subsets']
break
summarizer = dict(
# Include both individual datasets and the summary metrics we want to see
dataset_abbrs=bbeh_subsets + ['bbeh_naive_average'] + ['bbeh_harmonic_mean'],
# Define the summary group for bbeh
summary_groups=[
{
'name': 'bbeh_naive_average',
'subsets': bbeh_subsets,
'metric': 'naive_average' # Explicitly specify the metric to use
},
{
'name': 'bbeh_harmonic_mean',
'subsets': bbeh_subsets,
'metric': 'harmonic_mean'
}
]
)

13
opencompass/summarizers/default.py
View File

@ -171,6 +171,8 @@ class DefaultSummarizer:
default_metric = 'sum'
elif sg.get('weights', []):
default_metric = 'weighted_average'
elif sg.get('harmonic_mean', False):
default_metric = 'harmonic_mean'
else:
default_metric = 'naive_average'
@ -204,6 +206,17 @@ class DefaultSummarizer:
avg = sum(scores[metric].values()) / len(scores[metric])
variance = sum((scores[metric][k] - avg) ** 2 for k in scores[metric]) / len(scores[metric])
scores[metric] = result[metric] = math.sqrt(variance)
elif default_metric == 'harmonic_mean':
# Check for non-positive values that would cause issues in harmonic mean
if any(scores[metric][k] <= 0 for k in scores[metric]):
self.logger.warning(f'Non-positive values found when calculating harmonic mean for {sg["name"]}')
# Handle non-positive values (either skip or use a small positive value)
numerator = len(scores[metric])
denominator = sum(1 / max(scores[metric][k], 1) for k in scores[metric])
else:
numerator = len(scores[metric])
denominator = sum(1 / scores[metric][k] for k in scores[metric])
scores[metric] = result[metric] = numerator / denominator
else:
if sg.get('weights', []):
# check sg['weights'][k] != 0 in case of scores[metric][k] is NaN