jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

update_summerizer

Browse Source
This commit is contained in:
yufeng zhao 2025-03-17 05:22:44 +00:00
parent 4941df996f
commit b6fe20b20e
3 changed files with 13 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
opencompass/summarizers/default.py
View File

@ -171,6 +171,8 @@ class DefaultSummarizer:
default_metric = 'sum' default_metric = 'sum'
elif sg.get('weights', []): elif sg.get('weights', []):
default_metric = 'weighted_average' default_metric = 'weighted_average'
elif 'harmonic_mean' in sg:
default_metric = 'harmonic_mean'
else: else:
default_metric = 'naive_average' default_metric = 'naive_average'
@ -204,6 +206,17 @@ class DefaultSummarizer:
avg = sum(scores[metric].values()) / len(scores[metric]) avg = sum(scores[metric].values()) / len(scores[metric])
variance = sum((scores[metric][k] - avg) ** 2 for k in scores[metric]) / len(scores[metric]) variance = sum((scores[metric][k] - avg) ** 2 for k in scores[metric]) / len(scores[metric])
scores[metric] = result[metric] = math.sqrt(variance) scores[metric] = result[metric] = math.sqrt(variance)
elif default_metric == 'harmonic_mean':
# Check for non-positive values that would cause issues in harmonic mean
if any(scores[metric][k] <= 0 for k in scores[metric]):
self.logger.warning(f'Non-positive values found when calculating harmonic mean for {sg["name"]}')
# Handle non-positive values (either skip or use a small positive value)
numerator = len(scores[metric])
denominator = sum(1 / max(scores[metric][k], 1e-10) for k in scores[metric])
else:
numerator = len(scores[metric])
denominator = sum(1 / scores[metric][k] for k in scores[metric])
scores[metric] = result[metric] = numerator / denominator
else: else:
if sg.get('weights', []): if sg.get('weights', []):
# check sg['weights'][k] != 0 in case of scores[metric][k] is NaN # check sg['weights'][k] != 0 in case of scores[metric][k] is NaN

11
opencompass/summarizers/multi_model.py
View File

@ -203,17 +203,6 @@ class MultiModelSummarizer:
numerator = sum(results[k] * sg['weights'][k] for k in sg['weights']) numerator = sum(results[k] * sg['weights'][k] for k in sg['weights'])
denominator = sum(sg['weights'].values()) denominator = sum(sg['weights'].values())
metric = 'weighted_average' metric = 'weighted_average'
elif 'harmonic_mean' in sg:
# Check for non-positive values that would cause issues in harmonic mean
if any(results[k] <= 0 for k in results):
self.logger.warning(f'Non-positive values found when calculating harmonic mean for {sg["name"]}')
# Handle non-positive values (either skip or use a small positive value)
numerator = len(results)
denominator = sum(1 / max(results[k], 1e-10) for k in results)
else:
numerator = len(results)
denominator = sum(1 / results[k] for k in results)
metric = 'harmonic_mean'
else: else:
numerator = sum(results[k] for k in results) numerator = sum(results[k] for k in results)
denominator = len(results) denominator = len(results)

11
opencompass/summarizers/summarizer_pretrain.py
View File

@ -115,17 +115,6 @@ class PretrainSummarizer:
numerator = sum(results[k] * sg['weights'][k] for k in sg['weights']) numerator = sum(results[k] * sg['weights'][k] for k in sg['weights'])
denominator = sum(sg['weights'].values()) denominator = sum(sg['weights'].values())
metric = 'weighted_average' metric = 'weighted_average'
elif 'harmonic_mean' in sg:
# Check for non-positive values that would cause issues in harmonic mean
if any(results[k] <= 0 for k in results):
self.logger.warning(f'Non-positive values found when calculating harmonic mean for {sg["name"]}')
# Handle non-positive values (either skip or use a small positive value)
numerator = len(results)
denominator = sum(1 / max(results[k], 1e-10) for k in results)
else:
numerator = len(results)
denominator = sum(1 / results[k] for k in results)
metric = 'harmonic_mean'
else: else:
numerator = sum(results[k] for k in results) numerator = sum(results[k] for k in results)
denominator = len(results) denominator = len(results)