From f9599c1f3285f218b6e32f545bf98205b67e5496 Mon Sep 17 00:00:00 2001 From: yufeng zhao Date: Mon, 17 Mar 2025 05:22:44 +0000 Subject: [PATCH] update_summerizer --- opencompass/summarizers/default.py | 13 +++++++++++++ opencompass/summarizers/multi_model.py | 11 ----------- opencompass/summarizers/summarizer_pretrain.py | 11 ----------- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/opencompass/summarizers/default.py b/opencompass/summarizers/default.py index 8a0da5b2..f4163ecd 100644 --- a/opencompass/summarizers/default.py +++ b/opencompass/summarizers/default.py @@ -171,6 +171,8 @@ class DefaultSummarizer: default_metric = 'sum' elif sg.get('weights', []): default_metric = 'weighted_average' + elif 'harmonic_mean' in sg: + default_metric = 'harmonic_mean' else: default_metric = 'naive_average' @@ -204,6 +206,17 @@ class DefaultSummarizer: avg = sum(scores[metric].values()) / len(scores[metric]) variance = sum((scores[metric][k] - avg) ** 2 for k in scores[metric]) / len(scores[metric]) scores[metric] = result[metric] = math.sqrt(variance) + elif default_metric == 'harmonic_mean': + # Check for non-positive values that would cause issues in harmonic mean + if any(scores[metric][k] <= 0 for k in scores[metric]): + self.logger.warning(f'Non-positive values found when calculating harmonic mean for {sg["name"]}') + # Handle non-positive values (either skip or use a small positive value) + numerator = len(scores[metric]) + denominator = sum(1 / max(scores[metric][k], 1e-10) for k in scores[metric]) + else: + numerator = len(scores[metric]) + denominator = sum(1 / scores[metric][k] for k in scores[metric]) + scores[metric] = result[metric] = numerator / denominator else: if sg.get('weights', []): # check sg['weights'][k] != 0 in case of scores[metric][k] is NaN diff --git a/opencompass/summarizers/multi_model.py b/opencompass/summarizers/multi_model.py index 9cda4086..45d159d5 100644 --- a/opencompass/summarizers/multi_model.py +++ b/opencompass/summarizers/multi_model.py @@ -203,17 +203,6 @@ class MultiModelSummarizer: numerator = sum(results[k] * sg['weights'][k] for k in sg['weights']) denominator = sum(sg['weights'].values()) metric = 'weighted_average' - elif 'harmonic_mean' in sg: - # Check for non-positive values that would cause issues in harmonic mean - if any(results[k] <= 0 for k in results): - self.logger.warning(f'Non-positive values found when calculating harmonic mean for {sg["name"]}') - # Handle non-positive values (either skip or use a small positive value) - numerator = len(results) - denominator = sum(1 / max(results[k], 1e-10) for k in results) - else: - numerator = len(results) - denominator = sum(1 / results[k] for k in results) - metric = 'harmonic_mean' else: numerator = sum(results[k] for k in results) denominator = len(results) diff --git a/opencompass/summarizers/summarizer_pretrain.py b/opencompass/summarizers/summarizer_pretrain.py index 482340e5..6c19ae72 100644 --- a/opencompass/summarizers/summarizer_pretrain.py +++ b/opencompass/summarizers/summarizer_pretrain.py @@ -115,17 +115,6 @@ class PretrainSummarizer: numerator = sum(results[k] * sg['weights'][k] for k in sg['weights']) denominator = sum(sg['weights'].values()) metric = 'weighted_average' - elif 'harmonic_mean' in sg: - # Check for non-positive values that would cause issues in harmonic mean - if any(results[k] <= 0 for k in results): - self.logger.warning(f'Non-positive values found when calculating harmonic mean for {sg["name"]}') - # Handle non-positive values (either skip or use a small positive value) - numerator = len(results) - denominator = sum(1 / max(results[k], 1e-10) for k in results) - else: - numerator = len(results) - denominator = sum(1 / results[k] for k in results) - metric = 'harmonic_mean' else: numerator = sum(results[k] for k in results) denominator = len(results)