From 78d94e7bbd2bd3ed8c1b72e96b03c4af118771b3 Mon Sep 17 00:00:00 2001
From: yufeng zhao <zhaoyufeng@pjlab.org.cn>
Date: Sun, 16 Mar 2025 03:25:32 +0000
Subject: [PATCH] harmonic

---
 opencompass/configs/summarizers/bbeh.py        | 13 +++++++++++++
 opencompass/summarizers/multi_model.py         | 11 +++++++++++
 opencompass/summarizers/summarizer_pretrain.py | 11 +++++++++++
 3 files changed, 35 insertions(+)
 create mode 100644 opencompass/configs/summarizers/bbeh.py

diff --git a/opencompass/configs/summarizers/bbeh.py b/opencompass/configs/summarizers/bbeh.py
new file mode 100644
index 00000000..66111bd3
--- /dev/null
+++ b/opencompass/configs/summarizers/bbeh.py
@@ -0,0 +1,13 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .groups.bbeh import bbeh_summary_groups
+
+summarizer = dict(
+    dataset_abbrs=[
+        ['bbeh', 'naive_average'],
+        ['bbeh', 'harmonic_mean']
+    ],
+    summary_groups=sum(
+        [v for k, v in locals().items() if k.endswith('_summary_groups')], []),
+)
\ No newline at end of file
diff --git a/opencompass/summarizers/multi_model.py b/opencompass/summarizers/multi_model.py
index 45d159d5..9cda4086 100644
--- a/opencompass/summarizers/multi_model.py
+++ b/opencompass/summarizers/multi_model.py
@@ -203,6 +203,17 @@ class MultiModelSummarizer:
                         numerator = sum(results[k] * sg['weights'][k] for k in sg['weights'])
                         denominator = sum(sg['weights'].values())
                         metric = 'weighted_average'
+                    elif 'harmonic_mean' in sg:
+                        # Check for non-positive values that would cause issues in harmonic mean
+                        if any(results[k] <= 0 for k in results):
+                            self.logger.warning(f'Non-positive values found when calculating harmonic mean for {sg["name"]}')
+                            # Handle non-positive values (either skip or use a small positive value)
+                            numerator = len(results)
+                            denominator = sum(1 / max(results[k], 1e-10) for k in results)
+                        else:
+                            numerator = len(results)
+                            denominator = sum(1 / results[k] for k in results)
+                        metric = 'harmonic_mean'
                     else:
                         numerator = sum(results[k] for k in results)
                         denominator = len(results)
diff --git a/opencompass/summarizers/summarizer_pretrain.py b/opencompass/summarizers/summarizer_pretrain.py
index 6c19ae72..482340e5 100644
--- a/opencompass/summarizers/summarizer_pretrain.py
+++ b/opencompass/summarizers/summarizer_pretrain.py
@@ -115,6 +115,17 @@ class PretrainSummarizer:
                         numerator = sum(results[k] * sg['weights'][k] for k in sg['weights'])
                         denominator = sum(sg['weights'].values())
                         metric = 'weighted_average'
+                    elif 'harmonic_mean' in sg:
+                        # Check for non-positive values that would cause issues in harmonic mean
+                        if any(results[k] <= 0 for k in results):
+                            self.logger.warning(f'Non-positive values found when calculating harmonic mean for {sg["name"]}')
+                            # Handle non-positive values (either skip or use a small positive value)
+                            numerator = len(results)
+                            denominator = sum(1 / max(results[k], 1e-10) for k in results)
+                        else:
+                            numerator = len(results)
+                            denominator = sum(1 / results[k] for k in results)
+                        metric = 'harmonic_mean'
                     else:
                         numerator = sum(results[k] for k in results)
                         denominator = len(results)