From 523f0f1f18158f6c1c08a06d46f0f3d5c5532d6c Mon Sep 17 00:00:00 2001
From: Yu Sun <62860713+yusun-nlp@users.noreply.github.com>
Date: Tue, 27 May 2025 19:11:45 +0800
Subject: [PATCH] update 0-shot acc postprocessor

---
 opencompass/datasets/smolinstruct.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/opencompass/datasets/smolinstruct.py b/opencompass/datasets/smolinstruct.py
index 333d3177..cb8bb994 100644
--- a/opencompass/datasets/smolinstruct.py
+++ b/opencompass/datasets/smolinstruct.py
@@ -451,6 +451,11 @@ def smolinstruct_acc_postprocess(text: str) -> str:
 
 @TEXT_POSTPROCESSORS.register_module('smolinstruct-acc-0shot')
 def smolinstruct_acc_0shot_postprocess(text: str) -> str:
+    # Remove <think> tags if they exist
+    if "</think>" in text:
+        text = text.split("</think>")[-1].strip()
+
+    # Check for exact "yes" or "no" responses
     if text.strip().lower() == 'yes':
         return "<BOOLEAN> Yes </BOOLEAN>"
     elif text.strip().lower() == 'no':
@@ -475,4 +480,13 @@ def smolinstruct_acc_0shot_postprocess(text: str) -> str:
                 return "<BOOLEAN> Yes </BOOLEAN>"
             elif answer.lower() == 'no':
                 return "<BOOLEAN> No </BOOLEAN>"
+
+    # If no patterns matched, check for simple "yes" or "no"
+    text = text.strip().lower()
+    if text.startswith('yes') or text.endswith('yes'):
+        return "<BOOLEAN> Yes </BOOLEAN>"
+    elif text.startswith('no') or text.endswith('no'):
+        return "<BOOLEAN> No </BOOLEAN>"
+
+    # If no patterns matched, return an empty string
     return ""