jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

timed re.search and _executor made global

Browse Source
This commit is contained in:
Francesco Bertolotti 2025-05-21 13:08:48 +02:00
parent cf029d8045
commit c5c9e8628f
1 changed files with 19 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
opencompass/datasets/mbpp.py
View File

@ -4,7 +4,7 @@ import itertools
import json
import multiprocessing
import os.path as osp
import re
import regex as re
import signal
import tempfile
from collections import defaultdict
@ -330,7 +330,7 @@ class MBPPEvaluator(BaseEvaluator):
r"'(.*)'\s*\[DONE\]",
]
for p in patterns:
match = re.search(p, text, re.DOTALL)
match = re.search(p, text, re.DOTALL, timeout=10.0)
if match:
text = match.group(1)
break
@ -383,6 +383,22 @@ class MBPPEvaluator2(MBPPEvaluator):
text = text[1:]
return text
def _execution(programs, timeout, key):
try:
# Add exec globals to prevent the exec to raise
# unnecessary NameError for correct answer
exec_globals = {}
with swallow_io():
with time_limit(timeout):
exec(programs, exec_globals)
key.append('pass')
except TimeOutException:
key.append('timeout')
except AssertionError:
key.append('wrong_answer')
except BaseException as e:
print(e)
key.append('failed')
def execution(programs, task_id, timeout):
"""Execution function for running generation code.
@ -400,29 +416,12 @@ def execution(programs, task_id, timeout):
control the process.
"""
def _execution(programs, timeout):
try:
# Add exec globals to prevent the exec to raise
# unnecessary NameError for correct answer
exec_globals = {}
with swallow_io():
with time_limit(timeout):
exec(programs, exec_globals)
key.append('pass')
except TimeOutException:
key.append('timeout')
except AssertionError:
key.append('wrong_answer')
except BaseException as e:
print(e)
key.append('failed')
manager = multiprocessing.Manager()
key = manager.list()
# `signal` cannot be used in child thread, therefore, we
# need to create a process in the thread.
p = multiprocessing.Process(target=_execution,
args=(programs, timeout - 1))
args=(programs, timeout - 1, key))
p.start()
p.join(timeout=timeout)
if p.is_alive():