jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Fix] use ProcessPoolExecutor during mbpp eval (#1159)

Browse Source
This commit is contained in:
Fengzhe Zhou 2024-05-15 13:48:29 +08:00 committed by GitHub
parent 8a8987be0b
commit 80f831b425
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
opencompass/datasets/mbpp.py
View File

@ -8,7 +8,7 @@ import re
import signal import signal
import tempfile import tempfile
from collections import defaultdict from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ProcessPoolExecutor, as_completed
from typing import List, Sequence, Union from typing import List, Sequence, Union
import numpy as np import numpy as np
@ -213,8 +213,7 @@ class MBPPEvaluator(BaseEvaluator):
if self.metric == 'MBPP': if self.metric == 'MBPP':
result = {'pass': 0, 'timeout': 0, 'failed': 0, 'wrong_answer': 0} result = {'pass': 0, 'timeout': 0, 'failed': 0, 'wrong_answer': 0}
details = {} details = {}
# change to thread pool for better killing blocked instance with ProcessPoolExecutor() as executor:
with ThreadPoolExecutor() as executor:
futures = [] futures = []
for i, (refer, pred) in enumerate(zip(references, for i, (refer, pred) in enumerate(zip(references,
predictions)): predictions)):
@ -439,7 +438,7 @@ class MBPPPassKEvaluator(MBPPEvaluator):
task_total = defaultdict(int) task_total = defaultdict(int)
result = {'pass': 0, 'timeout': 0, 'failed': 0, 'wrong_answer': 0} result = {'pass': 0, 'timeout': 0, 'failed': 0, 'wrong_answer': 0}
with ThreadPoolExecutor() as executor: with ProcessPoolExecutor() as executor:
futures = [] futures = []
for refer, preds in zip(references, predictions): for refer, preds in zip(references, predictions):
# suits for two case # suits for two case