From 80f831b425a7312e4580cbd2361187b5a0f1df42 Mon Sep 17 00:00:00 2001 From: Fengzhe Zhou Date: Wed, 15 May 2024 13:48:29 +0800 Subject: [PATCH] [Fix] use ProcessPoolExecutor during mbpp eval (#1159) --- opencompass/datasets/mbpp.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/opencompass/datasets/mbpp.py b/opencompass/datasets/mbpp.py index 51f87c5b..90ac1410 100644 --- a/opencompass/datasets/mbpp.py +++ b/opencompass/datasets/mbpp.py @@ -8,7 +8,7 @@ import re import signal import tempfile from collections import defaultdict -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import ProcessPoolExecutor, as_completed from typing import List, Sequence, Union import numpy as np @@ -213,8 +213,7 @@ class MBPPEvaluator(BaseEvaluator): if self.metric == 'MBPP': result = {'pass': 0, 'timeout': 0, 'failed': 0, 'wrong_answer': 0} details = {} - # change to thread pool for better killing blocked instance - with ThreadPoolExecutor() as executor: + with ProcessPoolExecutor() as executor: futures = [] for i, (refer, pred) in enumerate(zip(references, predictions)): @@ -439,7 +438,7 @@ class MBPPPassKEvaluator(MBPPEvaluator): task_total = defaultdict(int) result = {'pass': 0, 'timeout': 0, 'failed': 0, 'wrong_answer': 0} - with ThreadPoolExecutor() as executor: + with ProcessPoolExecutor() as executor: futures = [] for refer, preds in zip(references, predictions): # suits for two case