[Update] Update BigCodeBench & LCBench load path (#1857)

* BigCodeBench update

* update LCBench

* update LCBench 2

* update code
This commit is contained in:
Dongsheng Zhu 2025-02-08 15:15:47 +08:00 committed by GitHub
parent 9c2e6a192c
commit 3fd8b4e0cd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 10 additions and 3 deletions

View File

@ -22,7 +22,10 @@ from .base import BaseDataset
class LCDataset(BaseDataset):
@staticmethod
def load(path: str, num_repeats: int = 1, difficulty='ALL'):
def load(path: str,
num_repeats: int = 1,
difficulty='ALL',
local_mode=False):
"""Load LC dataset for pass k mode.
Note that you can use num_repeats > 1 when your model does not support
@ -38,7 +41,7 @@ class LCDataset(BaseDataset):
num_repeats(int): Number of repetition for this dataset to get
multiple responses in special cases.
"""
path = get_data_path(path, local_mode=True)
path = get_data_path(path, local_mode=local_mode)
def processing_test(example):
example['test_case'] = example['test_list']

View File

@ -73,6 +73,8 @@ class BigCodeBenchEvaluator(BaseEvaluator):
eval_type='instruct',
remote_execute_api='https://bigcode-bigcodebench-evaluator.hf.space/', # noqa
dataset_version: str = 'full',
local_mode: bool = False,
path: str = 'opencompass/bigcodebench',
pass_k: str = '1,5,10',
parallel: int = -1,
min_time_limit: float = 1,
@ -84,7 +86,9 @@ class BigCodeBenchEvaluator(BaseEvaluator):
super().__init__()
self.dataset = BigCodeBenchDataset.load(
release_version=release_version,
dataset_version=dataset_version)['test']
dataset_version=dataset_version,
local_mode=local_mode,
path=path)['test']
self.eval_type = eval_type
self.remote_execute_api = remote_execute_api