jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

refactor: delete unnecessary comment

Browse Source
This commit is contained in:
chenzihong-gavin 2025-04-14 21:20:29 +08:00
parent f9b1636598
commit db04df78d4
3 changed files with 85 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
opencompass/configs/datasets/SeedBench/seedbench_gen_44868b.py → opencompass/configs/datasets/SeedBench/seedbench_gen_5d5ea1.py
View File

@ -53,7 +53,6 @@ dataset_configs = [
{'abbr': 'seedbench_3-5', 'data_file': '3-5.json', 'evaluator': 'AccScoreStr_Evaluator'},
]
seedbench_datasets = []
for stage in ['zero-shot','one-shot']:
for config in dataset_configs:

193
opencompass/datasets/SeedBench.py
View File

@ -1,27 +1,34 @@
import os
import random
import datasets
from typing import List
from .base import BaseDataset
from opencompass.openicl.icl_evaluator.icl_base_evaluator import BaseEvaluator
import numpy as np
import re
from typing import List
import datasets
import jieba
import numpy as np
from rouge_chinese import Rouge
from opencompass.openicl.icl_evaluator.icl_base_evaluator import BaseEvaluator
from opencompass.registry import ICL_EVALUATORS, TEXT_POSTPROCESSORS
from .base import BaseDataset
class SeedBenchDataset(BaseDataset):
@staticmethod
def load(data_files: str, path: str = 'json', split: str = None, **kwargs) -> datasets.Dataset:
def load(data_files: str,
path: str = 'json',
split: str = None,
**kwargs) -> datasets.Dataset:
dataset = datasets.load_dataset(path, data_files=data_files, **kwargs)
if split is None:
split = list(dataset.keys())[0]
print(f"my datasets split : {split}")
print(f'my datasets split : {split}')
if split not in dataset:
raise ValueError(f"Split '{split}' not found. Available splits: {list(dataset.keys())}")
raise ValueError(f"Split '{split}' not found. \
Available splits: {list(dataset.keys())}")
return dataset[split]
@ -30,7 +37,8 @@ class F1Evaluator(BaseEvaluator):
"""F1 Score evaluator for multiple choice questions.
Args:
seed (int): Seed for randomness, ensuring reproducibility. Defaults to 0.
seed (int): Seed for randomness, ensuring reproducibility.
Defaults to 0.
"""
def __init__(self, seed: int = 0) -> None:
@ -38,41 +46,15 @@ class F1Evaluator(BaseEvaluator):
super().__init__()
def _preprocess(self, predictions: List, references: List) -> dict:
"""Preprocess the final predictions and references to needed format.
Args:
predictions (List): List of predictions for each sample.
references (List): List of reference answers for each sample.
Returns:
dict: Preprocessed predictions and references in the required format.
"""
return {
'predictions': predictions,
'references': references,
}
def _postprocess(self, scores: dict) -> dict:
"""Postprocess the final score for F1.
Args:
scores (dict): Dictionary of calculated F1 score.
Returns:
dict: Postprocessed F1 score.
"""
return scores
def score(self, predictions: List, references: List) -> dict:
"""Calculate F1 score.
Args:
predictions (List): List of predicted answers for each sample.
references (List): List of reference answers for each sample.
Returns:
dict: Calculated F1 score.
"""
random_state = random.getstate()
np_random_state = np.random.get_state()
details = []
@ -82,7 +64,8 @@ class F1Evaluator(BaseEvaluator):
if len(predictions) != len(references):
return {
'error': 'predictions and references have different '
'error':
'predictions and references have different '
f'length. len(predictions): {len(predictions)}, '
f'len(references): {len(references)}'
}
@ -105,18 +88,29 @@ class F1Evaluator(BaseEvaluator):
true_positives += sample_tp
false_positives += sample_fp
false_negatives += sample_fn
sample_precision = sample_tp / (sample_tp + sample_fp) if (sample_tp + sample_fp) > 0 else 0
sample_recall = sample_tp / (sample_tp + sample_fn) if (sample_tp + sample_fn) > 0 else 0
sample_f1 = (2 * sample_precision * sample_recall) / (sample_precision + sample_recall) if (sample_precision + sample_recall) > 0 else 0
details.append({'pred': hyp, 'answer': ref, 'correct': sample_f1 * 100})
sample_precision = sample_tp / (sample_tp + sample_fp) if (
sample_tp + sample_fp) > 0 else 0
sample_recall = sample_tp / (sample_tp + sample_fn) if (
sample_tp + sample_fn) > 0 else 0
sample_f1 = (2 * sample_precision * sample_recall) / (
sample_precision + sample_recall) if (sample_precision +
sample_recall) > 0 else 0
details.append({
'pred': hyp,
'answer': ref,
'correct': sample_f1 * 100
})
precision = true_positives / (true_positives + false_positives) if (true_positives + false_positives) > 0 else 0
recall = true_positives / (true_positives + false_negatives) if (true_positives + false_negatives) > 0 else 0
f1 = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0
precision = true_positives / (true_positives + false_positives) if (
true_positives + false_positives) > 0 else 0
recall = true_positives / (true_positives + false_negatives) if (
true_positives + false_negatives) > 0 else 0
f1 = (2 * precision *
recall) / (precision + recall) if (precision + recall) > 0 else 0
result = {
"ours_F1Score": f1 * 100, # 总体 F1 分数
"details": details
'ours_F1Score': f1 * 100, # 总体 F1 分数
'details': details
}
random.setstate(random_state)
np.random.set_state(np_random_state)
@ -126,6 +120,7 @@ class F1Evaluator(BaseEvaluator):
@ICL_EVALUATORS.register_module()
class F1ScoreEvaluator(F1Evaluator):
"""F1 Score evaluator for multiple choice questions."""
def __init__(self) -> None:
super().__init__()
@ -143,7 +138,8 @@ class AverageRougeEvaluator(BaseEvaluator):
"""Average Rouge Score evaluator for fill-in-the-blank tasks.
Args:
seed (int): Seed for randomness, ensuring reproducibility. Defaults to 0.
seed (int): Seed for randomness, ensuring reproducibility.
Defaults to 0.
"""
def __init__(self, seed: int = 0) -> None:
@ -151,17 +147,11 @@ class AverageRougeEvaluator(BaseEvaluator):
super().__init__()
def _preprocess(self, predictions: List, references: List) -> dict:
"""Preprocess the final predictions and references to needed format.
Args:
predictions (List): List of predictions for each sample.
references (List): List of reference answers for each sample.
Returns:
dict: Preprocessed predictions and references in the required format.
"""
pattern = r"(正确答案[:：]|correct answer[:：])"
cleaned_predictions = [re.sub(pattern, "", pred, flags=re.IGNORECASE).strip() for pred in predictions]
pattern = r'(正确答案[:：]|correct answer[:：])'
cleaned_predictions = [
re.sub(pattern, '', pred, flags=re.IGNORECASE).strip()
for pred in predictions
]
return {
'predictions': cleaned_predictions,
@ -169,35 +159,19 @@ class AverageRougeEvaluator(BaseEvaluator):
}
def _postprocess(self, scores: dict) -> dict:
"""Postprocess the final Rouge scores.
Args:
scores (dict): Dictionary of calculated average Rouge scores.
Returns:
dict: Postprocessed Rouge scores.
"""
return scores
def score(self, predictions: List, references: List) -> dict:
"""Calculate average Rouge-L score.
Args:
predictions (List): List of predicted strings for each sample.
references (List): List of reference strings for each sample.
Returns:
dict: Calculated average Rouge-L score.
"""
def rouge_score(hyps, refs):
assert(len(hyps) == len(refs))
assert (len(hyps) == len(refs))
hyps = [' '.join(jieba.cut(h)) for h in hyps]
hyps = [h if h.strip() != "" else "无内容" for h in hyps]
hyps = [h if h.strip() != '' else '无内容' for h in hyps]
refs = [' '.join(jieba.cut(r)) for r in refs]
rouge_scores = Rouge().get_scores(hyps, refs)
rouge_ls = [score["rouge-l"]["f"] for score in rouge_scores]
rouge_ls = [score['rouge-l']['f'] for score in rouge_scores]
average_rouge_l = sum(rouge_ls) / len(rouge_ls)
return {"score": average_rouge_l * 100}
return {'score': average_rouge_l * 100}
random_state = random.getstate()
np_random_state = np.random.get_state()
@ -207,13 +181,15 @@ class AverageRougeEvaluator(BaseEvaluator):
if len(predictions) != len(references):
return {
'error': 'predictions and references have different '
'error':
'predictions and references have different '
f'length. len(predictions): {len(predictions)}, '
f'len(references): {len(references)}'
}
preprocessed_data = self._preprocess(predictions, references)
hyps, refs = preprocessed_data['predictions'], preprocessed_data['references']
hyps, refs = preprocessed_data['predictions'], preprocessed_data[
'references']
scores = []
for i in range(len(hyps)):
@ -227,19 +203,22 @@ class AverageRougeEvaluator(BaseEvaluator):
word_level_hyps = [h.strip() for h in word_level_hyps]
if len(word_level_hyps) < len(word_level_refs):
word_level_hyps += ['无内容'] * (len(word_level_refs) - len(word_level_hyps))
word_level_hyps += ['无内容'] * (len(word_level_refs) -
len(word_level_hyps))
else:
word_level_hyps = word_level_hyps[:len(word_level_refs)]
sample_score = rouge_score(word_level_hyps, word_level_refs)["score"]
sample_score = rouge_score(word_level_hyps,
word_level_refs)['score']
scores.append(sample_score)
details.append({'pred': word_level_hyps, 'answer': word_level_refs, 'correct': sample_score})
details.append({
'pred': word_level_hyps,
'answer': word_level_refs,
'correct': sample_score
})
average_score = sum(scores) / len(scores)
result = {
"AvgRougeScore": average_score,
"details": details
}
result = {'AvgRougeScore': average_score, 'details': details}
random.setstate(random_state)
np.random.set_state(np_random_state)
@ -258,7 +237,8 @@ class AccScoreStrEvaluator(BaseEvaluator):
"""Accuracy evaluator based on string matching.
Args:
seed (int): Seed for randomness, ensuring reproducibility. Defaults to 0.
seed (int): Seed for randomness, ensuring reproducibility.
Defaults to 0.
"""
def __init__(self, seed: int = 0) -> None:
@ -266,41 +246,15 @@ class AccScoreStrEvaluator(BaseEvaluator):
super().__init__()
def _preprocess(self, predictions: List, references: List) -> dict:
"""Preprocess the final predictions and references to needed format.
Args:
predictions (List): List of predictions for each sample.
references (List): List of reference answers for each sample.
Returns:
dict: Preprocessed predictions and references in the required format.
"""
return {
'predictions': predictions,
'references': references,
}
def _postprocess(self, scores: dict) -> dict:
"""Postprocess the final accuracy score.
Args:
scores (dict): Dictionary of calculated accuracy score.
Returns:
dict: Postprocessed accuracy score.
"""
return scores
def score(self, predictions: List, references: List) -> dict:
"""Calculate accuracy score.
Args:
predictions (List): List of predicted strings for each sample.
references (List): List of reference strings for each sample.
Returns:
dict: Calculated accuracy score.
"""
random_state = random.getstate()
np_random_state = np.random.get_state()
details = []
@ -309,7 +263,8 @@ class AccScoreStrEvaluator(BaseEvaluator):
if len(predictions) != len(references):
return {
'error': 'predictions and references have different '
'error':
'predictions and references have different '
f'length. len(predictions): {len(predictions)}, '
f'len(references): {len(references)}'
}
@ -317,16 +272,14 @@ class AccScoreStrEvaluator(BaseEvaluator):
preprocessed_data = self._preprocess(predictions, references)
correct = 0
for hyp, ref in zip(preprocessed_data['predictions'], preprocessed_data['references']):
for hyp, ref in zip(preprocessed_data['predictions'],
preprocessed_data['references']):
is_correct = 1 if ref.strip().lower() in hyp.strip().lower() else 0
correct += is_correct
details.append({'pred': hyp, 'answer': ref, 'correct': is_correct})
accuracy = correct / len(predictions)
result = {
"ACCStrScore": accuracy * 100,
"details": details
}
result = {'ACCStrScore': accuracy * 100, 'details': details}
random.setstate(random_state)
np.random.set_state(np_random_state)