OpenCompass/opencompass/datasets/lcsts.py

import os.path as osp

from datasets import Dataset

from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS

from .base import BaseDataset


@LOAD_DATASET.register_module()
class LCSTSDataset(BaseDataset):

    @staticmethod
    def load(path: str):
        src_path = osp.join(path, 'test.src.txt')
        tgt_path = osp.join(path, 'test.tgt.txt')

        src_lines = open(src_path, 'r', encoding='utf-8').readlines()
        tgt_lines = open(tgt_path, 'r', encoding='utf-8').readlines()

        data = {'content': [], 'abst': []}

        for _, (src_text, tgt_text) in enumerate(zip(src_lines, tgt_lines)):
            data['content'].append(src_text.strip())
            data['abst'].append(tgt_text.strip())

        dataset = Dataset.from_dict({
            'content': data['content'],
            'abst': data['abst']
        })
        return dataset


@TEXT_POSTPROCESSORS.register_module('lcsts')
def lcsts_postprocess(text: str) -> str:
    text = text.strip().split('\n')[0].replace('своей', '').strip()
    text = text.replace('1. ', '') if text.startswith('1. ') else text
    text = text.replace('- ', '') if text.startswith('- ') else text
    text = text.strip('“，。！”')
    return text