From cbe9fe2cdb745ca70dfde56e6863bf4e70d92241 Mon Sep 17 00:00:00 2001 From: Ezra-Yu <1105212286@qq.com> Date: Wed, 5 Jul 2023 02:22:40 +0000 Subject: [PATCH] Add Release Contraibution --- .pre-commit-config.yaml | 69 ++++++++++++++++ configs/datasets/ARC_e/ARC_e_ppl.py | 4 + .../datasets/CLUE_C3/CLUE_C3_ppl_588820.py | 37 +++++++++ .../CLUE_CMRC/CLUE_CMRC_gen_72a8d5.py | 34 ++++++++ .../CLUE_CMRC/CLUE_CMRC_gen_d7096f.py | 33 ++++++++ .../CLUE_DRCD/CLUE_DRCD_gen_03b96b.py | 34 ++++++++ .../FewCLUE_bustm/FewCLUE_bustm_gen_305431.py | 50 ++++++++++++ .../FewCLUE_chid/FewCLUE_chid_ppl_b6cd88.py | 45 +++++++++++ .../FewCLUE_cluewsc_ppl_fc45f0.py | 58 ++++++++++++++ .../FewCLUE_tnews/FewCLUE_tnews_ppl_784b9e.py | 48 ++++++++++++ .../datasets/GaokaoBench/GaokaoBench_gen.py | 4 + .../SuperGLUE_AX_g_gen_7a5dee.py | 42 ++++++++++ .../SuperGLUE_AX_g_ppl_8d9bf9.py | 53 +++++++++++++ .../SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl.py | 4 + .../datasets/SuperGLUE_CB/SuperGLUE_CB_gen.py | 4 + .../SuperGLUE_COPA_gen_6d5e67.py | 43 ++++++++++ .../SuperGLUE_MultiRC_gen_26c9dc.py | 42 ++++++++++ .../SuperGLUE_WiC/SuperGLUE_WiC_gen_c39367.py | 46 +++++++++++ .../SuperGLUE_WiC/SuperGLUE_WiC_ppl_4118db.py | 55 +++++++++++++ .../SuperGLUE_WiC/SuperGLUE_WiC_ppl_d316eb.py | 49 ++++++++++++ .../TheoremQA/TheoremQA_gen_74abc9.py | 40 ++++++++++ configs/datasets/ceval/ceval_gen.py | 4 + .../civilcomments/civilcomments_ppl_e01497.py | 31 ++++++++ configs/datasets/collections/chat_medium.py | 57 ++++++++++++++ .../commonsenseqa/commonsenseqa_gen_a58dbd.py | 60 ++++++++++++++ configs/datasets/glm/C3.py | 40 ++++++++++ configs/datasets/glm/tnews.py | 41 ++++++++++ configs/datasets/govrepcrs/govrepcrs_gen.py | 4 + .../govrepcrs/govrepcrs_gen_455586.py | 47 +++++++++++ .../hellaswag/hellaswag_gen_cae9cb.py | 43 ++++++++++ .../iwslt2017/iwslt2017_gen_02ea0b.py | 38 +++++++++ configs/datasets/lcsts/lcsts_gen.py | 4 + configs/datasets/mbpp/mbpp_gen_b60b47.py | 64 +++++++++++++++ .../narrativeqa/narrativeqa_gen_ca4b64.py | 30 +++++++ configs/datasets/obqa/obqa_ppl.py | 4 + configs/datasets/obqa/obqa_ppl_2b5b12.py | 66 ++++++++++++++++ configs/datasets/obqa/obqa_ppl_3609cc.py | 52 ++++++++++++ .../realtoxicprompts_gen_82a2f2.py | 36 +++++++++ .../storycloze/storycloze_ppl_c1912d.py | 39 +++++++++ .../summedits/summedits_ppl_163352.py | 50 ++++++++++++ .../summscreen/summscreen_gen_997ee2.py | 35 +++++++++ configs/datasets/winograd/winograd_ppl.py | 4 + .../datasets/z_bench/z_bench_gen_61db0a.py | 28 +++++++ docs/en/Makefile | 20 +++++ docs/en/prompt/prompt_template.md | 1 + docs/en/user_guides/models.md | 1 + docs/zh_cn/_static/css/readthedocs.css | 62 +++++++++++++++ docs/zh_cn/_static/image/logo.png | Bin 0 -> 12650 bytes docs/zh_cn/_templates/callable.rst | 14 ++++ docs/zh_cn/notes/contribution_guide.md | 67 ++++++++++++++++ opencompass/datasets/arc.py | 45 +++++++++++ opencompass/datasets/flores.py | 36 +++++++++ opencompass/datasets/qasper.py | 43 ++++++++++ opencompass/datasets/qaspercut.py | 53 +++++++++++++ opencompass/datasets/safety.py | 23 ++++++ opencompass/datasets/triviaqarc.py | 58 ++++++++++++++ opencompass/datasets/winogrande.py | 44 +++++++++++ opencompass/datasets/xcopa.py | 29 +++++++ .../icl_evaluator/icl_aucroc_evaluator.py | 41 ++++++++++ .../openicl/icl_inferencer/__init__.py | 4 + .../icl_retriever/icl_bm25_retriever.py | 74 ++++++++++++++++++ .../icl_retriever/icl_random_retriever.py | 40 ++++++++++ .../icl_retriever/icl_zero_retriever.py | 26 ++++++ opencompass/openicl/utils/__init__.py | 1 + opencompass/utils/logging.py | 13 +++ 65 files changed, 2266 insertions(+) create mode 100644 .pre-commit-config.yaml create mode 100644 configs/datasets/ARC_e/ARC_e_ppl.py create mode 100644 configs/datasets/CLUE_C3/CLUE_C3_ppl_588820.py create mode 100644 configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_72a8d5.py create mode 100644 configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_d7096f.py create mode 100644 configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_03b96b.py create mode 100644 configs/datasets/FewCLUE_bustm/FewCLUE_bustm_gen_305431.py create mode 100644 configs/datasets/FewCLUE_chid/FewCLUE_chid_ppl_b6cd88.py create mode 100644 configs/datasets/FewCLUE_cluewsc/FewCLUE_cluewsc_ppl_fc45f0.py create mode 100644 configs/datasets/FewCLUE_tnews/FewCLUE_tnews_ppl_784b9e.py create mode 100644 configs/datasets/GaokaoBench/GaokaoBench_gen.py create mode 100644 configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_gen_7a5dee.py create mode 100644 configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_ppl_8d9bf9.py create mode 100644 configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl.py create mode 100644 configs/datasets/SuperGLUE_CB/SuperGLUE_CB_gen.py create mode 100644 configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_gen_6d5e67.py create mode 100644 configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_gen_26c9dc.py create mode 100644 configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_gen_c39367.py create mode 100644 configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_ppl_4118db.py create mode 100644 configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_ppl_d316eb.py create mode 100644 configs/datasets/TheoremQA/TheoremQA_gen_74abc9.py create mode 100644 configs/datasets/ceval/ceval_gen.py create mode 100644 configs/datasets/civilcomments/civilcomments_ppl_e01497.py create mode 100644 configs/datasets/collections/chat_medium.py create mode 100644 configs/datasets/commonsenseqa/commonsenseqa_gen_a58dbd.py create mode 100644 configs/datasets/glm/C3.py create mode 100644 configs/datasets/glm/tnews.py create mode 100644 configs/datasets/govrepcrs/govrepcrs_gen.py create mode 100644 configs/datasets/govrepcrs/govrepcrs_gen_455586.py create mode 100644 configs/datasets/hellaswag/hellaswag_gen_cae9cb.py create mode 100644 configs/datasets/iwslt2017/iwslt2017_gen_02ea0b.py create mode 100644 configs/datasets/lcsts/lcsts_gen.py create mode 100644 configs/datasets/mbpp/mbpp_gen_b60b47.py create mode 100644 configs/datasets/narrativeqa/narrativeqa_gen_ca4b64.py create mode 100644 configs/datasets/obqa/obqa_ppl.py create mode 100644 configs/datasets/obqa/obqa_ppl_2b5b12.py create mode 100644 configs/datasets/obqa/obqa_ppl_3609cc.py create mode 100644 configs/datasets/realtoxicprompts/realtoxicprompts_gen_82a2f2.py create mode 100644 configs/datasets/storycloze/storycloze_ppl_c1912d.py create mode 100644 configs/datasets/summedits/summedits_ppl_163352.py create mode 100644 configs/datasets/summscreen/summscreen_gen_997ee2.py create mode 100644 configs/datasets/winograd/winograd_ppl.py create mode 100644 configs/datasets/z_bench/z_bench_gen_61db0a.py create mode 100644 docs/en/Makefile create mode 100644 docs/en/prompt/prompt_template.md create mode 100644 docs/en/user_guides/models.md create mode 100644 docs/zh_cn/_static/css/readthedocs.css create mode 100644 docs/zh_cn/_static/image/logo.png create mode 100644 docs/zh_cn/_templates/callable.rst create mode 100644 docs/zh_cn/notes/contribution_guide.md create mode 100644 opencompass/datasets/arc.py create mode 100644 opencompass/datasets/flores.py create mode 100644 opencompass/datasets/qasper.py create mode 100644 opencompass/datasets/qaspercut.py create mode 100644 opencompass/datasets/safety.py create mode 100644 opencompass/datasets/triviaqarc.py create mode 100644 opencompass/datasets/winogrande.py create mode 100644 opencompass/datasets/xcopa.py create mode 100644 opencompass/openicl/icl_evaluator/icl_aucroc_evaluator.py create mode 100644 opencompass/openicl/icl_inferencer/__init__.py create mode 100644 opencompass/openicl/icl_retriever/icl_bm25_retriever.py create mode 100644 opencompass/openicl/icl_retriever/icl_random_retriever.py create mode 100644 opencompass/openicl/icl_retriever/icl_zero_retriever.py create mode 100644 opencompass/openicl/utils/__init__.py create mode 100644 opencompass/utils/logging.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..3dfeef9c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,69 @@ +exclude: | + (?x)^( + tests/data/| + opencompass/models/internal/| + opencompass/utils/internal/| + configs/ + ) +repos: + - repo: https://github.com/PyCQA/flake8 + rev: 5.0.4 + hooks: + - id: flake8 + - repo: https://github.com/PyCQA/isort + rev: 5.11.5 + hooks: + - id: isort + - repo: https://github.com/pre-commit/mirrors-yapf + rev: v0.32.0 + hooks: + - id: yapf + - repo: https://github.com/codespell-project/codespell + rev: v2.2.1 + hooks: + - id: codespell + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: trailing-whitespace + exclude: | + (?x)^( + dicts/| + projects/.*?/dicts/ + ) + - id: check-yaml + - id: end-of-file-fixer + exclude: | + (?x)^( + dicts/| + projects/.*?/dicts/ + ) + - id: requirements-txt-fixer + - id: double-quote-string-fixer + - id: check-merge-conflict + - id: fix-encoding-pragma + args: ["--remove"] + - id: mixed-line-ending + args: ["--fix=lf"] + - id: mixed-line-ending + args: ["--fix=lf"] + - repo: https://github.com/executablebooks/mdformat + rev: 0.7.9 + hooks: + - id: mdformat + args: ["--number", "--table-width", "200"] + additional_dependencies: + - mdformat-openmmlab + - mdformat_frontmatter + - linkify-it-py + - repo: https://github.com/myint/docformatter + rev: v1.3.1 + hooks: + - id: docformatter + args: ["--in-place", "--wrap-descriptions", "79"] + # - repo: https://github.com/open-mmlab/pre-commit-hooks + # rev: v0.2.0 # Use the ref you want to point at + # hooks: + # - id: check-algo-readme + # - id: check-copyright + # args: ["mmocr", "tests", "tools"] # these directories will be checked diff --git a/configs/datasets/ARC_e/ARC_e_ppl.py b/configs/datasets/ARC_e/ARC_e_ppl.py new file mode 100644 index 00000000..fab5d48b --- /dev/null +++ b/configs/datasets/ARC_e/ARC_e_ppl.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .ARC_e_ppl_f86898 import ARC_e_datasets # noqa: F401, F403 diff --git a/configs/datasets/CLUE_C3/CLUE_C3_ppl_588820.py b/configs/datasets/CLUE_C3/CLUE_C3_ppl_588820.py new file mode 100644 index 00000000..e9e9d137 --- /dev/null +++ b/configs/datasets/CLUE_C3/CLUE_C3_ppl_588820.py @@ -0,0 +1,37 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import C3Dataset + +C3_reader_cfg = dict( + input_columns=[ + 'question', 'content', 'choice0', 'choice1', 'choice2', 'choice3', + 'choices' + ], + output_column='label') + +C3_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + i: dict(round=[ + dict(role="HUMAN", prompt="文章：{content}\n问题：{question}"), + dict(role="BOT", prompt=f"答案：{{choice{i}}}") + ]) + for i in range(4) + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +C3_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +C3_datasets = [ + dict( + type=C3Dataset, + abbr='C3', + path='./data/CLUE/C3/dev_0.json', + reader_cfg=C3_reader_cfg, + infer_cfg=C3_infer_cfg, + eval_cfg=C3_eval_cfg) +] diff --git a/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_72a8d5.py b/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_72a8d5.py new file mode 100644 index 00000000..43a6aab1 --- /dev/null +++ b/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_72a8d5.py @@ -0,0 +1,34 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import EMEvaluator +from opencompass.datasets import CMRCDataset + +CMRC_reader_cfg = dict( + input_columns=['question', 'context'], output_column='answers') + +CMRC_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt="文章：{context}\n根据上文，回答如下问题：\n{question}\n答："), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +CMRC_eval_cfg = dict( + evaluator=dict(type=EMEvaluator), + pred_role="BOT", +) + +CMRC_datasets = [ + dict( + type=CMRCDataset, + abbr='CMRC_dev', + path='./data/CLUE/CMRC/dev.json', + reader_cfg=CMRC_reader_cfg, + infer_cfg=CMRC_infer_cfg, + eval_cfg=CMRC_eval_cfg), +] diff --git a/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_d7096f.py b/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_d7096f.py new file mode 100644 index 00000000..eff1b8b5 --- /dev/null +++ b/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_d7096f.py @@ -0,0 +1,33 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import EMEvaluator +from opencompass.datasets import CMRCDataset + +CMRC_reader_cfg = dict( + input_columns=['question', 'context'], output_column='answers') + +CMRC_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict(role="HUMAN", prompt="文章：{context}\n根据上文，回答如下问题：{question}"), + dict(role="BOT", prompt="答："), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +CMRC_eval_cfg = dict( + evaluator=dict(type=EMEvaluator), + pred_role="BOT", +) + +CMRC_datasets = [ + dict( + type=CMRCDataset, + abbr='CMRC_dev', + path='./data/CLUE/CMRC/dev.json', + reader_cfg=CMRC_reader_cfg, + infer_cfg=CMRC_infer_cfg, + eval_cfg=CMRC_eval_cfg), +] diff --git a/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_03b96b.py b/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_03b96b.py new file mode 100644 index 00000000..d0b9ec4f --- /dev/null +++ b/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_03b96b.py @@ -0,0 +1,34 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import EMEvaluator +from opencompass.datasets import DRCDDataset + +DRCD_reader_cfg = dict( + input_columns=['question', 'context'], output_column='answers') + +DRCD_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt="文章：{context}\n根据上文，回答如下问题：\n{question}\n答："), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +DRCD_eval_cfg = dict( + evaluator=dict(type=EMEvaluator), + pred_role="BOT", +) + +DRCD_datasets = [ + dict( + type=DRCDDataset, + abbr='DRCD_dev', + path='./data/CLUE/DRCD/dev.json', + reader_cfg=DRCD_reader_cfg, + infer_cfg=DRCD_infer_cfg, + eval_cfg=DRCD_eval_cfg), +] diff --git a/configs/datasets/FewCLUE_bustm/FewCLUE_bustm_gen_305431.py b/configs/datasets/FewCLUE_bustm/FewCLUE_bustm_gen_305431.py new file mode 100644 index 00000000..b9698c7a --- /dev/null +++ b/configs/datasets/FewCLUE_bustm/FewCLUE_bustm_gen_305431.py @@ -0,0 +1,50 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import AFQMCDataset_V2 + +bustm_reader_cfg = dict( + input_columns=["sentence1", "sentence2"], + output_column="label", + test_split="train") + +bustm_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt= + "语句一：“{sentence1}”\n语句二：“{sentence2}”\n请判断语句一和语句二说的是否是一个意思？\nA. 无关\nB. 相关\n请从“A”，“B”中进行选择。\n答：", + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +bustm_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), +) + +bustm_datasets = [ + dict( + abbr="bustm-dev", + type=AFQMCDataset_V2, # bustm share the same format with AFQMC + path="./data/FewCLUE/bustm/dev_few_all.json", + reader_cfg=bustm_reader_cfg, + infer_cfg=bustm_infer_cfg, + eval_cfg=bustm_eval_cfg, + ), + dict( + abbr="bustm-test", + type=AFQMCDataset_V2, # bustm share the same format with AFQMC + path="./data/FewCLUE/bustm/test_public.json", + reader_cfg=bustm_reader_cfg, + infer_cfg=bustm_infer_cfg, + eval_cfg=bustm_eval_cfg, + ), +] diff --git a/configs/datasets/FewCLUE_chid/FewCLUE_chid_ppl_b6cd88.py b/configs/datasets/FewCLUE_chid/FewCLUE_chid_ppl_b6cd88.py new file mode 100644 index 00000000..0a30b0fb --- /dev/null +++ b/configs/datasets/FewCLUE_chid/FewCLUE_chid_ppl_b6cd88.py @@ -0,0 +1,45 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import CHIDDataset + +chid_reader_cfg = dict( + input_columns=[f'content{i}' for i in range(7)], output_column='answer') + +chid_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + i: dict( + round=[ + dict(role="HUMAN", prompt=f"以下句子是否通顺？\n{{content{i}}}"), + dict(role="BOT", prompt="这个句子是通顺的。"), + ], ) + for i in range(7) + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +chid_eval_cfg = dict(evaluator=dict(type=AccEvaluator), pred_role="BOT") + +chid_datasets = [ + dict( + type=CHIDDataset, + path='json', + abbr='chid-dev', + data_files='./data/FewCLUE/chid/dev_few_all.json', + split='train', + reader_cfg=chid_reader_cfg, + infer_cfg=chid_infer_cfg, + eval_cfg=chid_eval_cfg), + dict( + type=CHIDDataset, + path='json', + abbr='chid-test', + data_files='./data/FewCLUE/chid/test_public.json', + split='train', + reader_cfg=chid_reader_cfg, + infer_cfg=chid_infer_cfg, + eval_cfg=chid_eval_cfg), +] diff --git a/configs/datasets/FewCLUE_cluewsc/FewCLUE_cluewsc_ppl_fc45f0.py b/configs/datasets/FewCLUE_cluewsc/FewCLUE_cluewsc_ppl_fc45f0.py new file mode 100644 index 00000000..7f6308a6 --- /dev/null +++ b/configs/datasets/FewCLUE_cluewsc/FewCLUE_cluewsc_ppl_fc45f0.py @@ -0,0 +1,58 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import CluewscDataset + +cluewsc_reader_cfg = dict( + input_columns=['span1', 'span2', 'text', 'new_text'], + output_column='answer') + +cluewsc_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: + dict(round=[ + dict( + role="HUMAN", + prompt= + "{text}\nHere, is the pronoun \"{span2}\" used to mean \"{span1}\"?" + ), + dict(role="BOT", prompt="No.") + ]), + 1: + dict(round=[ + dict( + role="HUMAN", + prompt= + "{text}\nHere, is the pronoun \"{span2}\" used to mean \"{span1}\"?" + ), + dict(role="BOT", prompt="Yes.") + ]), + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +cluewsc_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +cluewsc_datasets = [ + dict( + type=CluewscDataset, + path='json', + abbr='cluewsc-dev', + data_files='./data/FewCLUE/cluewsc/dev_few_all.json', + split='train', + reader_cfg=cluewsc_reader_cfg, + infer_cfg=cluewsc_infer_cfg, + eval_cfg=cluewsc_eval_cfg), + dict( + type=CluewscDataset, + path='json', + abbr='cluewsc-test', + data_files='./data/FewCLUE/cluewsc/test_public.json', + split='train', + reader_cfg=cluewsc_reader_cfg, + infer_cfg=cluewsc_infer_cfg, + eval_cfg=cluewsc_eval_cfg), +] diff --git a/configs/datasets/FewCLUE_tnews/FewCLUE_tnews_ppl_784b9e.py b/configs/datasets/FewCLUE_tnews/FewCLUE_tnews_ppl_784b9e.py new file mode 100644 index 00000000..6ca84485 --- /dev/null +++ b/configs/datasets/FewCLUE_tnews/FewCLUE_tnews_ppl_784b9e.py @@ -0,0 +1,48 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import TNewsDataset + +tnews_reader_cfg = dict(input_columns='sentence', output_column='label_desc2') + +tnews_labels = [ + '农业新闻', '旅游新闻', '游戏新闻', '科技类别公司新闻', '体育类别新闻', '初升高教育新闻', '娱乐圈新闻', '投资资讯', + '军事类别常识', '车辆新闻', '楼市新闻', '环球不含中国类别新闻', '书籍文化历史类别新闻', '故事类别新闻', '股票市场类别新闻' +] + +tnews_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + lb: dict(round=[ + dict(role='HUMAN', prompt='{sentence}\n上述内容属于什么新闻？'), + dict(role='BOT', prompt=lb) + ]) + for lb in tnews_labels + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +tnews_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +tnews_datasets = [ + dict( + type=TNewsDataset, + path='json', + abbr='tnews-dev', + data_files='./data/FewCLUE/tnews/dev_few_all.json', + split='train', + reader_cfg=tnews_reader_cfg, + infer_cfg=tnews_infer_cfg, + eval_cfg=tnews_eval_cfg), + dict( + type=TNewsDataset, + path='json', + abbr='tnews-test', + data_files='./data/FewCLUE/tnews/test_public.json', + split='train', + reader_cfg=tnews_reader_cfg, + infer_cfg=tnews_infer_cfg, + eval_cfg=tnews_eval_cfg) +] diff --git a/configs/datasets/GaokaoBench/GaokaoBench_gen.py b/configs/datasets/GaokaoBench/GaokaoBench_gen.py new file mode 100644 index 00000000..26aa3fd2 --- /dev/null +++ b/configs/datasets/GaokaoBench/GaokaoBench_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .GaokaoBench_gen_aed980 import GaokaoBench_datasets # noqa: F401, F403 diff --git a/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_gen_7a5dee.py b/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_gen_7a5dee.py new file mode 100644 index 00000000..b879cedd --- /dev/null +++ b/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_gen_7a5dee.py @@ -0,0 +1,42 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import AXDataset_V2 + +AX_g_reader_cfg = dict( + input_columns=["hypothesis", "premise"], + output_column="label", +) + +AX_g_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt= + "{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:" + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +AX_g_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), +) + +AX_g_datasets = [ + dict( + abbr="AX_g", + type=AXDataset_V2, + path="./data/SuperGLUE/AX-g/AX-g.jsonl", + reader_cfg=AX_g_reader_cfg, + infer_cfg=AX_g_infer_cfg, + eval_cfg=AX_g_eval_cfg, + ) +] diff --git a/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_ppl_8d9bf9.py b/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_ppl_8d9bf9.py new file mode 100644 index 00000000..24609587 --- /dev/null +++ b/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_ppl_8d9bf9.py @@ -0,0 +1,53 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +AX_g_reader_cfg = dict( + input_columns=["hypothesis", "premise"], + output_column="label", + test_split="train") + +AX_g_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + "entailment": + dict(round=[ + dict( + role="HUMAN", + prompt= + "{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?" + ), + dict(role="BOT", prompt="Yes"), + ]), + "not_entailment": + dict(round=[ + dict( + role="HUMAN", + prompt= + "{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?" + ), + dict(role="BOT", prompt="No"), + ]) + }, + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer), +) + +AX_g_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +AX_g_datasets = [ + dict( + type=HFDataset, + abbr="AX_g", + path="json", + data_files="./data/SuperGLUE/AX-g/AX-g.jsonl", + split="train", + reader_cfg=AX_g_reader_cfg, + infer_cfg=AX_g_infer_cfg, + eval_cfg=AX_g_eval_cfg, + ) +] diff --git a/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl.py b/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl.py new file mode 100644 index 00000000..d2f2d5ed --- /dev/null +++ b/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .SuperGLUE_BoolQ_ppl_f80fb0 import BoolQ_datasets # noqa: F401, F403 diff --git a/configs/datasets/SuperGLUE_CB/SuperGLUE_CB_gen.py b/configs/datasets/SuperGLUE_CB/SuperGLUE_CB_gen.py new file mode 100644 index 00000000..7fe6c83d --- /dev/null +++ b/configs/datasets/SuperGLUE_CB/SuperGLUE_CB_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .SuperGLUE_CB_gen_bb97e1 import CB_datasets # noqa: F401, F403 diff --git a/configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_gen_6d5e67.py b/configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_gen_6d5e67.py new file mode 100644 index 00000000..879390b9 --- /dev/null +++ b/configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_gen_6d5e67.py @@ -0,0 +1,43 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import COPADataset_V2 + +COPA_reader_cfg = dict( + input_columns=["question", "premise", "choice1", "choice2"], + output_column="label", +) + +COPA_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict( + role="HUMAN", + prompt= + "{premise}\nQuestion: Which may be the {question}?\nA. {choice1}\nB. {choice2}\nAnswer:" + ), + ], ), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +COPA_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), +) + +COPA_datasets = [ + dict( + abbr="COPA", + type=COPADataset_V2, + path="./data/SuperGLUE/COPA/val.jsonl", + reader_cfg=COPA_reader_cfg, + infer_cfg=COPA_infer_cfg, + eval_cfg=COPA_eval_cfg, + ) +] diff --git a/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_gen_26c9dc.py b/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_gen_26c9dc.py new file mode 100644 index 00000000..a3cce8e5 --- /dev/null +++ b/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_gen_26c9dc.py @@ -0,0 +1,42 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import MultiRCDataset_V2 + +MultiRC_reader_cfg = dict( + input_columns=["question", "text", "answer"], + output_column="label", +) + +MultiRC_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt= + "{text}\nQuestion: {question}\nAnswer: {answer}\nIs it true?\nA. Yes\nB. No\nAnswer:" + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +MultiRC_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), +) + +MultiRC_datasets = [ + dict( + abbr="MultiRC", + type=MultiRCDataset_V2, + path="./data/SuperGLUE/MultiRC/val.jsonl", + reader_cfg=MultiRC_reader_cfg, + infer_cfg=MultiRC_infer_cfg, + eval_cfg=MultiRC_eval_cfg, + ) +] diff --git a/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_gen_c39367.py b/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_gen_c39367.py new file mode 100644 index 00000000..cfa15e4a --- /dev/null +++ b/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_gen_c39367.py @@ -0,0 +1,46 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import WiCDataset_V2 + +WiC_reader_cfg = dict( + input_columns=[ + "word", + "sentence1", + "sentence2", + ], + output_column="label", +) + +WiC_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt= + "Sentence 1: {sentence1}\nSentence 2: {sentence2}\nAre '{word}' in the above two sentenses the same?\nA. Yes\nB. No\nAnswer:" + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +WiC_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), +) + +WiC_datasets = [ + dict( + abbr="WiC", + type=WiCDataset_V2, + path="./data/SuperGLUE/WiC/val.jsonl", + reader_cfg=WiC_reader_cfg, + infer_cfg=WiC_infer_cfg, + eval_cfg=WiC_eval_cfg, + ) +] diff --git a/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_ppl_4118db.py b/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_ppl_4118db.py new file mode 100644 index 00000000..e7e59441 --- /dev/null +++ b/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_ppl_4118db.py @@ -0,0 +1,55 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import WiCDataset + +WiC_reader_cfg = dict( + input_columns=[ + "word", + "sentence1", + "sentence2", + ], + output_column="answer", + test_split="train") + +WiC_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: + dict(round=[ + dict( + role="HUMAN", + prompt= + "Sentence 1: {sentence1}\nSentence 2: {sentence2}\n'{word}' in the above two sentenses are different." + ), + ]), + 1: + dict(round=[ + dict( + role="HUMAN", + prompt= + "Sentence 1: {sentence1}\nSentence 2: {sentence2}\n'{word}' in the above two sentenses are the same." + ), + ]), + }, + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer), +) + +WiC_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +WiC_datasets = [ + dict( + type=WiCDataset, + abbr="WiC", + path="json", + data_files="./data/SuperGLUE/WiC/val.jsonl", + split="train", + reader_cfg=WiC_reader_cfg, + infer_cfg=WiC_infer_cfg, + eval_cfg=WiC_eval_cfg, + ) +] diff --git a/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_ppl_d316eb.py b/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_ppl_d316eb.py new file mode 100644 index 00000000..1af5a93d --- /dev/null +++ b/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_ppl_d316eb.py @@ -0,0 +1,49 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import WiCDataset + +WiC_reader_cfg = dict( + input_columns=[ + "word", + "sentence1", + "sentence2", + ], + output_column="answer", + test_split="train") + +WiC_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: + dict(round=[ + dict( + role="HUMAN", + prompt="{word} in {sentence1} and {sentence2} is different."), + ]), + 1: + dict(round=[ + dict(role="HUMAN", prompt="{word} in {sentence1} and {sentence2} is same."), + ]), + }, + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer), +) + +WiC_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +WiC_datasets = [ + dict( + type=WiCDataset, + abbr="WiC", + path="json", + data_files="./data/SuperGLUE/WiC/val.jsonl", + split="train", + reader_cfg=WiC_reader_cfg, + infer_cfg=WiC_infer_cfg, + eval_cfg=WiC_eval_cfg, + ) +] diff --git a/configs/datasets/TheoremQA/TheoremQA_gen_74abc9.py b/configs/datasets/TheoremQA/TheoremQA_gen_74abc9.py new file mode 100644 index 00000000..8046ce40 --- /dev/null +++ b/configs/datasets/TheoremQA/TheoremQA_gen_74abc9.py @@ -0,0 +1,40 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import TheoremQADataset + +TheoremQA_reader_cfg = dict( + input_columns=['Question', 'Answer_type'], + output_column='Answer', + train_split='test') + +TheoremQA_prompt1 = "Please read a math problem, and then think step by step to derive the answer. The answer is decided by Answer Type. " \ + "If the Answer type in [bool], the answer needs to be True or False. " \ + "Else if the Answer type in [integer, float] , The answer needs to be in numerical form. " \ + "Else if the Answer type in [list of integer, list of float] , the answer needs to be a list of number like [2, 3, 4]. " \ + "Else if the Answer type in [option], the answer needs to be an option like (a), (b), (c), (d)." \ + "You need to output the answer in your final sentence like 'Therefore, the answer is ...'." +TheoremQA_prompt2 = f"Below is an instruction that describes a task, paired with an input that provides further context. " \ + f"Write a response that appropriately completes the request.\n\n### Instruction:\n{TheoremQA_prompt1}\n\n### Input:\n{{Question}}\nAnswer_type:{{Answer_type}}\n### Response:\n" + +TheoremQA_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=TheoremQA_prompt2), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +TheoremQA_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type='TheoremQA')) + +TheoremQA_datasets = [ + dict( + abbr='TheoremQA', + type=TheoremQADataset, + path="./data/TheoremQA/test.csv", + reader_cfg=TheoremQA_reader_cfg, + infer_cfg=TheoremQA_infer_cfg, + eval_cfg=TheoremQA_eval_cfg) +] diff --git a/configs/datasets/ceval/ceval_gen.py b/configs/datasets/ceval/ceval_gen.py new file mode 100644 index 00000000..cc33b7ea --- /dev/null +++ b/configs/datasets/ceval/ceval_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .ceval_gen_ee2cb0 import ceval_datasets # noqa: F401, F403 diff --git a/configs/datasets/civilcomments/civilcomments_ppl_e01497.py b/configs/datasets/civilcomments/civilcomments_ppl_e01497.py new file mode 100644 index 00000000..02e30c01 --- /dev/null +++ b/configs/datasets/civilcomments/civilcomments_ppl_e01497.py @@ -0,0 +1,31 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import CLPInferencer +from opencompass.openicl.icl_evaluator import AUCROCEvaluator +from opencompass.datasets import CivilCommentsDataset + +civilcomments_reader_cfg = dict( + input_columns=['text'], + output_column='label', + train_split='test', + test_split='test') + +civilcomments_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='Text: {text}\nQuestion: ' + 'Does the above text contain rude, hateful, aggressive, disrespectful ' + 'or unreasonable language?\nAnswer:'), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=CLPInferencer)) + +civilcomments_eval_cfg = dict(evaluator=dict(type=AUCROCEvaluator), ) + +civilcomments_datasets = [ + dict( + type=CivilCommentsDataset, + path='civil_comments', + reader_cfg=civilcomments_reader_cfg, + infer_cfg=civilcomments_infer_cfg, + eval_cfg=civilcomments_eval_cfg) +] diff --git a/configs/datasets/collections/chat_medium.py b/configs/datasets/collections/chat_medium.py new file mode 100644 index 00000000..5bfb05f3 --- /dev/null +++ b/configs/datasets/collections/chat_medium.py @@ -0,0 +1,57 @@ +from mmengine.config import read_base + +with read_base(): + from ..mmlu.mmlu_gen_a568f1 import mmlu_datasets + from ..ceval.ceval_gen_ee2cb0 import ceval_datasets + from ..agieval.agieval_gen_dc7dae import agieval_datasets + from ..GaokaoBench.GaokaoBench_gen_aed980 import GaokaoBench_datasets + from ..bbh.bbh_gen_58abc3 import bbh_datasets + from ..humaneval.humaneval_gen_d428f1 import humaneval_datasets + from ..mbpp.mbpp_gen_4104e4 import mbpp_datasets + from ..CLUE_C3.CLUE_C3_gen_9e3de9 import C3_datasets + from ..CLUE_CMRC.CLUE_CMRC_gen_72a8d5 import CMRC_datasets + from ..CLUE_DRCD.CLUE_DRCD_gen_03b96b import DRCD_datasets + from ..CLUE_afqmc.CLUE_afqmc_gen_db509b import afqmc_datasets + from ..CLUE_cmnli.CLUE_cmnli_gen_316313 import cmnli_datasets + from ..CLUE_ocnli.CLUE_ocnli_gen_7c44b0 import ocnli_datasets + from ..FewCLUE_bustm.FewCLUE_bustm_gen_305431 import bustm_datasets + from ..FewCLUE_chid.FewCLUE_chid_gen_686c63 import chid_datasets + from ..FewCLUE_cluewsc.FewCLUE_cluewsc_gen_276956 import cluewsc_datasets + from ..FewCLUE_csl.FewCLUE_csl_gen_1b0c02 import csl_datasets + from ..FewCLUE_eprstmt.FewCLUE_eprstmt_gen_d6d06d import eprstmt_datasets + from ..FewCLUE_ocnli_fc.FewCLUE_ocnli_fc_gen_bef37f import ocnli_fc_datasets + from ..FewCLUE_tnews.FewCLUE_tnews_gen_8d59ba import tnews_datasets + from ..lcsts.lcsts_gen_427fde import lcsts_datasets + from ..lambada.lambada_gen_7ffe3d import lambada_datasets + from ..storycloze.storycloze_gen_c5a230 import storycloze_datasets + from ..SuperGLUE_AX_b.SuperGLUE_AX_b_gen_477186 import AX_b_datasets + from ..SuperGLUE_AX_g.SuperGLUE_AX_g_gen_7a5dee import AX_g_datasets + from ..SuperGLUE_BoolQ.SuperGLUE_BoolQ_gen_8525d1 import BoolQ_datasets + from ..SuperGLUE_CB.SuperGLUE_CB_gen_bb97e1 import CB_datasets + from ..SuperGLUE_COPA.SuperGLUE_COPA_gen_6d5e67 import COPA_datasets + from ..SuperGLUE_MultiRC.SuperGLUE_MultiRC_gen_26c9dc import MultiRC_datasets + from ..SuperGLUE_RTE.SuperGLUE_RTE_gen_ce346a import RTE_datasets + from ..SuperGLUE_ReCoRD.SuperGLUE_ReCoRD_gen_d8f19c import ReCoRD_datasets + from ..SuperGLUE_WiC.SuperGLUE_WiC_gen_c39367 import WiC_datasets + from ..SuperGLUE_WSC.SuperGLUE_WSC_gen_d8d441 import WSC_datasets + from ..race.race_gen_12de48 import race_datasets + from ..Xsum.Xsum_gen_d2126e import Xsum_datasets + from ..gsm8k.gsm8k_gen_2dd372 import gsm8k_datasets + from ..summedits.summedits_gen_4f35b5 import summedits_datasets + from ..math.math_gen_78bcba import math_datasets + from ..TheoremQA.TheoremQA_gen_891fcf import TheoremQA_datasets + from ..hellaswag.hellaswag_gen_cae9cb import hellaswag_datasets + from ..ARC_e.ARC_e_gen_0a29bf import ARC_e_datasets + from ..ARC_c.ARC_c_gen_3f3039 import ARC_c_datasets + from ..commonsenseqa.commonsenseqa_gen_a58dbd import commonsenseqa_datasets + from ..piqa.piqa_gen_8287ae import piqa_datasets + from ..siqa.siqa_gen_a3c714 import siqa_datasets + from ..strategyqa.strategyqa_gen_be3f8d import strategyqa_datasets + from ..winogrande.winogrande_gen_c19d87 import winogrande_datasets + from ..obqa.obqa_gen_b2cde9 import obqa_datasets + from ..nq.nq_gen_a6ffca import nq_datasets + from ..triviaqa.triviaqa_gen_cc3cbf import triviaqa_datasets + from ..flores.flores_gen_8eb9ca import flores_datasets + from ..crowspairs.crowspairs_gen_dd110a import crowspairs_datasets + +datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), []) diff --git a/configs/datasets/commonsenseqa/commonsenseqa_gen_a58dbd.py b/configs/datasets/commonsenseqa/commonsenseqa_gen_a58dbd.py new file mode 100644 index 00000000..9a3d008c --- /dev/null +++ b/configs/datasets/commonsenseqa/commonsenseqa_gen_a58dbd.py @@ -0,0 +1,60 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import MDLRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import commonsenseqaDataset + +commonsenseqa_reader_cfg = dict( + input_columns=["question", "A", "B", "C", "D", "E"], + output_column="answerKey", + test_split="validation") + +_ice_template = dict( + type=PromptTemplate, + template=dict( + begin="", + round=[ + dict( + role="HUMAN", + prompt= + "{question}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nE. {E}\nAnswer:", + ), + dict( + role="BOT", + prompt="{answerKey}", + ), + ], + ), + ice_token="", +) + +commonsenseqa_infer_cfg = dict( + ice_template=_ice_template, + retriever=dict( + type=MDLRetriever, + ice_num=8, + candidate_num=30, + select_time=10, + seed=1, + batch_size=12, + ice_template=_ice_template, + ), + inferencer=dict(type=GenInferencer), +) + +commonsenseqa_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type="first-capital"), +) + +commonsenseqa_datasets = [ + dict( + type=commonsenseqaDataset, + path="commonsense_qa", + reader_cfg=commonsenseqa_reader_cfg, + infer_cfg=commonsenseqa_infer_cfg, + eval_cfg=commonsenseqa_eval_cfg, + ) +] + +del _ice_template diff --git a/configs/datasets/glm/C3.py b/configs/datasets/glm/C3.py new file mode 100644 index 00000000..36c70e04 --- /dev/null +++ b/configs/datasets/glm/C3.py @@ -0,0 +1,40 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import C3Dataset + +C3_reader_cfg = dict( + input_columns=[ + 'question', 'content', 'choice0', 'choice1', 'choice2', 'choice3', + 'choices' + ], + output_column='label') + +C3_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: + "阅读以下内容,选择合适的选项回答: {content} 问题：{question}\n 选项: -{choice0} -{choice1} -{choice2} -{choice3} 答: [MASK]-{choice0}", + 1: + "阅读以下内容,选择合适的选项回答: {content} 问题：{question}\n 选项: -{choice0} -{choice1} -{choice2} -{choice3} 答: [MASK]-{choice1}", + 2: + "阅读以下内容,选择合适的选项回答: {content} 问题：{question}\n 选项: -{choice0} -{choice1} -{choice2} -{choice3} 答: [MASK]-{choice2}", + 3: + "阅读以下内容,选择合适的选项回答: {content} 问题：{question}\n 选项: -{choice0} -{choice1} -{choice2} -{choice3} 答: [MASK]-{choice3}", + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +C3_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +C3_datasets = [ + dict( + type=C3Dataset, + abbr='C3', + path='./data/CLUE/C3/dev_0.json', + reader_cfg=C3_reader_cfg, + infer_cfg=C3_infer_cfg, + eval_cfg=C3_eval_cfg) +] diff --git a/configs/datasets/glm/tnews.py b/configs/datasets/glm/tnews.py new file mode 100644 index 00000000..7b4e651e --- /dev/null +++ b/configs/datasets/glm/tnews.py @@ -0,0 +1,41 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GLMChoiceInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import TNewsDataset + +tnews_reader_cfg = dict(input_columns='sentence', output_column='label_desc2') + +tnews_labels = [ + '农业新闻', '旅游新闻', '游戏新闻', '科技类别公司新闻', '体育类别新闻', '初升高教育新闻', '娱乐圈新闻', '投资资讯', + '军事类别常识', '车辆新闻', '楼市新闻', '环球不含中国类别新闻', '书籍文化历史类别新闻', '故事类别新闻', '股票市场类别新闻' +] + +tnews_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template={lb: f'这篇新闻属于：{lb}' + for lb in tnews_labels}, + column_token_map={'sentence': ''}, + ice_token=''), + prompt_template=dict( + type=PromptTemplate, + template='\n以上这篇新闻属于', + column_token_map={'sentence': ''}, + ice_token=''), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GLMChoiceInferencer, choices=tnews_labels)) + +tnews_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +tnews_datasets = [ + dict( + type=TNewsDataset, + path='json', + abbr='tnews', + data_files='./data/FewCLUE/tnews/test_public.json', + split='train', + reader_cfg=tnews_reader_cfg, + infer_cfg=tnews_infer_cfg, + eval_cfg=tnews_eval_cfg) +] diff --git a/configs/datasets/govrepcrs/govrepcrs_gen.py b/configs/datasets/govrepcrs/govrepcrs_gen.py new file mode 100644 index 00000000..8ff35942 --- /dev/null +++ b/configs/datasets/govrepcrs/govrepcrs_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .govrepcrs_gen_455586 import govrepcrs_datasets # noqa: F401, F403 diff --git a/configs/datasets/govrepcrs/govrepcrs_gen_455586.py b/configs/datasets/govrepcrs/govrepcrs_gen_455586.py new file mode 100644 index 00000000..9af1402a --- /dev/null +++ b/configs/datasets/govrepcrs/govrepcrs_gen_455586.py @@ -0,0 +1,47 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import BleuEvaluator +from opencompass.datasets import GovRepcrsDataset + +govrepcrs_reader_cfg = dict( + input_columns='content', + output_column='summary', + train_split='test', + test_split='test') + +govrepcrs_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + fallback_role="HUMAN", + prompt= + 'Please summarize the following English report in English:' + ), + ], + round=[ + dict(role='HUMAN', prompt='{content}'), + dict(role='BOT', prompt='{summary}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict( + type=GenInferencer, batch_size=4, max_out_len=500, max_seq_len=8192)) + +govrepcrs_eval_cfg = dict( + evaluator=dict(type=BleuEvaluator), + pred_role='BOT', + pred_postprocessor=dict(type='general_cn'), + dataset_postprocessor=dict(type='general_cn')) + +govrepcrs_datasets = [ + dict( + type=GovRepcrsDataset, + path='./data/govrep/', + abbr='GovRepcrs', + reader_cfg=govrepcrs_reader_cfg, + infer_cfg=govrepcrs_infer_cfg, + eval_cfg=govrepcrs_eval_cfg) +] diff --git a/configs/datasets/hellaswag/hellaswag_gen_cae9cb.py b/configs/datasets/hellaswag/hellaswag_gen_cae9cb.py new file mode 100644 index 00000000..5fc8d799 --- /dev/null +++ b/configs/datasets/hellaswag/hellaswag_gen_cae9cb.py @@ -0,0 +1,43 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import hellaswagDataset_V2 + +hellaswag_reader_cfg = dict( + input_columns=["ctx", "A", "B", "C", "D"], + output_column="label", + test_split="validation") + +hellaswag_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt=( + "{ctx}\nQuestion: Which ending makes the most sense?\n" + "A. {A}\nB. {B}\nC. {C}\nD. {D}\n" + "You may choose from 'A', 'B', 'C', 'D'.\n" + "Answer:"), + ), + ]), + ), + retriever=dict(type=ZeroRetriever, ), + inferencer=dict(type=GenInferencer), +) + +hellaswag_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), +) + +hellaswag_datasets = [ + dict( + type=hellaswagDataset_V2, + path="hellaswag", + reader_cfg=hellaswag_reader_cfg, + infer_cfg=hellaswag_infer_cfg, + eval_cfg=hellaswag_eval_cfg) +] diff --git a/configs/datasets/iwslt2017/iwslt2017_gen_02ea0b.py b/configs/datasets/iwslt2017/iwslt2017_gen_02ea0b.py new file mode 100644 index 00000000..a696c372 --- /dev/null +++ b/configs/datasets/iwslt2017/iwslt2017_gen_02ea0b.py @@ -0,0 +1,38 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import BM25Retriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import BleuEvaluator +from opencompass.datasets import IWSLT2017Dataset + +iwslt2017_reader_cfg = dict( + input_columns='en', output_column='de', train_split='validation') + +iwslt2017_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template=dict( + begin='', + round=[ + dict(role='HUMAN', prompt='Please translate the following English statements to German:\n{en}'), + dict(role='BOT', prompt='{de}'), + ] + ), + ice_token=''), + retriever=dict(type=BM25Retriever, ice_num=1), + inferencer=dict(type=GenInferencer)) + +iwslt2017_eval_cfg = dict( + evaluator=dict(type=BleuEvaluator), + pred_role='BOT', + pred_postprocessor=dict(type='general_cn'), + dataset_postprocessor=dict(type='general_cn')) + +iwslt2017_datasets = [ + dict( + type=IWSLT2017Dataset, + path='iwslt2017', + name='iwslt2017-en-de', + reader_cfg=iwslt2017_reader_cfg, + infer_cfg=iwslt2017_infer_cfg, + eval_cfg=iwslt2017_eval_cfg) +] \ No newline at end of file diff --git a/configs/datasets/lcsts/lcsts_gen.py b/configs/datasets/lcsts/lcsts_gen.py new file mode 100644 index 00000000..d3cc71b9 --- /dev/null +++ b/configs/datasets/lcsts/lcsts_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .lcsts_gen_427fde import lcsts_datasets # noqa: F401, F403 diff --git a/configs/datasets/mbpp/mbpp_gen_b60b47.py b/configs/datasets/mbpp/mbpp_gen_b60b47.py new file mode 100644 index 00000000..18facf02 --- /dev/null +++ b/configs/datasets/mbpp/mbpp_gen_b60b47.py @@ -0,0 +1,64 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import MBPPDataset, MBPPEvaluator + +mbpp_reader_cfg = dict( + input_columns=['text', 'test_list'], output_column='code') + +mbpp_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict( + role="HUMAN", + prompt= + "You are an expert Python programmer, and here is your task: Write a function to find the similar elements from the given two tuple lists. Your code should pass these tests:\n\n assert similar_elements((3, 4, 5, 6),(5, 7, 4, 10)) == (4, 5)\n assert similar_elements((1, 2, 3, 4),(5, 4, 3, 7)) == (3, 4) \n assert similar_elements((11, 12, 14, 13),(17, 15, 14, 13)) == (13, 14) \n" + ), + dict( + role="BOT", + prompt= + "[BEGIN]\n 'def similar_elements(test_tup1, test_tup2):\r\n res = tuple(set(test_tup1) & set(test_tup2))\r\n return (res)' \n[DONE] \n\n " + ), + dict( + role="HUMAN", + prompt= + "You are an expert Python programmer, and here is your task: Write a python function to identify non-prime numbers. Your code should pass these tests:\n\n assert is_not_prime(2) == False \n assert is_not_prime(10) == True \n assert is_not_prime(35) == True \n" + ), + dict( + role="BOT", + prompt= + "[BEGIN]\n 'import math\r\ndef is_not_prime(n):\r\n result = False\r\n for i in range(2,int(math.sqrt(n)) + 1):\r\n if n % i == 0:\r\n result = True\r\n return result' \n[DONE] \n\n " + ), + dict( + role="HUMAN", + prompt= + "You are an expert Python programmer, and here is your task: Write a function to find the largest integers from a given list of numbers using heap queue algorithm. Your code should pass these tests:\n\n assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],3)==[85, 75, 65] \n assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],2)==[85, 75] \n assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],5)==[85, 75, 65, 58, 35] \n" + ), + dict( + role="BOT", + prompt= + "[BEGIN]\n 'import heapq as hq\r\ndef heap_queue_largest(nums,n):\r\n largest_nums = hq.nlargest(n, nums)\r\n return largest_nums' \n[DONE] \n\n " + ), + dict( + role="HUMAN", + prompt= + "You are an expert Python programmer, and here is your task: {text} Your code should pass these tests:\n\n {test_list} \n" + ), + dict(role="BOT", prompt="[BEGIN]\n"), + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +mbpp_eval_cfg = dict(evaluator=dict(type=MBPPEvaluator), pred_role="BOT") + +mbpp_datasets = [ + dict( + type=MBPPDataset, + abbr='mbpp', + path='./data/mbpp/mbpp.jsonl', + reader_cfg=mbpp_reader_cfg, + infer_cfg=mbpp_infer_cfg, + eval_cfg=mbpp_eval_cfg) +] diff --git a/configs/datasets/narrativeqa/narrativeqa_gen_ca4b64.py b/configs/datasets/narrativeqa/narrativeqa_gen_ca4b64.py new file mode 100644 index 00000000..37de9e64 --- /dev/null +++ b/configs/datasets/narrativeqa/narrativeqa_gen_ca4b64.py @@ -0,0 +1,30 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import NarrativeQADataset, TriviaQAEvaluator + +narrativeqa_reader_cfg = dict( + input_columns=['question', 'evidence'], + output_column='answer', + train_split='valid', + test_split='valid') + +narrativeqa_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template="{evidence}\nAnswer these questions:\nQ: {question}?\nA:"), + retriever=dict(type=ZeroRetriever), + inferencer=dict( + type=GenInferencer, max_out_len=50, max_seq_len=8192, batch_size=4)) + +narrativeqa_eval_cfg = dict(evaluator=dict(type=TriviaQAEvaluator)) + +narrativeqa_datasets = [ + dict( + type=NarrativeQADataset, + abbr='NarrativeQA', + path='./data/narrativeqa/', + reader_cfg=narrativeqa_reader_cfg, + infer_cfg=narrativeqa_infer_cfg, + eval_cfg=narrativeqa_eval_cfg) +] diff --git a/configs/datasets/obqa/obqa_ppl.py b/configs/datasets/obqa/obqa_ppl.py new file mode 100644 index 00000000..82a03738 --- /dev/null +++ b/configs/datasets/obqa/obqa_ppl.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .obqa_ppl_2b5b12 import obqa_datasets # noqa: F401, F403 diff --git a/configs/datasets/obqa/obqa_ppl_2b5b12.py b/configs/datasets/obqa/obqa_ppl_2b5b12.py new file mode 100644 index 00000000..9a4c8546 --- /dev/null +++ b/configs/datasets/obqa/obqa_ppl_2b5b12.py @@ -0,0 +1,66 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import OBQADataset + +_input_columns = [ + ['question_stem', 'A', 'B', 'C', 'D'], + ['question_stem', 'A', 'B', 'C', 'D', 'fact1'], +] +_template = [ + { + ans: dict( + round=[ + dict( + role="HUMAN", + prompt= + "Question: {question_stem}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer:" + ), + dict(role="BOT", prompt=ans), + ], ) + for ans in ['A', 'B', 'C', 'D'] + }, + { + ans: dict( + round=[ + dict( + role="HUMAN", + prompt= + "Given the fact: {fact1}\nQuestion: {question_stem}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer:" + ), + dict(role="BOT", prompt=ans), + ], ) + for ans in ['A', 'B', 'C', 'D'] + } +] + +obqa_datasets = [ + dict( + type=OBQADataset, + path='openbookqa', + split='test', + ), + dict( + abbr='openbookqa_fact', + type=OBQADataset, + path='openbookqa', + name='additional', + split='test', + ), +] +for _i in range(2): + obqa_reader_cfg = dict( + input_columns=_input_columns[_i], output_column="answerKey") + obqa_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=_template[_i]), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer), + ) + obqa_eval_cfg = dict(evaluator=dict(type=AccEvaluator), ) + + obqa_datasets[_i]["reader_cfg"] = obqa_reader_cfg + obqa_datasets[_i]["infer_cfg"] = obqa_infer_cfg + obqa_datasets[_i]["eval_cfg"] = obqa_eval_cfg diff --git a/configs/datasets/obqa/obqa_ppl_3609cc.py b/configs/datasets/obqa/obqa_ppl_3609cc.py new file mode 100644 index 00000000..bb07200f --- /dev/null +++ b/configs/datasets/obqa/obqa_ppl_3609cc.py @@ -0,0 +1,52 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import OBQADataset + +_input_columns = [ + ['question_stem', 'A', 'B', 'C', 'D'], + ['question_stem', 'A', 'B', 'C', 'D', 'fact1'], +] +_template = [{ + 'A': "{question_stem} {A}", + 'B': "{question_stem} {B}", + 'C': "{question_stem} {C}", + 'D': "{question_stem} {D}", +}, { + 'A': "Given the fact {fact1}, we know that {question_stem} {A}", + 'B': "Given the fact {fact1}, we know that {question_stem} {B}", + 'C': "Given the fact {fact1}, we know that {question_stem} {C}", + 'D': "Given the fact {fact1}, we know that {question_stem} {D}", +}] + +obqa_datasets = [ + dict( + abbr="openbookqa", + type=OBQADataset, + path="openbookqa", + split="test", + ), + dict( + abbr="openbookqa_fact", + type=OBQADataset, + path="openbookqa", + name="additional", + split="test", + ), +] +for _i in range(2): + obqa_reader_cfg = dict( + input_columns=_input_columns[_i], output_column="answerKey") + obqa_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=_template[_i]), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer), + ) + obqa_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + + obqa_datasets[_i]["reader_cfg"] = obqa_reader_cfg + obqa_datasets[_i]["infer_cfg"] = obqa_infer_cfg + obqa_datasets[_i]["eval_cfg"] = obqa_eval_cfg diff --git a/configs/datasets/realtoxicprompts/realtoxicprompts_gen_82a2f2.py b/configs/datasets/realtoxicprompts/realtoxicprompts_gen_82a2f2.py new file mode 100644 index 00000000..91a5e0e3 --- /dev/null +++ b/configs/datasets/realtoxicprompts/realtoxicprompts_gen_82a2f2.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import ToxicEvaluator +from opencompass.datasets import RealToxicPromptsDataset + +realtoxicprompts_reader_cfg = dict( + input_columns=['prompt_text'], + output_column='filename', + train_split='train', + test_split='train') + +# TODO: allow empty output-column +realtoxicprompts_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[dict(role="HUMAN", prompt="{prompt_text}")])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +# When key is set to "ENV", the key will be fetched from the environment +# variable $PerspectiveAPIkey. Otherwise, set key in here directly. +realtoxicprompts_eval_cfg = dict( + evaluator=dict(type=ToxicEvaluator, key='ENV'), + pred_role='BOT', +) + +realtoxicprompts_datasets = [ + dict( + type=RealToxicPromptsDataset, + path='allenai/real-toxicity-prompts', + challenging_subset=True, + reader_cfg=realtoxicprompts_reader_cfg, + infer_cfg=realtoxicprompts_infer_cfg, + eval_cfg=realtoxicprompts_eval_cfg) +] diff --git a/configs/datasets/storycloze/storycloze_ppl_c1912d.py b/configs/datasets/storycloze/storycloze_ppl_c1912d.py new file mode 100644 index 00000000..05aea9ba --- /dev/null +++ b/configs/datasets/storycloze/storycloze_ppl_c1912d.py @@ -0,0 +1,39 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import storyclozeDataset + +storycloze_reader_cfg = dict( + input_columns=['context', 'sentence_quiz1', 'sentence_quiz2'], + output_column='answer_right_ending', + train_split='test', + test_split='test') + +storycloze_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + i: dict(round=[ + dict(role="HUMAN", prompt="{context}"), + dict(role="BOT", prompt=f"{{sentence_quiz{i}}}"), + ]) + for i in range(1, 3) + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +storycloze_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +# The original story cloze dataset and repo are not long maintaining. +# Using multilingual version of this dataset. +storycloze_datasets = [ + dict( + abbr='story_cloze', + type=storyclozeDataset, + path='juletxara/xstory_cloze', + name='en', + reader_cfg=storycloze_reader_cfg, + infer_cfg=storycloze_infer_cfg, + eval_cfg=storycloze_eval_cfg) +] diff --git a/configs/datasets/summedits/summedits_ppl_163352.py b/configs/datasets/summedits/summedits_ppl_163352.py new file mode 100644 index 00000000..57171a98 --- /dev/null +++ b/configs/datasets/summedits/summedits_ppl_163352.py @@ -0,0 +1,50 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +summedits_reader_cfg = dict( + input_columns=['doc', 'summary'], + output_column='label', + test_split='train') + +summedits_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: + dict(round=[ + dict( + role="HUMAN", + prompt= + """\nDocument:\n{doc}\nSummary:\n{summary}\nIs the summary factually consistent with the document? """ + ), + dict(role="BOT", prompt="No") + ]), + 1: + dict(round=[ + dict( + role="HUMAN", + prompt= + """Document:\n{doc}\nSummary:\n{summary}\nIs the summary factually consistent with the document? """ + ), + dict(role="BOT", prompt="Yes") + ]), + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +summedits_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +summedits_datasets = [ + dict( + type=HFDataset, + abbr='summedits', + path='json', + split='train', + data_files='./data/summedits/summedits.jsonl', + reader_cfg=summedits_reader_cfg, + infer_cfg=summedits_infer_cfg, + eval_cfg=summedits_eval_cfg) +] diff --git a/configs/datasets/summscreen/summscreen_gen_997ee2.py b/configs/datasets/summscreen/summscreen_gen_997ee2.py new file mode 100644 index 00000000..c1729348 --- /dev/null +++ b/configs/datasets/summscreen/summscreen_gen_997ee2.py @@ -0,0 +1,35 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import BleuEvaluator +from opencompass.datasets import SummScreenDataset + +summscreen_reader_cfg = dict( + input_columns='content', + output_column='summary', + train_split='dev', + test_split='dev') + +summscreen_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template= + "Please summarize the following English report in English:{content}\n{summary}."), + retriever=dict(type=ZeroRetriever), + inferencer=dict( + type=GenInferencer, batch_size=4, max_out_len=500, max_seq_len=8192)) + +summscreen_eval_cfg = dict( + evaluator=dict(type=BleuEvaluator), + pred_postprocessor=dict(type='general_cn'), + dataset_postprocessor=dict(type='general_cn')) + +summscreen_datasets = [ + dict( + type=SummScreenDataset, + path='./data/SummScreen/', + abbr='SummScreen', + reader_cfg=summscreen_reader_cfg, + infer_cfg=summscreen_infer_cfg, + eval_cfg=summscreen_eval_cfg) +] diff --git a/configs/datasets/winograd/winograd_ppl.py b/configs/datasets/winograd/winograd_ppl.py new file mode 100644 index 00000000..a3dbf68b --- /dev/null +++ b/configs/datasets/winograd/winograd_ppl.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .winograd_ppl_c1c427 import winograd_datasets # noqa: F401, F403 diff --git a/configs/datasets/z_bench/z_bench_gen_61db0a.py b/configs/datasets/z_bench/z_bench_gen_61db0a.py new file mode 100644 index 00000000..63cfded8 --- /dev/null +++ b/configs/datasets/z_bench/z_bench_gen_61db0a.py @@ -0,0 +1,28 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HFDataset + +z_bench_reader_cfg = dict( + ds_size=4, + input_columns=['text'], + output_column='category', + train_split='test') + +z_bench_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[dict(role="HUMAN", prompt="{text}")]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +z_bench_dataset = dict( + type=HFDataset, + path= + '/mnt/petrelfs/gaotong/llm_eval/openagieval_dataset/eval_datasets/z_bench', + data_dir= + '/mnt/petrelfs/gaotong/llm_eval/openagieval_dataset/eval_datasets/z_bench', + name='question', + reader_cfg=z_bench_reader_cfg, + infer_cfg=z_bench_infer_cfg) diff --git a/docs/en/Makefile b/docs/en/Makefile new file mode 100644 index 00000000..d4bb2cbb --- /dev/null +++ b/docs/en/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/en/prompt/prompt_template.md b/docs/en/prompt/prompt_template.md new file mode 100644 index 00000000..468f5524 --- /dev/null +++ b/docs/en/prompt/prompt_template.md @@ -0,0 +1 @@ +# Prompt Template \ No newline at end of file diff --git a/docs/en/user_guides/models.md b/docs/en/user_guides/models.md new file mode 100644 index 00000000..c93550a7 --- /dev/null +++ b/docs/en/user_guides/models.md @@ -0,0 +1 @@ +# Prepare Models diff --git a/docs/zh_cn/_static/css/readthedocs.css b/docs/zh_cn/_static/css/readthedocs.css new file mode 100644 index 00000000..1891c912 --- /dev/null +++ b/docs/zh_cn/_static/css/readthedocs.css @@ -0,0 +1,62 @@ +.header-logo { + background-image: url("../image/logo.png"); + background-size: 183px 50px; + height: 50px; + width: 183px; +} + +@media screen and (min-width: 1100px) { + .header-logo { + top: -12px; + } +} + +pre { + white-space: pre; +} + +@media screen and (min-width: 2000px) { + .pytorch-content-left { + width: 1200px; + margin-left: 30px; + } + article.pytorch-article { + max-width: 1200px; + } + .pytorch-breadcrumbs-wrapper { + width: 1200px; + } + .pytorch-right-menu.scrolling-fixed { + position: fixed; + top: 45px; + left: 1580px; + } +} + + +article.pytorch-article section code { + padding: .2em .4em; + background-color: #f3f4f7; + border-radius: 5px; +} + +/* Disable the change in tables */ +article.pytorch-article section table code { + padding: unset; + background-color: unset; + border-radius: unset; +} + +table.autosummary td { + width: 50% +} + +img.align-center { + display: block; + margin-left: auto; + margin-right: auto; +} + +article.pytorch-article p.rubric { + font-weight: bold; +} diff --git a/docs/zh_cn/_static/image/logo.png b/docs/zh_cn/_static/image/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..8b5efee6f2263d3ff798682e8a266d830b002e3c GIT binary patch literal 12650 zcmV-wF_q4VP)!@i001BWNkl zd7KYu|gexH^DkzEqA%FI+?Rw? zASd@J0tp}@BKVaX!Wqc5$L!2>S3S?~kDi&GnVs1jy9pcV_iO*y>FKVn>ZyAAsi&Uj zDH0J3CBiVw7lNitnuww!B4Vwz)@h@ib&ffcP7zVQPzXay1g*3&MjM^Y_N09OkdVHAyH-9j( z*5&erbSeeFI%h*`onvOLmDbw#Ja}IDWUaNewkeYVz|3J75|MMxcm_aICasi;z6!FT zW7bM(t%)f5G5X$JYi+GtM@@~1S?8?p<7)^>E9YD;pVwL!g1|YaloAoGbvl(&s<-h1 zL?4XSIhLMW&N&tl&-0QA-xw2Vi>f5il~P(MXPx#HfK)Qsr~+E+Sh0%vZ{@#Wd<_+8 z6$%Iy_PzXY6qetnSv(7o|taF)6S}R3F3Ex*rC480b>G6yKBwr{9 z*UoLV(at%*17M8)5SkYg^P%$JF}{Y1v;vS!Bz!+oM2ihpikLZ)?f|q>#xu!eqAb@F z3Ew(roh724Tyeszl&YMQJIB%GF)<%y{tL#}P?1)iDb|FUopaW)bB>t- z5viFEcV~M<#29U~7LiEL%ohsf$IUFceBNlCNciivK`}8yh-c8e-JcGr)S#6~5OiG(q_CzsC^NNepJXM1`)&+~n+yvmSHC8Gyto$KkzMUUHB zm+*_vPHWZNl=;xw6%#XL5Rq(8j+rABsXV?2SYO81x|3EL@BEY9w;g}=MMrrE1s*Y= z!f37wQqDm`KeSR&1Vaj{VtqMl#LJ$DZT4gfnLZOh&_IRF2CK5zLZhWm9X(i4%lEh!0 zd2`&vb3p`=zu*0%4O>cgC-|BHShg(5gakpVXv#B2t30QMx9674{?uOQq{zUygm0d` z=Z8sERr#i-Ox<4p)kJG zI=KsJG5`Yt03rZ5bfu+I03fJ09{JJhkDT?RW4HGVv(u3}s+3YH^mIyC(nvZbj#W>d z_c`R}wfFe2A7xC;dO@L3FuG!V5n0#k1{G;fHKGbbV|=X>X(cFFWN0}LmI$^e zz9O_Xp2GcqTe{u0TOPFMcni3YgCG=1SSjo^=?5=Ae)>K;ZKi|)e(m+zWmF3b*mcpI zn3(kzYi$_XWHJH3ISztQP&EvPGBjwBHyn$$T z6)O+TJgPh)WwwWY`-VJ-nHc)IUrzHBj5f)UGXU_4x$WP&ZG)DsOE35a<*iEgihlt% z+u;|ay2p=cd2rUL1YTEaV@%9?12bo{J$}NE%;edg90A6dl4rwEXV8YUC{ZT$i{C#r zcH()P?0nw1iRW#ys@GW`3}=r|(td^2DCgJr-%8(tqD{`y(#6>X9=qTL5Pm z-2foQOYB6zk1u=3RmE~!v6i41WPR*^_yuPi+bgOpdWv>LLG>+SVq(@O2G7b4APV3X z2hpJ5%ST+n==H~caON?6mq0CG1neZQDFr4{rQ?=hF2Cu~C0%s-Q9EtF`!Ub_^8)}P z&Mup+{KWlFy|U*Q#z!YR;FxpnN?AKr4ec~1=pwe?9 zCMITmVUVR&FJe0bedI|OtjM>Kq(iUc(_=Rcw9;%8<#*j?^dZwHRGiJ%-}%hDOSDq< z%e#Dh+p*0xg#`}D)@dLo@Rq!D(}-s6pvwmQKB=Y8j(=MO?=)0`6U$7%v64zaF;#OOZ z(7kEA8r(_p4!)x(^hg*`V1NM8&>tT&?9@YBSSs^^0f10&`{N&o1A(PfK2=-Tk3M|+ zi)TLf&)1jix$~x`!UvZ@qg#Cdjtk?r+~n@-j(q0%rPGeR@ZAOH6ypd){~1#WcrSyi=V~4 zzPM}LR)-N6?`p&b-FFqg6d?k@B3dL+s$?k>oN(Tq076Sk7QLL6G})#1UH?B|0Z}{` zjEPzASQFh!s)sr&7-4W0z^iXA%M5QGnekMl0a1k_RMv)y2B#2`0Ic$A>#*0hqaJzY z_0X}W0Vt5(Y*aI`0g4NOA(Ma!3V;aBUhsBGh5JqzOC?ct-@@0vxYIa74oE7G=3Mh_ zDrm_TVTWvPse5;(Q_6G2{=e33Zd&CcCcXVbekI%j6u$3z^g8LV~w&nN@ zHrk*mQO~DBL{#g6B!%*kLk|4HbyuBL{^%U*$PHBhxL@D!4-rpu_sK`>91EwInEwt| z+uuq=#%$Q(scx!Pu`d73(GYZYcGsToR%w?D5qakI?gthwt{a(5AW%?w57$pWPTc#$NTcj*yID+oElBMV-=`p@?oQ?woOtXk{E%s%dP7PYpZf)r3t?nmBIFTND!$GlUuB+@VHmh=4?a_L#WE z9y^^-{%VU2y*J)3I5vW$2A7TPcRcYnn25Xg+xg>-iWR^XmKtxgNS4VI^YJ*P5dvNm zDNSrd^=f7$3_{UaD<&pp{b#NDviGI9L|B|bmUW`}kH5u43CKP>Zqn$|NdF30HzoJB z$z&v~lyF;{eDLO-Ebv+n5DA3XSE(RacGelkoc*JN1)!BUfyKnctT(JfY1I%0)XIEI zdrx+jSW&8FKh~i*BVBix)LvqeQs;6-U}gs-m644%l$T$aqX`O*L_+|zj*|9qBsZh~ z2|z)^hs17QePKxH7a@*S=?n3;&k2s;Z_P#RF6$n`-;RIxD81SSC_f7u_tbM)6I3wp#x8&l9==wl2~xjdRS zm?qReuHUGR)KE-FwR9^Z8^lIaoCdS-O@N4*mLBfB(k|OO|(wOOl2-tBeNuYzw4) z#S&(3Vo8CZE3z94goHpWV1b4RMGovD72cs1Ag~6N{9VNnIJBr))?5Re6EkeGc1hMj z!;{1)IMPW^ebBS~p#k1;J;iY@G3*GI02FFxoT}}OBw)=5eD0Y&1oknA`7l{MX|+i? zJE$lSfdXYa&pGXYqh{<(h+=EX;=o$#wAMr}2m{}12JpZm&)Z!fwF;?+3Q)kR1lyI9;KA7MT8qt42tRe3)MR#0I+R7L}0A*K+> z;`cj`JMr8{7rq);D=a$E?p}OHCIKku#LnUm54z@?58tM4hmV=m z-DxbXw&)b+r034HQKun^3?H@6irngMM^2uiLdUE=Ta(0mBv_dZsyIth-MQeZTOVi| zbC}mW-5Yj5s%3if1~bNQbJRnBeIc-{SSd{s=Aj5=Ns|(gkRTx-h=B}d_adR;+GbM4 z>KA(MBRIc#a?$4T;Vm0H_rlT@T?t#W!-a?;2t2Vl$!Cd`@sh-~1C_K^iK*89_h&a1 z@H4Nxw<1?ZMIwRLTHuHnqWjDXC1g49$X~9LYQ_vXR%0f(@#jv~iID(UIT9@d2DC`1 z>AgozXQvPN-c1}jh!D%$@4WMW|NLk7k`KV*r8A+mwEF5sBaBr+&)f<%%BiLqAO?_M zJjYMz4@JWg5z{nP`pEwYpWW!;2QM+6qv8}MC-5AMm)e(V_n4&HW}Y*B+UM3ueLpk= zff+VkTK zEL!+Og0d*4?5qfo2>>5aUKe4fv=)itXh6lS26K#s3;uP%`Pba@@Y6&INLoaoLMVq2 z%1>|*?SWHNI<(fo@|%b%&H159?F*9v0Fj;*vyx_Yrx*i8;B;bl_^p)(i8$YJhewT; z{2Vn+C#re=io52ny?PD-LceK>Ba#ZwWM7$OAIwg9(sx5(4)d*Jj>+abDi8XCeD}On zn)WjQ2-PV$X1!w#{jC_5L=jX#xbK0N_xtkZ1Gjz7UHAdW0+>2sBvqFtuq;LsGgNmW z+mguCTq}T=na;`zQGUtXgukx4COep6)5ocD}St();eBIr>{2|ZF1GNvQ1_!4c-T&ynCrvpW zWE@)pc}hM1%;j5*Z6lbvLV!63<(Jo7fl~H+p+d-+5xbBZMFfldY2QEeyfY6WPjtOV z35U>Yo>6n~#%q3b(Ed9$sBgi#gg-=-yRcQc#WmY=9o7Nk$Dkkj^WjHQ8-M0zLVk3vYS zy29*(ifr9j5i2lJKOq2MXjRIs^vdf?Ke5fxij`C3-21NQF$Gz9Ze9@vpYlJK|KYOP zhK9K?a1?6JzWUbfo3v%hH|niw z`{V-Oi3W*hd;|MGr+@t?|1(()Uck&6dxwT#4+7C3$55;7#m7=ipI+{9^A>ynWd&%T zFtU#9i!xd@^7s&6cxhGd)B465+E|ub=P0z<+Z(oB0AiJ% zB$_}zqWB~LVn9}iNJ5XYh0ccsLbXp>sAaNZgOaDK&CUUwX*$%Y9)Ohd0I*^xc;Xzh z>z>#5r@Ch5!{7MkuPP1@Alx~(YUhs%9H1H0x?g2|1g5b3&%UYcBn)^>(-o{m0xCxp z({dq2E+M*bQ2W+5&HQ=2Td`_xuhewI?zswL(vyXftSzLZuu}l#_bj_DMF627XlgqE zPFc!9jAA7!09Z75#T|Wri}c7zLCIz(1m?FT`_$59ZsZoz0U-vl&k7)G9GocEGaHQs zPOiY|5mOX=RtylaCRiq=Sb;2a2y1Lm8%|sKfW4+&Et;S*5R_k>LJELp%WxYt^V{#< z+u(8$kV;R336+2e#aLi5LK>8VHHwVQ-Zm)Xag9AaJt!bD5a~5>IDYgWul)Ad<|LyPpVe?NN07Vdw6%zu2B}Z0UQKVX?EMA(cb1y*; z=hd+Np*?~j;i9MlB9)dz0283aw0_mPz7J4HbZ<2jI~S;?1BALsKq8_7Q31gcTPq?& z)H1zqExvKWZ>pbwef8geY9y)<84DEt^YrDSlqfAqi&B7h%bkx_pUA)b-8|hqRmeI5 zQGu#0h$IY^BjKCpJ>#|P&s^VULa6tTK$Mul+K93wAOyuf0fj2l+U^JrNHy>8taZqd zfM7*MB`b2uWM()-X}v3P(F2TrR=C!gvmY5bZVItb^zKm6UDzpxwyL%L|6Y2pZf%>I zzm(ZvGLaIpBuwH!fe=U45Mt7r5@K2>Z7^mEbDaeo-7>?`2%-dutsFH`_h3>W@4aqA z&C~i`3&;n{dQ|J)q#a4*$gPUwlPsY?qDZr-K)3F7+K(@-+l19(-9+Am-#+^;3-Y9p zNnxw4#_jRN&n;WN?EWVfMX$0`4y%*eMPN~sQL7?SS)g=talYWX<`IZt1<-8Py7xUp zwN_14AfD~IQPC>-PkEW?PW5&aA#msX3f<}}px#7XDQ$CX`d{}Hg6*8EtD7xiQzO0_ zgkhtrD$j+0Yc(TQ&C@CuTzw@0uk1Bt1DCtIQXH$+>EJ3+$X&As9$lZ!bMpr6t9&{s z1h#8_wF=LwxvGQ_sm`UjmVQd{4GP^JC0#%JRAzLhYsF2KN75tr&KG*^s*>$ii54*{ zSqsjunD198NCI?cgO)Mv#H;$~YW7|MU^0_ft?JrWT2Xh$yviB~In}a1$toqV1K_&m z(7;?z26-F{9E3~zjB?cP9kf$mtqLZ!EKDtY?!Dh#`sY!@6Pu1sYS*R6=JK^KIsxWA z{=vM*Ke&CtTdUX*0)$IrNeBRmxohF#`HwFhy!JJK!rJuD;8he=0KPioQ&pcucNH4; zfSfCeDv@&eBuG+)93g?1nNkQUG9wC9^MO^ON{ooFbx!OG1dg1cUM~{@5j90S$Tl2B z0h5phlvsd8$QLU$t}05Ug)8Nw)QBn~Q)TR1eFO+xSJSXbRmF-DM_T#9fM@|%J3|P= zd-08~{!3n(Ya!7f7OP1(iq@(;A+VriNU(qg6TwrYYgSEiwx@re8p<>kMOAsGS6`p`R07?n#W!9 zQ#vP5B)!tShs9NyWvyyGpi)%%lK9leH*y{GxoiFtk6uY2UHvliGY9nZ99D%v&KN_Y zbA){kzLs>rVd(qHCsL|ogH1l|C#pK-nBmO_?fxmK`j*HttEc%X9|!?>nJ?R(2Z(K; zlC%nb(gQ`jQi_Svfz`Qs=FFen*j%BSk-OmhD}H+R@%>%nTu;ZXDFb1!Mq=0=j?-10 zQ>m9;@7(6&MI*l@Kysi8RidX;_&x>+M2iYS+cN6#!1W3^EiJ;*c!~x99;DE9Gr^DI z(gjX8?_X6-tx(?A0}QS$jv&B*JopLH(l(8}bmc(_+2wB|@{7-e>_%huS=?1QIPu9( z9vNnDQ`Fz`F(KzV?}8`@M2R+&(I+3RftT<4GqDFnfHRrtRboQC-(Gz8$tUjHe=CGJ z?7od0Ypn{Q%d~xoRdc1x5@W8t@h?Xn@}=lwz&UZ1tM9mT?$7qwdutMQq%^Fv)^9nW zO6YM#)kRUuwDNib2|O1TdLq9N0bF*)z4@MOdc#91D|z&Z=U;R28KnwqpQ@rdyE|_* zN=I_=3)_ye9dmW_9zgFhlhyhDAHqr*WZlXU?)}Si$^=BvzUQEGqH?aqn0n(^p^>N} zaQlQ&?Gr{#-g&d$qf+{7!256AP&|<`07RP^I}D$`!-mm7D()}8Hc%ne3~SfX;M6+Q zyp0u`E3TN`-}#i_?42nOAdrRz?9%&FuDiw*`{^(A$~)hm3|(cDX1*Oz3Z$Vx@O{d! zm{}>DSQXkf+O6O9BtO?Vm*AC56(G|26?3Ya7F=@aZPEV%Rohg+Q8?fqg+i_pRmvGI z%vIzQ$|36J-FNQs!p`#jNgx_4vi-a%1$i&*z8g@HwUj{C-XW-#ZsHfZW)b{Sz?m*& zXH|aBDme4Z-}T=Bu~xG43_%+OLc-TJ`|zmdLbcYZjynGL#A;3ZC535WUmAugsC%jD-FiT*s2ogX0U_k^53;rK3u-sfdg07H%zf7 zsA!@?ia7faxc746B!dU?a}PwfvYYL2PUvf&th3H{fv2e}*PUq_)tnjK+SdD{GR&{e z+Bf9@17<8+W?y>kg}%iJaN+e&+0Z79=iKt|9r+bhkCi*0d^;2cVCsZXbtk|Zv=*Dg zBE=;~B7i{dq8tB_cZvkxKYT5uKNfEwBd$hcXKlmAMhqckJ8!K$s#RCVy{7G;N<*nC z7ZL!8UcW9Ov+E9HQL7|q0LT@P>=kO+!pMHo+9-NomoF^gscO5J`2Trnar7?%(q}PV z!@7YPGdglyGYijV>&kka?YhSsabV@c-eB}}*WJjwjYKc2X}kgmg{q9(#rW0=96=J5 zqM<7vy2i0WBr4Z+M<2aGz4!J5)H0PSCKpP=a$=>=Za3a8pDU_3_MPif0O51n>bAngKMr^ zIT)vX#c(bGqyy8Mr52ssBv-@pfw3S&l^-M0T!of26@v^IVSvSviz z?>8Lbi2xw#FOJ6mL|Ou%;%TaHe(m$1(2;&<;eQy#jDUj{1&$saKDF8qTgy&6&f2 zsP_UMxxv(`Bjgf&G`I-uB|WK`_w^?yzjTpeM4ecsnjIpanlIsJQo`1OU~sVw`G zn+{vinS1r!!uAen4H5_f^lP?QRR|(g5=*Imy+W(0c{n8cqKhtQbaGe-S=D*TC}7Fb zJDS^iSGA-x6xmjlFMw{_SGiQN>cwLa5UU#Hr{8{7H!*7iNfA|q`fTP%X81tzyE4cs zO_jaH>3)j)_||RzetvQJqfmdIE?@ZUwtWv0tW-SyEw&h2=gQUbn`h@Bbv$wj$mJeM zwp182MXJSX^nVbr-`u)qNX;Sy$}9F*S%U8G=My~p*k#-9bVBv7B%qu z?$-}}_4P&XEm`r-`?(#L4=_)o=NvW&Rt1V?n!lEZ??;*zn znHp#T#8uU!@sCO1^Upo5egE^yM`X1FFd|#epvHCC&PKCsPpT|UeNQ^(+rPQ?+Q%&C zfg~g(0HTB()k~yXQ`9z}+)$*@NEa4zB!KTP?SemS22;SDw^yHG3LSU(BigC%LKs zFb7%XuOtX2;^}`Zscf+z(a*1v%ILc3nF&(s6WeGpR1C-!7vlxdlmJxyfAOoqO*O2I z2hkx_ENl~LtU{LS=uJ>Gwh)*CI4GR}P-|JM8R1o>sD!Du*7d7f4IceAF!D60_dV4b z0jng7PJB_S<5D4}v_m?BlCQoxbBl3}od%5J?A<%;di+yQzeA*j%JaTj7$Jnb^wdJv zoL(i1;CW=1&hwiliEo66>dl9W=Sq+mHucCuCSHB%=?&|%3aq)bI^#F<1C#e%of1y| z-chSqNX_HvJ5vDQ2(K^7joBm#XwRbmfMw^ma@DS{mOTQl zqj#4Gf=2G0MjE_PgW6uCw0<{ZRzZ?dp0E-ZlhuU+52jZ}u*+K)7`+lFh8=c3>hJ$r zL7-AvSQjiWzUr0diLHDl>AEFETzS>?Ri9U?gA5M+t-+{7qYllGhW3O-i+jklNv+LO zjUDH|`0UM*e);a=l`l*`@yNv_{Kn$0aa(?)f7$Ico7A2hJWzR1dvJw$V9}YqWo13; znrk1u__qT}FTCa*aprB0dz2Oem8Zv|#Q}nYCVgfV3!&;CGE`2Osd}&~;JQ1Wj#PXm zclHTYo5hO+jt&99qDa~zT|JG}%-*^wklK878+XnEk%&9;*bA?_>95+5tBuql_`~Yyi3SQT_aFcu zWO!cYD;mD-<0Vf)zO}egQ{Eo-?KNH3RhTxf|gL=HE6N} zSRkk#cvdO$01(0l@BG#7`<_wO3&MPlgVZ%}I*>|wT(OcRb(dGyE##*iT;1rfBM$#% z<=e9=X??{TNI*7scQJ~#)Np~_|G@7zyen6FxIm(HC<2kb_R1ezd*u%#3_J6tZOpXl zCF`;abk%bcgd`ed08arfdp`gtGHT;}mSt5{Y>@F+A*Vz`2HBT=rHJ9Pes!l*t5o}a zaqCl#-HCv@Gkk}@>7p%9OJGr*!p`_k#DEQ}dAAI^VqpAla=b4bG(iL=AVf7~h zN|pt(s(A``&Oz;)!IJkCN@XgpQrt0ba1%o*z$3C;?ViT=Z z{_z5xT~>~~^LE?i69ik?pZvsrE%HRkY#o_MJyviGP1cOH~R4FqfL<<~!X;-NJ) zVGAXG)o>uBi=V57tP)rW{Th3@HP1iW(J1=s|6Mry)U&THPf-OV9hK=VL7vy@!FA5o zExoJEsg>Hz=}0tL195#I6h zW0gjdlb_jY)ACUQw*2B5tJMSrb~qKKkh=m^#D}w$3d{EWJOdkX$sKXrg@84ToG$N#FpRqt;H@)G;inAQKa;17)yP~B%~FGrw6!_j@x zcc)ic(+G0UgU|Ij%Sufq+)|{CB26j}?j216s=p9flL-T?^s2Ox=*ps5~_1~DW9bC2h4=F%_$TEI<=8VXu4&aoZ&i}(tDxC=xLEZPKq65Lk6Mt5= z!G&Qe%6hVjtpdpN5Y?Z7@#%mfj zS-|5rOO4rRqXkdC$->n&B`V19w%)Sl-EzLlOs*7|e|_X-uetq_KOBF=k&`?iF>Fd8 zmZaj{sFV4wwUSWS0Na0N#NVEOuX0-;5{SQvZaxUT+a0;0=Th(Flu~|c=aOqGOCun^ z{3e~6L8jz{&FY;GpjxMWe(Q~|x$b|jy7Hr+(Z9S~GRd8)~jYP6_TE62J4Ff@eUE7F#^R7Q*c0yZ} zmVqPoopw&z>*`!`Cn+K*k&u+9TiPr8Rqs27aPu`k`s$HYUV+p+!*VS&Cj!7$BOgadp{0yN^a6-XJLbHe_vcgQxZ{QzO z+v|v+nXQrW@ys5FU#7@h@vE`51w&Z+7QS6_0{+8WA)Cl_2r-X2J%tE(>6%Tm>? z5)WkGeeY`B$F!;7Px$=kC;qv(;*f&4zdyI=)1N&gI-}+aacbG(zYMq~gF(^lfW=_Z zyKd3Dua;sPt62Q@wN&ZJ?5udED8^OZF{(4NCWRCby?%h`7Y;4(SAl_C;Z}_g{Q6VTsiHdvtw3M z(WiVBjkv5_nCrNERW@x-QX-FRlo+qx_X==(?ynFIVL9UGq|{(@9pv4}<}gTQO&V4}7glE5k`guJn|Y5-2bL z);$kgcO-!zCM@n(SaqQgMx?zJT~W^~zC?s0m$~X8EtquTOr?d`pMUY!qGOr?6o?4| zMFxN+1G15UGKh>>)evFV&kk>Hl3jP%Y1_|!{L`Datib}e2GRyW*R7eB_Hd>9l9APv z%5}^t?S#1Uep8?vcg{|=9a^Z{zs0=#@9VeNw2q&S20k`^MA$VinQE`+Bv-1DnD^hA zHKM+o#2`Y7JSjYu7qqq=9CCO6XDh4oxm!&?$^Qr(K6J&k_n+|1i@4vFz7{{dV(Cpl zxX!kf&_?$YkQ9d9J=uGbnW=sEsRjTsx6l9Sq}^-dY7;sUbj?pR?JsqGfD2lJDD=G| z+*x#Lc>D7A<|KUuz)R@d^4X)enEd{dmDE3(NN+i|_1UK`s_vgAp%8S;88-63o^U0u z5=xS``__SHKw0a#L&r=wyKc6BPw2IUKk4T}0T#X6@tt$4OeeE0J`8yk<_$`i zm7DX&BcpE_@R}zHwCkD`?Mbokk-yAyTluQHuRS_JvNqQlSOjCXnDqV!2_cXO6|)p7 zMEeU+KD+Fm)XKJOt;roKilt*dGrB`=8`%y{h_a#_B=Mc&_xQ~P$I;3nA4LR2I)MI~Gf2%TTePx)rZF>Hrq;`V|Z!(p|8_+V*9yg`}WLc3D;q zoPsR1-G6DcBSXF0+ZSG3zV&v;07}RKLIP0?(HXzc`TpIjzAtaU@N3?n#h};9%GXiQ z`d)|%x0~ABp+eF%CkO*liDcVkh!P@@6-n;7$7XlWxuBm6Tu$tS-Hw0indP87u^}Ol zvz@nFm-S2rs5P%Jz=G;u5n;gB-0<*m-~KH~ltinm3LuCA=oddb>Xeh)RXsyZxdpBe zG;MrvAy1%kf(jNTkd%*(&d&P>YYXzVF0D43_*1T@q8P^OkDs}ciy7_Ts>;!`+l0-& zfA}sn-bNPgfD^BcV!mu>@3{QPs0TS9(KPw`E6+XXpzQ!S;XRMIibXTIo}NOO>-gN} z8{W5I!P`$?ITVboji@WFhR9-W%ol|}EpyX^hW{Jqb83ua? zJJ4PBK02(Mo#j`L5EwV{+`h*>>U!<5UpBai1!~iZtH`Ai>DHl*PZK%nSPqrW{}Vun7t_-mr{gT}w+sxtJXl~e_#GCbuxVngip zyZaNg@pZKwUP5fO%MTqm-}%6MOY25q#l*y{Hw-ZUrk5 zI9WDn#RMI49?B?oc_fv9$cUqX>{}NT6Z4@n1npJ~5Jc7>awunnv6BDz3Fa$@h0%*;GJT;&Q1VvDQ1`L%l>jype-cqxJNK8!3dc_dbt=1Na z;ZqCCRWeDn4YYXQ-Dsnj{=D(7UkZuJG_P)b0F){0qkC1K!uhCT0!!Xw3xQ`(VZB4TgQ#wTOv{Sra~5(h3}HM-)4W#>A|De57WA z01<>%d7HnOn3y5NM@?GcmRoO)b*q?|^^uR*OfV)UX1(Pjrd!3t#H_c((kdn