[Bench] Support APPS (#963)

* [Feat] support apps

* [Feat] support apps

* [Feat] support apps

* update README

configs/datasets/apps/README.md

@@ -0,0 +1,41 @@
+# APPS
+## Dataset Description
+APPS is a benchmark for code generation with 10000 problems. It can be used to evaluate the ability of language models to generate code from natural language specifications.
+
+## Dataset Structure
+```python
+DatasetDict({
+    train: Dataset({
+        features: ['problem_id', 'question', 'solutions', 'input_output', 'difficulty', 'url', 'starter_code'],
+        num_rows: 5000
+    })
+    test: Dataset({
+        features: ['problem_id', 'question', 'solutions', 'input_output', 'difficulty', 'url', 'starter_code'],
+        num_rows: 5000
+    })
+})
+```
+## How to Use
+You can also filter the dataset based on difficulty level: introductory, interview and competition. Just pass a list of difficulty levels to the filter. For example, if you want the most challenging questions, you need to select the competition level:
+```python
+ds = load_dataset("codeparrot/apps", split="train", difficulties=["competition"])
+print(next(iter(ds))["question"])
+```
+## Evaluation results
+
+
+| Dataset | Metric | Baichuan2-7B | Baichuan2-13B | InternLM2-7B | InternLM2-20B |
+|---------|--------|---------------|----------------|---------------|----------------|
+| APPS(testset) | pass@1 | 0.0 | 0.06 | 0.0 | 0.0 |
+
+Please refer to Table 3 of [code llama](https://scontent-nrt1-2.xx.fbcdn.net/v/t39.2365-6/369856151_1754812304950972_1159666448927483931_n.pdf?_nc_cat=107&ccb=1-7&_nc_sid=3c67a6&_nc_ohc=TxT1PKkNBZoAX8zMHbm&_nc_ht=scontent-nrt1-2.xx&oh=00_AfDmmQAPzqX1-QOKIDUV5lGKzaZqt0CZUVtxFjHtnh6ycQ&oe=65F5AF8F) for original results if needed. 
+
+## Citation
+```
+@article{hendrycksapps2021,
+  title={Measuring Coding Challenge Competence With APPS},
+  author={Dan Hendrycks and Steven Basart and Saurav Kadavath and Mantas Mazeika and Akul Arora and Ethan Guo and Collin Burns and Samir Puranik and Horace He and Dawn Song and Jacob Steinhardt},
+  journal={NeurIPS},
+  year={2021}
+}
+```

configs/datasets/apps/apps_gen.py

@@ -1,4 +1,4 @@
 from mmengine.config import read_base
 
 with read_base():
-    from .apps_gen_7fbb95 import apps_datasets  # noqa: F401, F403
+    from .apps_gen_d82929 import APPS_datasets  # noqa: F401, F403

configs/datasets/apps/apps_gen_d82929.py

@@ -0,0 +1,28 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import APPSDataset, APPSEvaluator
+
+APPS_reader_cfg = dict(input_columns=["question", "starter"], output_column="problem_id", train_split='test')
+
+APPS_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template="\nQUESTION:\n{question} {starter}\nANSWER:\n"),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512),
+)
+
+APPS_eval_cfg = dict(evaluator=dict(type=APPSEvaluator), pred_role="BOT")
+
+APPS_datasets = [
+    dict(
+        type=APPSDataset,
+        abbr="apps",
+        path="codeparrot/apps",
+        num_repeats=1,
+        reader_cfg=APPS_reader_cfg,
+        infer_cfg=APPS_infer_cfg,
+        eval_cfg=APPS_eval_cfg,
+    )
+]

opencompass/datasets/apps.py

@@ -0,0 +1,810 @@
+import faulthandler
+import itertools
+import json
+import multiprocessing
+import os
+import platform
+import re
+# to run the solution files we're using a timing based approach
+import signal
+import sys
+from collections import defaultdict
+# used for debugging to time steps
+from datetime import datetime
+from enum import Enum
+# for capturing the stdout
+from io import StringIO
+# used for testing the code that reads from input
+from unittest.mock import mock_open, patch
+
+import numpy as np
+from datasets import Dataset, DatasetDict, load_dataset
+from pyext import RuntimeModule
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from .base import BaseDataset
+
+TIMEOUT = 10
+
+
+@LOAD_DATASET.register_module()
+class APPSDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str, num_repeats: int = 1):
+        dataset = load_dataset(path)
+        new_dataset = DatasetDict()
+        # add new column "starter" in the prompt
+        for split in dataset.keys():
+            new_samples = []
+            for _, sample in enumerate(dataset[split]):
+                starter_code = None if len(
+                    sample['starter_code']) == 0 else sample['starter_code']
+                try:
+                    input_output = json.loads(sample['input_output'])
+                    fn_name = (None if not input_output.get('fn_name') else
+                               input_output['fn_name'])
+                except ValueError:
+                    fn_name = None
+                starter = ''
+                if starter_code:
+                    starter += starter_code
+                if (not fn_name) and (not starter_code):
+                    call_format = '\\nUse Standard Input format'
+                    starter += call_format
+                else:
+                    call_format = '\\nUse Call-Based format'
+                    starter += call_format
+                # Add the new column "starter" to the sample
+                sample['starter'] = starter
+                new_samples.append(sample)
+            new_data = {
+                key: [sample[key] for sample in new_samples]
+                for key in new_samples[0].keys()
+            }
+            new_dataset[split] = Dataset.from_dict(new_data)
+
+        # num_repeats duplicate
+        train_repeated = []
+        test_repeated = []
+        for sample in new_dataset['train']:
+            train_repeated.extend([sample] * num_repeats)
+        for sample in new_dataset['test']:
+            test_repeated.extend([sample] * num_repeats)
+
+        dataset_train_repeated = new_dataset['train'].from_list(train_repeated)
+        dataset_test_repeated = new_dataset['test'].from_list(test_repeated)
+
+        return DatasetDict({
+            'train': dataset_train_repeated,
+            'test': dataset_test_repeated
+        })
+
+
+EOF_STRINGS = ['\nQUESTION', '\n---', '\nANSWER', '<|endoftext|>']
+
+
+@ICL_EVALUATORS.register_module()
+class APPSEvaluator(BaseEvaluator):
+
+    def truncate_after_eof_strings(self, text):
+        pattern = '|'.join(re.escape(s) for s in EOF_STRINGS)
+        match = re.search(pattern, text)
+
+        if match:
+            return text[:match.start()]
+        else:
+            return text
+
+    TIMEOUT = 10
+
+    def check_correctness(self, sample, generation, timeout, debug=True):
+        """Check correctness of code generation with a global timeout.
+
+        The global timeout is to catch some extreme/rare cases not handled by
+        the timeouts inside `run_test`
+        """
+
+        def _temp_run(sample, generation, debug, result):
+            result.append(run_test(sample, test=generation, debug=debug))
+
+        manager = multiprocessing.Manager()
+        result = manager.list()
+        p = multiprocessing.Process(target=_temp_run,
+                                    args=(sample, generation, debug, result))
+        p.start()
+        p.join(timeout=timeout + 1)
+        if p.is_alive():
+            p.kill()
+        if not result:
+            in_outs = json.loads(sample['input_output'])
+            # consider that all tests failed
+            result = [[-1 for i in range(len(in_outs['inputs']))]]
+            if debug:
+                print('global timeout')
+        return result[0]
+
+    def evaluate_generations(self,
+                             generations,
+                             samples,
+                             idx=None,
+                             debug=False):
+        assert len(generations.keys()) == len(samples)
+        results = {}
+        idx = 0
+        for task_id, problem_generations in generations.items():
+            sample = samples.iloc[idx]
+            res = []
+            # loop over the generations
+            for o_idx, o in enumerate(problem_generations):
+                curr_res = [-2]
+                try:
+                    curr_res = self.check_correctness(sample,
+                                                      o,
+                                                      timeout=TIMEOUT,
+                                                      debug=debug)
+                    if debug:
+                        print(f'\nSuccessful compilation of task {o_idx}!')
+                    fixed = []
+                    for e in curr_res:
+                        if isinstance(e, np.ndarray):
+                            e = e.item(0)
+                        if isinstance(e, np.bool_):
+                            e = bool(e)
+                        fixed.append(e)
+                    curr_res = fixed
+                    if not np.all(curr_res):
+                        if debug:
+                            print('Results were not True for all test cases')
+                except Exception as e:
+                    if debug:
+                        print(
+                            f'Compilation failed, test framework exception = {repr(e)}{e}\n'  # noqa: E501
+                        )
+                    break
+                finally:
+                    assert isinstance(curr_res, list)
+                    res.append(curr_res)
+            results[task_id] = res
+            idx += 1
+        return results
+
+    def estimate_pass_at_k(self, num_samples, num_correct, k):
+        """Estimates pass@k of each problem and returns them in an array."""
+
+        def estimator(n: int, c: int, k: int) -> float:
+            """Calculates 1 - comb(n - c, k) / comb(n, k)."""
+            if n - c < k:
+                return 1.0
+            return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+        if isinstance(num_samples, int):
+            num_samples_it = itertools.repeat(num_samples, len(num_correct))
+        else:
+            assert len(num_samples) == len(num_correct)
+            num_samples_it = iter(num_samples)
+
+        return np.array([
+            estimator(int(n), int(c), k)
+            for n, c in zip(num_samples_it, num_correct)
+        ])
+
+    def compute_metrics(self, results, k_list=[1, 10, 100]):
+        total = []
+        correct = []
+        task_ids = []
+        for task_id, res in results.items():
+            all_correct = []
+            for generation in res:
+                gen = np.array(generation)
+                all_correct.append(np.all(gen > 0))
+            task_ids.append(task_id)
+            total.append(len(all_correct))
+            correct.append(sum(all_correct))
+        total = np.array(total)
+        correct = np.array(correct)
+        ks = k_list
+        detail_pass_at_k = {
+            f'pass@{k}': self.estimate_pass_at_k(total, correct, k).tolist()
+            for k in ks if (total >= k).all()
+        }
+        pass_at_k = {
+            f'pass@{k}':
+            100 * (self.estimate_pass_at_k(total, correct, k).mean())
+            for k in ks if (total >= k).all()
+        }
+        detail_metrics = {
+            k: dict(zip(task_ids, v))
+            for k, v in detail_pass_at_k.items()
+        }
+        pass_at_k['detail'] = detail_metrics
+        return pass_at_k
+
+    def score(self, predictions, references, test_set):
+        assert len(predictions) == len(references)
+        generations = defaultdict(list)
+        for refer, pred in zip(references, predictions):
+            pred = self.truncate_after_eof_strings(pred)
+            generations[refer].append(pred)
+        # convert to non-duplicated version
+        test_set = test_set.to_pandas()
+        test_set_origin = test_set.drop_duplicates(subset='problem_id')
+        results = self.evaluate_generations(generations, test_set_origin)
+        metrics = self.compute_metrics(results)
+        return metrics
+
+
+class CODE_TYPE(Enum):
+    call_based = 0
+    standard_input = 1
+
+
+# stuff for setting up signal timer
+class TimeoutException(Exception):
+    pass
+
+
+def timeout_handler(signum, frame):
+    print('alarm went off')
+
+    raise TimeoutException
+
+
+signal.signal(signal.SIGALRM, timeout_handler)
+timeout = 4  # seconds
+
+
+# used to capture stdout as a list
+# from https://stackoverflow.com/a/16571630/6416660
+# alternative use redirect_stdout() from contextlib
+class Capturing(list):
+
+    def __enter__(self):
+        self._stdout = sys.stdout
+        sys.stdout = self._stringio = StringIO()
+        # Make closing the StringIO a no-op
+        self._stringio.close = lambda x: 1
+        return self
+
+    def __exit__(self, *args):
+        self.extend(self._stringio.getvalue().splitlines())
+        del self._stringio  # free up some memory
+        sys.stdout = self._stdout
+
+
+def run_test(sample, test=None, debug=False):
+    """if test(generated_code) is not None it'll try to run the code.
+
+    otherwise it'll just return an input and output pair.
+    """
+    # Disable functionalities that can make destructive changes to the test.
+    reliability_guard()
+
+    if debug:
+        print(f'start = {datetime.now().time()}')
+
+    try:
+        in_outs = json.loads(sample['input_output'])
+    except ValueError:
+        in_outs = None
+    if in_outs:
+        if in_outs.get('fn_name') is None:
+            which_type = CODE_TYPE.standard_input  # Standard input
+            method_name = None
+        else:
+            which_type = CODE_TYPE.call_based  # Call-based
+            method_name = in_outs['fn_name']
+
+    if debug:
+        print(f'loaded input_output = {datetime.now().time()}')
+
+    if test is None:
+        return in_outs
+    elif test is not None:
+        results = []
+        sol = 'import sys\nimport time\nimport itertools\nfrom itertools import accumulate, product, permutations, combinations\nimport collections\nfrom collections import Counter, OrderedDict, deque, defaultdict, ChainMap\nfrom functools import lru_cache\nimport math\nfrom math import sqrt, sin, cos, tan, ceil, fabs, floor, gcd, exp, log, log2\nimport fractions\nfrom typing import List, Tuple\nimport numpy as np\nimport random\nimport heapq\nfrom heapq import *\n'  # noqa: E501
+        if debug:
+            print(f'loading test code = {datetime.now().time()}')
+
+        if which_type == CODE_TYPE.call_based:
+            sol += test
+            if debug:
+                print(f'sol = {sol}')
+            signal.alarm(timeout)
+            try:
+                tmp_sol = RuntimeModule.from_string('tmp_sol', '', sol)
+                if 'class Solution' not in test:
+                    tmp = tmp_sol
+                else:
+                    tmp = tmp_sol.Solution()
+                signal.alarm(0)
+            except Exception as e:
+                signal.alarm(0)
+                if debug:
+                    print(f'type 0 compilation error = {e}')
+                results.append(-2)
+                return results
+            signal.alarm(0)
+
+        elif which_type == CODE_TYPE.standard_input:
+            # sol
+            tmp_test = test.split('\n')
+
+            new_test = []
+            for x in tmp_test:
+                if (not x.startswith('from ')) and (
+                        not x.startswith('import ')):
+                    new_test.append('\t' + x + '\n')
+                else:
+                    new_test.append(x + '\n')
+            tmp_test = new_test
+
+            new_test = ''
+            started = False
+            for i in tmp_test:
+                if i.startswith('\t') and not started:
+                    new_test += 'stdin = sys.stdin\nstdout = sys.stdout\n'
+                    new_test += 'def code():\n'
+                    new_test += i
+                    started = True
+                elif started and ((i.startswith('from ')) or
+                                  (i.startswith('import '))):
+                    new_test += '\t' + i
+                else:
+                    new_test += i
+            tmp_test = new_test
+
+            sol += tmp_test
+            if debug:
+                print(f'sol = {sol}')
+            method_name = 'code'
+            signal.alarm(timeout)
+            try:
+                tmp_sol = RuntimeModule.from_string('tmp_sol', '', sol)
+                tmp = tmp_sol
+                signal.alarm(0)
+            except Exception as e:
+                signal.alarm(0)
+                if debug:
+                    print(f'type 1 compilation error = {e}')
+                results.append(-2)
+                return results
+            signal.alarm(0)
+        if debug:
+            print(f'get method = {datetime.now().time()}')
+
+        try:
+            method = getattr(tmp,
+                             method_name)  # get_attr second arg must be str
+        except:  # noqa: E722
+            signal.alarm(0)
+            e = sys.exc_info()
+            print(f'unable to get function error = {e}')
+            results.append(-2)
+            return results
+
+        for index, inputs in enumerate(in_outs['inputs']):
+            # JSON forces dictionaries to have string keys; this undoes this (assuming a singleton list) # noqa: E501
+            try:
+                if isinstance(inputs[0], dict):
+                    inputs = [{int(k): v for k, v in inputs[0].items()}]
+            except:  # noqa: E722
+                True
+            try:
+                if isinstance(in_outs['outputs'][index], dict):
+                    in_outs['outputs'][index] = [{
+                        int(k): v
+                        for k, v in in_outs['outputs'][index].items()
+                    }]
+            except:  # noqa: E722
+                True
+            try:
+                if isinstance(in_outs['outputs'][index][0], dict):
+                    in_outs['outputs'][index] = [{
+                        int(k): v
+                        for k, v in in_outs['outputs'][index][0].items()
+                    }]
+            except:  # noqa: E722
+                True
+
+            if debug:
+                print(
+                    f'time: {datetime.now().time()} testing index = {index}  inputs = {inputs}, {type(inputs)}. type = {which_type}'  # noqa: E501
+                )
+            if which_type == CODE_TYPE.call_based:  # Call-based
+                signal.alarm(timeout)
+                faulthandler.enable()
+                try:
+                    output = method(*inputs)
+
+                    # ground truth sequences are not tuples
+                    if isinstance(output, tuple):
+                        output = list(output)
+
+                    tmp_result = output == in_outs['outputs'][index]
+                    if isinstance(in_outs['outputs'][index],
+                                  list) and in_outs['outputs'][index]:
+                        tmp_result = tmp_result or (
+                            output == in_outs['outputs'][index][0])
+
+                    # ground truth sequences are not tuples
+                    try:
+                        if isinstance(output[0], tuple):
+                            tmp_result = tmp_result or ([
+                                list(x) for x in output
+                            ] == in_outs['outputs'][index][0])
+                    except:  # noqa: E722
+                        True
+                    results.append(tmp_result)
+
+                    # reset the alarm
+                    signal.alarm(0)
+                except Exception as e:
+                    signal.alarm(0)
+                    faulthandler.disable()
+                    if debug:
+                        print(
+                            f'Standard input runtime error or time limit exceeded error = {e}'  # noqa: E501
+                        )
+                    results.append(-1)
+                    continue
+                faulthandler.disable()
+                signal.alarm(0)
+                if debug:
+                    print(
+                        f"outputs = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}"  # noqa: E501
+                    )
+            elif which_type == CODE_TYPE.standard_input:  # Standard input
+                faulthandler.enable()
+                signal.alarm(timeout)
+                passed = False
+
+                if isinstance(inputs, list):
+                    inputs = '\n'.join(inputs)
+                if isinstance(in_outs['outputs'][index], list):
+                    in_outs['outputs'][index] = '\n'.join(
+                        in_outs['outputs'][index])
+
+                with Capturing() as output:
+                    try:
+                        call_method(method, inputs)
+                        # reset the alarm
+                        signal.alarm(0)
+                        passed = True
+                    except Exception as e:
+                        # runtime error or took too long
+                        signal.alarm(0)
+                        print(
+                            f'Call-based runtime error or time limit exceeded error = {repr(e)}{e}'  # noqa: E501
+                        )
+                        results.append(-1)
+                    signal.alarm(0)
+
+                if not passed:
+                    if debug:
+                        nl = '\n'
+                        if not isinstance(inputs, list):
+                            print(
+                                f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}"  # noqa: E501
+                            )
+                        else:
+                            print(
+                                f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}"  # noqa: E501
+                            )
+                    continue
+
+                if passed and debug:
+                    print(
+                        f"==> output = {output}, test outputs = {in_outs['outputs'][index]}"  # noqa: E501
+                    )
+
+                if custom_compare_(output, in_outs['outputs'][index]):
+                    tmp_result = True
+                    results.append(tmp_result)
+                    continue
+
+                # ground truth sequences are expressed as lists not tuples
+                if isinstance(output, tuple):
+                    output = list(output)
+
+                tmp_result = False
+                try:
+                    tmp_result = (output == [in_outs['outputs'][index]])
+                    if isinstance(in_outs['outputs'][index], list):
+                        tmp_result = tmp_result or (
+                            output == in_outs['outputs'][index])
+                        if isinstance(output[0], str):
+                            tmp_result = tmp_result or ([
+                                e.strip() for e in output
+                            ] == in_outs['outputs'][index])
+                except Exception as e:
+                    if debug:
+                        print(f'Failed check1 exception = {e}')
+                    pass
+
+                if tmp_result == True:  # noqa: E712
+                    results.append(tmp_result)
+                    continue
+
+                # try one more time without \n
+                if isinstance(in_outs['outputs'][index], list):
+                    for tmp_index, i in enumerate(in_outs['outputs'][index]):
+                        in_outs['outputs'][index][tmp_index] = i.split('\n')
+                        in_outs['outputs'][index][tmp_index] = [
+                            x.strip()
+                            for x in in_outs['outputs'][index][tmp_index] if x
+                        ]
+                else:
+                    in_outs['outputs'][index] = in_outs['outputs'][
+                        index].split('\n')
+                    in_outs['outputs'][index] = list(
+                        filter(len, in_outs['outputs'][index]))
+                    in_outs['outputs'][index] = list(
+                        map(lambda x: x.strip(), in_outs['outputs'][index]))
+
+                try:
+                    tmp_result = (output == [in_outs['outputs'][index]])
+                    if isinstance(in_outs['outputs'][index], list):
+                        tmp_result = tmp_result or (
+                            output == in_outs['outputs'][index])
+                except Exception as e:
+                    if debug:
+                        print(f'Failed check2 exception = {e}')
+                    pass
+
+                if tmp_result == True:  # noqa: E712
+                    results.append(tmp_result)
+                    continue
+
+                # try by converting the output into a split up list too
+                if isinstance(output, list):
+                    output = list(filter(len, output))
+
+                if debug:
+                    nl = '\n'
+                    if not isinstance(inputs, list):
+                        print(
+                            f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}"  # noqa: E501
+                        )
+                    else:
+                        print(
+                            f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}"  # noqa: E501
+                        )
+
+                if tmp_result == True:  # noqa: E712
+                    results.append(tmp_result)
+                    continue
+
+                try:
+                    tmp_result = (output == [in_outs['outputs'][index]])
+                    if isinstance(in_outs['outputs'][index], list):
+                        tmp_result = tmp_result or (
+                            output == in_outs['outputs'][index])
+                except Exception as e:
+                    if debug:
+                        print(f'Failed check3 exception = {e}')
+                    pass
+
+                try:
+                    output_float = [float(e) for e in output]
+                    gt_float = [float(e) for e in in_outs['outputs'][index]]
+                    tmp_result = tmp_result or (
+                        (len(output_float) == len(gt_float))
+                        and np.allclose(output_float, gt_float))
+                except Exception:
+                    pass
+                try:
+                    if isinstance(output[0], list):
+                        output_float = [float(e) for e in output[0]]
+                        gt_float = [
+                            float(e) for e in in_outs['outputs'][index][0]
+                        ]
+                        tmp_result = tmp_result or (
+                            (len(output_float) == len(gt_float))
+                            and np.allclose(output_float, gt_float))
+                except Exception:
+                    pass
+
+                if tmp_result == True:  # noqa: E712
+                    results.append(tmp_result)
+                    continue
+
+                # try by converting the stuff into split up list
+                if isinstance(in_outs['outputs'][index], list):
+                    for tmp_index, i in enumerate(in_outs['outputs'][index]):
+                        in_outs['outputs'][index][tmp_index] = set(i.split())
+                else:
+                    in_outs['outputs'][index] = set(
+                        in_outs['outputs'][index].split())
+
+                try:
+                    tmp_result = (output == in_outs['outputs'][index])
+                except Exception as e:
+                    if debug:
+                        print(f'Failed check4 exception = {e}')
+                    continue
+
+                if tmp_result == True:  # noqa: E712
+                    results.append(tmp_result)
+                    continue
+
+                # try by converting the output into a split up list too
+                if isinstance(output, list):
+                    for tmp_index, i in enumerate(output):
+                        output[tmp_index] = i.split()
+                    output = list(filter(len, output))
+                    for tmp_index, i in enumerate(output):
+                        output[tmp_index] = set(i)
+                else:
+                    output = output.split()
+                    output = list(filter(len, output))
+                    output = set(output)
+
+                try:
+                    tmp_result = (set(frozenset(s) for s in output) == set(
+                        frozenset(s) for s in in_outs['outputs'][index]))
+                except Exception as e:
+                    if debug:
+                        print(f'Failed check5 exception = {e}')
+
+                # if they are all numbers, round so that similar numbers are treated as identical # noqa: E501
+                try:
+                    tmp_result = tmp_result or (
+                        set(
+                            frozenset(round(float(t), 3) for t in s)
+                            for s in output) ==  # noqa: E502, E128
+                        set(
+                            frozenset(round(float(t), 3) for t in s)
+                            for s in in_outs['outputs'][index]))  # noqa: E501
+                except Exception as e:
+                    if debug:
+                        print(f'Failed check6 exception = {e}')
+
+                if tmp_result == True and debug:  # noqa: E712
+                    print('PASSED')
+
+                results.append(tmp_result)
+
+                if debug:
+                    nl = '\n'
+                    if not isinstance(inputs, list):
+                        print(
+                            f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}"  # noqa: E501
+                        )
+                    else:
+                        print(
+                            f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}"  # noqa: E501
+                        )
+
+    return results
+
+
+def custom_compare_(output, ground_truth):
+
+    if isinstance(output, list):
+        output_1 = '\n'.join(output)
+        if stripped_string_compare(output_1, ground_truth):
+            return True
+
+    if isinstance(output, list):
+        output_2 = [o.lstrip().rstrip() for o in output]
+        output_2 = '\n'.join(output_2)
+        if stripped_string_compare(output_2, ground_truth):
+            return True
+
+    return False
+
+
+def stripped_string_compare(s1, s2):
+    s1 = s1.lstrip().rstrip()
+    s2 = s2.lstrip().rstrip()
+    return s1 == s2
+
+
+def call_method(method, inputs):
+
+    if isinstance(inputs, list):
+        inputs = '\n'.join(inputs)
+
+    inputs_line_iterator = iter(inputs.split('\n'))
+
+    # sys.setrecursionlimit(10000)
+
+    # @patch('builtins.input', side_effect=inputs.split("\n"))
+    @patch('builtins.open', mock_open(read_data=inputs))
+    @patch('sys.stdin', StringIO(inputs))
+    @patch('sys.stdin.readline', lambda *args: next(inputs_line_iterator))
+    @patch('sys.stdin.readlines', lambda *args: inputs.split('\n'))
+    @patch('sys.stdin.read', lambda *args: inputs)
+    # @patch('sys.stdout.write', print)
+    def _inner_call_method(_method):
+        try:
+            return _method()
+        except SystemExit:
+            pass
+        finally:
+            pass
+
+    return _inner_call_method(method)
+
+
+def reliability_guard(maximum_memory_bytes=None):
+    """This disables various destructive functions and prevents the generated
+    code from interfering with the test (e.g. fork bomb, killing other
+    processes, removing filesystem files, etc.) WARNING This function is NOT a
+    security sandbox.
+
+    Untrusted code, including, model- generated code, should not be blindly
+    executed outside of one. See the Codex paper for more information about
+    OpenAI's code sandbox, and proceed with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+
+        resource.setrlimit(resource.RLIMIT_AS,
+                           (maximum_memory_bytes, maximum_memory_bytes))
+        resource.setrlimit(resource.RLIMIT_DATA,
+                           (maximum_memory_bytes, maximum_memory_bytes))
+        if not platform.uname().system == 'Darwin':
+            resource.setrlimit(resource.RLIMIT_STACK,
+                               (maximum_memory_bytes, maximum_memory_bytes))
+
+    faulthandler.disable()
+
+    import builtins
+
+    builtins.exit = None
+    builtins.quit = None
+
+    os.environ['OMP_NUM_THREADS'] = '1'
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__['help'] = None
+
+    import sys
+
+    sys.modules['ipdb'] = None
+    sys.modules['joblib'] = None
+    sys.modules['resource'] = None
+    sys.modules['psutil'] = None
+    sys.modules['tkinter'] = None