From 9fa38ffcfe54a0bd449eee9ffd73806c9e26f097 Mon Sep 17 00:00:00 2001 From: "Mr. Python" <2789762371@qq.com> Date: Sun, 16 Feb 2025 19:49:09 +0800 Subject: [PATCH 1/6] Add type hints for Compare.program --- cyaron/compare.py | 175 +++++++++++++++++++---------------- cyaron/io.py | 4 +- cyaron/tests/compare_test.py | 36 ++++--- 3 files changed, 118 insertions(+), 97 deletions(-) diff --git a/cyaron/compare.py b/cyaron/compare.py index 3b4fec0..858f9b2 100644 --- a/cyaron/compare.py +++ b/cyaron/compare.py @@ -1,14 +1,19 @@ from __future__ import absolute_import, print_function -from .io import IO -from . import log -from cyaron.utils import * -from cyaron.consts import * -from cyaron.graders import CYaRonGraders -import subprocess + import multiprocessing +import os +import subprocess import sys +from concurrent.futures import ThreadPoolExecutor from io import open -import os +from typing import List, Optional, Tuple, Union + +from cyaron.consts import * +from cyaron.graders import CYaRonGraders +from cyaron.utils import * + +from . import log +from .io import IO class CompareMismatch(ValueError): @@ -34,13 +39,18 @@ def __compare_two(name, content, std, grader): raise CompareMismatch(name, info) @staticmethod - def __process_file(file): + def __process_output_file(file: Union[str, IO]): if isinstance(file, IO): + if file.output_filename is None: + raise ValueError("IO object has no output file.") file.flush_buffer() - file.output_file.seek(0) - return file.output_filename, file.output_file.read() + with open(file.output_filename, + "r", + newline="\n", + encoding='utf-8') as f: + return file.output_filename, f.read() else: - with open(file, "r", newline="\n") as f: + with open(file, "r", newline="\n", encoding="utf-8") as f: return file, f.read() @staticmethod @@ -87,7 +97,7 @@ def output(cls, *files, **kwargs): pass def get_std(): - return cls.__process_file(std)[1] + return cls.__process_output_file(std)[1] if job_pool is not None: std = job_pool.submit(get_std).result() @@ -95,7 +105,7 @@ def get_std(): std = get_std() def do(file): - (file_name, content) = cls.__process_file(file) + (file_name, content) = cls.__process_output_file(file) cls.__compare_two(file_name, content, std, grader) if job_pool is not None: @@ -104,35 +114,36 @@ def do(file): [x for x in map(do, files)] @classmethod - def program(cls, *programs, **kwargs): - kwargs = unpack_kwargs( - "program", - kwargs, - ( - "input", - ("std", None), - ("std_program", None), - ("grader", DEFAULT_GRADER), - ("max_workers", -1), - ("job_pool", None), - ("stop_on_incorrect", None), - ), - ) - input = kwargs["input"] - std = kwargs["std"] - std_program = kwargs["std_program"] - grader = kwargs["grader"] - max_workers = kwargs["max_workers"] - job_pool = kwargs["job_pool"] - if kwargs["stop_on_incorrect"] is not None: + def program(cls, + *programs: Optional[Union[str, Tuple[str, ...], List[str]]], + input: Union[IO, str], + std: Optional[Union[str, IO]] = None, + std_program: Optional[Union[str, Tuple[str, ...], + List[str]]] = None, + grader: Optional[str] = DEFAULT_GRADER, + max_workers: int = -1, + job_pool: Optional[ThreadPoolExecutor] = None, + stop_on_incorrect=None): + """ + Compare the output of the programs with the standard output. + + Args: + programs: The programs to be compared. + input: The input file. + std: The standard output file. + std_program: The program that generates the standard output. + grader: The grader to be used. + max_workers: The maximum number of workers. + job_pool: The job pool. + stop_on_incorrect: Deprecated and has no effect. + """ + if stop_on_incorrect is not None: log.warn( "parameter stop_on_incorrect is deprecated and has no effect.") if (max_workers is None or max_workers >= 0) and job_pool is None: max_workers = cls.__normal_max_workers(max_workers) try: - from concurrent.futures import ThreadPoolExecutor - with ThreadPoolExecutor(max_workers=max_workers) as job_pool: return cls.program(*programs, input=input, @@ -144,74 +155,74 @@ def program(cls, *programs, **kwargs): except ImportError: pass - if not isinstance(input, IO): - raise TypeError("expect {}, got {}".format( - type(IO).__name__, - type(input).__name__)) - input.flush_buffer() - input.input_file.seek(0) + if isinstance(input, IO): + input.flush_buffer() if std_program is not None: - def get_std(): - with open(os.dup(input.input_file.fileno()), "r", - newline="\n") as input_file: - content = make_unicode( - subprocess.check_output( - std_program, - shell=(not list_like(std_program)), - stdin=input.input_file, - universal_newlines=True, - )) - input_file.seek(0) + def get_std_from_std_program(): + with open(input.input_filename + if isinstance(input, IO) else input, + "r", + newline="\n", + encoding="utf-8") as input_file: + content = subprocess.check_output( + std_program, + shell=(not list_like(std_program)), + stdin=input_file, + universal_newlines=True, + encoding="utf-8") return content if job_pool is not None: - std = job_pool.submit(get_std).result() + std = job_pool.submit(get_std_from_std_program).result() else: - std = get_std() + std = get_std_from_std_program() elif std is not None: - def get_std(): - return cls.__process_file(std)[1] + def get_std_from_std_file(): + return cls.__process_output_file(std)[1] if job_pool is not None: - std = job_pool.submit(get_std).result() + std = job_pool.submit(get_std_from_std_file).result() else: - std = get_std() + std = get_std_from_std_file() else: raise TypeError( "program() missing 1 required non-None keyword-only argument: 'std' or 'std_program'" ) - def do(program_name): - timeout = None - if (list_like(program_name) and len(program_name) == 2 - and int_like(program_name[-1])): - program_name, timeout = program_name - with open(os.dup(input.input_file.fileno()), "r", - newline="\n") as input_file: + with open(input.input_filename if isinstance(input, IO) else input, + "r", + newline="\n", + encoding="utf-8") as input_file: + + def do(program_name): + timeout = None + if (list_like(program_name) and len(program_name) == 2 + and int_like(program_name[-1])): + program_name, timeout = program_name if timeout is None: - content = make_unicode( - subprocess.check_output( - program_name, - shell=(not list_like(program_name)), - stdin=input_file, - universal_newlines=True, - )) + content = subprocess.check_output( + program_name, + shell=(not list_like(program_name)), + stdin=input_file, + universal_newlines=True, + encoding="utf-8", + ) else: - content = make_unicode( - subprocess.check_output( + content = subprocess.check_output( program_name, shell=(not list_like(program_name)), stdin=input_file, universal_newlines=True, timeout=timeout, - )) - input_file.seek(0) - cls.__compare_two(program_name, content, std, grader) + encoding="utf-8", + ) + cls.__compare_two(program_name, content, std, grader) - if job_pool is not None: - job_pool.map(do, programs) - else: - [x for x in map(do, programs)] + if job_pool is not None: + job_pool.map(do, programs) + else: + for program in programs: + do(program) diff --git a/cyaron/io.py b/cyaron/io.py index 61b5d4e..5e186b1 100644 --- a/cyaron/io.py +++ b/cyaron/io.py @@ -100,7 +100,7 @@ def __init__( # type: ignore output_file = "{}{{}}{}".format( self.__escape_format(file_prefix), self.__escape_format(output_suffix)) - self.input_filename, self.output_filename = None, None + self.input_filename, self.output_filename = cast(str, None), None self.__input_temp, self.__output_temp = False, False self.__init_file(input_file, data_id, "i", make_dirs) if not disable_output: @@ -357,3 +357,5 @@ def output_clear_content(self, pos: int = 0): def flush_buffer(self): """Flush the input file""" self.input_file.flush() + if self.output_file: + self.output_file.flush() diff --git a/cyaron/tests/compare_test.py b/cyaron/tests/compare_test.py index 7b6c487..3a6e109 100644 --- a/cyaron/tests/compare_test.py +++ b/cyaron/tests/compare_test.py @@ -108,28 +108,36 @@ def test_fulltext_program(self): correct_out = 'python correct.py: Correct \npython incorrect.py: !!!INCORRECT!!! Hash mismatch: read 53c234e5e8472b6ac51c1ae1cab3fe06fad053beb8ebfd8977b010655bfdd3c3, expected 4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865' self.assertEqual(result, correct_out) - def test_file_input(self): + def test_file_input_success(self): with open("correct.py", "w") as f: f.write("print(input())") - with open("std.py", "w") as f: f.write("print(input())") - - io = None - with captured_output() as (out, err): - io = IO() - + io = IO() io.input_writeln("233") - - with captured_output() as (out, err): - Compare.program("python correct.py", - std_program="python std.py", + with captured_output(): + Compare.program((sys.executable, "correct.py"), + std_program=(sys.executable, "std.py"), input=io, grader="NOIPStyle") - result = out.getvalue().strip() - correct_out = 'python correct.py: Correct' - self.assertEqual(result, correct_out) + def test_file_input_fail(self): + with open("correct.py", "w") as f: + f.write("print(input())") + with open("std.py", "w") as f: + f.write("print(input()+'154')") + io = IO() + io.input_writeln("233") + try: + with captured_output(): + Compare.program((sys.executable, "correct.py"), + std_program=(sys.executable, "std.py"), + input=io, + grader="NOIPStyle") + except CompareMismatch: + pass + else: + self.fail("Should raise CompareMismatch") def test_concurrent(self): programs = ['test{}.py'.format(i) for i in range(16)] From b5b84b12b4c5c383591f953623d69cbf046d2ad5 Mon Sep 17 00:00:00 2001 From: "Mr. Python" <2789762371@qq.com> Date: Sun, 16 Feb 2025 21:36:30 +0800 Subject: [PATCH 2/6] Support define grader by input function --- cyaron/compare.py | 27 ++++++++-------- cyaron/graders/__init__.py | 2 +- cyaron/graders/graderregistry.py | 24 +++++++++++--- cyaron/tests/compare_test.py | 54 ++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 19 deletions(-) diff --git a/cyaron/compare.py b/cyaron/compare.py index 858f9b2..46e8b28 100644 --- a/cyaron/compare.py +++ b/cyaron/compare.py @@ -9,7 +9,7 @@ from typing import List, Optional, Tuple, Union from cyaron.consts import * -from cyaron.graders import CYaRonGraders +from cyaron.graders import CYaRonGraders, GraderType from cyaron.utils import * from . import log @@ -27,11 +27,14 @@ def __str__(self): return "In program: '{}'. {}".format(self.name, self.mismatch) +PrgoramType = Optional[Union[str, Tuple[str, ...], List[str]]] + + class Compare: @staticmethod def __compare_two(name, content, std, grader): - (result, info) = CYaRonGraders.invoke(grader, content, std) + result, info = CYaRonGraders.invoke(grader, content, std) status = "Correct" if result else "!!!INCORRECT!!!" info = info if info is not None else "" log.debug("{}: {} {}".format(name, status, info)) @@ -85,8 +88,6 @@ def output(cls, *files, **kwargs): if (max_workers is None or max_workers >= 0) and job_pool is None: max_workers = cls.__normal_max_workers(max_workers) try: - from concurrent.futures import ThreadPoolExecutor - with ThreadPoolExecutor(max_workers=max_workers) as job_pool: return cls.output(*files, std=std, @@ -115,12 +116,12 @@ def do(file): @classmethod def program(cls, - *programs: Optional[Union[str, Tuple[str, ...], List[str]]], + *programs: Union[PrgoramType, Tuple[PrgoramType, float]], input: Union[IO, str], std: Optional[Union[str, IO]] = None, std_program: Optional[Union[str, Tuple[str, ...], List[str]]] = None, - grader: Optional[str] = DEFAULT_GRADER, + grader: Union[str, GraderType] = DEFAULT_GRADER, max_workers: int = -1, job_pool: Optional[ThreadPoolExecutor] = None, stop_on_incorrect=None): @@ -212,13 +213,13 @@ def do(program_name): ) else: content = subprocess.check_output( - program_name, - shell=(not list_like(program_name)), - stdin=input_file, - universal_newlines=True, - timeout=timeout, - encoding="utf-8", - ) + program_name, + shell=(not list_like(program_name)), + stdin=input_file, + universal_newlines=True, + timeout=timeout, + encoding="utf-8", + ) cls.__compare_two(program_name, content, std, grader) if job_pool is not None: diff --git a/cyaron/graders/__init__.py b/cyaron/graders/__init__.py index b11a375..68276e3 100644 --- a/cyaron/graders/__init__.py +++ b/cyaron/graders/__init__.py @@ -1,4 +1,4 @@ -from .graderregistry import CYaRonGraders +from .graderregistry import CYaRonGraders, GraderType from .fulltext import fulltext from .noipstyle import noipstyle \ No newline at end of file diff --git a/cyaron/graders/graderregistry.py b/cyaron/graders/graderregistry.py index 659f239..11ae780 100644 --- a/cyaron/graders/graderregistry.py +++ b/cyaron/graders/graderregistry.py @@ -1,18 +1,32 @@ +from typing import Callable, Tuple, Dict, Union + +__all__ = ['CYaRonGraders', 'GraderType'] + +GraderType = Callable[[str, str], Tuple[bool, Union[str, None]]] + + class GraderRegistry: - _registry = dict() + """A registry for grader functions.""" + _registry: Dict[str, GraderType] = {} - def grader(self, name): + def grader(self, name: str): + """A decorator to register a grader function.""" - def wrapper(func): + def wrapper(func: GraderType): self._registry[name] = func return func return wrapper - def invoke(self, name, content, std): - return self._registry[name](content, std) + def invoke(self, grader: Union[str, GraderType], content: str, std: str): + """Invoke a grader function by name or function object.""" + if isinstance(grader, str): + return self._registry[grader](content, std) + else: + return grader(content, std) def check(self, name): + """Check if a grader is registered.""" return name in self._registry diff --git a/cyaron/tests/compare_test.py b/cyaron/tests/compare_test.py index 3a6e109..ba4a1c5 100644 --- a/cyaron/tests/compare_test.py +++ b/cyaron/tests/compare_test.py @@ -8,6 +8,7 @@ from cyaron.output_capture import captured_output from cyaron.graders.mismatch import * from cyaron.compare import CompareMismatch +from cyaron.graders import CYaRonGraders log.set_verbose() @@ -176,3 +177,56 @@ def test_timeout(self): pass else: self.assertTrue(False) + + def test_custom_grader_by_name(self): + + @CYaRonGraders.grader("CustomTestGrader") + def custom_test_grader(content: str, std: str): + if content == '1\n' and std == '2\n': + return True, None + return False, "CustomTestGrader failed" + + io = IO() + io.output_writeln("2") + + Compare.program("echo 1", + std=io, + input=IO(), + grader="CustomTestGrader") + + try: + Compare.program("echo 2", + std=io, + input=IO(), + grader="CustomTestGrader") + except CompareMismatch as e: + self.assertEqual(e.name, 'echo 2') + self.assertEqual(e.mismatch, "CustomTestGrader failed") + else: + self.fail("Should raise CompareMismatch") + + def test_custom_grader_by_function(self): + + def custom_test_grader(content: str, std: str): + if content == '1\n' and std == '2\n': + return True, None + return False, "CustomTestGrader failed" + + io = IO() + io.output_writeln("2") + + Compare.program("echo 1", + std=io, + input=IO(), + grader=custom_test_grader) + + try: + Compare.program("echo 2", + std=io, + input=IO(), + grader=custom_test_grader) + except CompareMismatch as e: + self.assertEqual(e.name, 'echo 2') + self.assertEqual(e.mismatch, "CustomTestGrader failed") + else: + self.fail("Should raise CompareMismatch") From a1313749aca091048d21ae3b8dd16fa5b9d7ac7d Mon Sep 17 00:00:00 2001 From: "Mr. Python" <2789762371@qq.com> Date: Mon, 17 Feb 2025 23:11:25 +0800 Subject: [PATCH 3/6] =?UTF-8?q?grader=E6=94=AF=E6=8C=81=E8=8E=B7=E5=8F=96i?= =?UTF-8?q?nput?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cyaron/compare.py | 86 ++++++++++++++++---------------- cyaron/graders/__init__.py | 2 +- cyaron/graders/graderregistry.py | 42 ++++++++++++---- cyaron/tests/compare_test.py | 56 +++++++++++++++------ 4 files changed, 117 insertions(+), 69 deletions(-) diff --git a/cyaron/compare.py b/cyaron/compare.py index 46e8b28..53ec9c5 100644 --- a/cyaron/compare.py +++ b/cyaron/compare.py @@ -1,15 +1,14 @@ from __future__ import absolute_import, print_function import multiprocessing -import os import subprocess import sys from concurrent.futures import ThreadPoolExecutor from io import open -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Union, cast from cyaron.consts import * -from cyaron.graders import CYaRonGraders, GraderType +from cyaron.graders import CYaRonGraders, GraderType3 from cyaron.utils import * from . import log @@ -27,14 +26,16 @@ def __str__(self): return "In program: '{}'. {}".format(self.name, self.mismatch) -PrgoramType = Optional[Union[str, Tuple[str, ...], List[str]]] +PrgoramType = Union[str, Tuple[str, ...], List[str]] class Compare: @staticmethod - def __compare_two(name, content, std, grader): - result, info = CYaRonGraders.invoke(grader, content, std) + def __compare_two(name: PrgoramType, content: str, std: str, + input_content: str, grader: Union[str, GraderType3]): + result, info = CYaRonGraders.invoke(grader, content, std, + input_content) status = "Correct" if result else "!!!INCORRECT!!!" info = info if info is not None else "" log.debug("{}: {} {}".format(name, status, info)) @@ -77,7 +78,7 @@ def output(cls, *files, **kwargs): ("stop_on_incorrect", None), ), ) - std = kwargs["std"] + std: IO = kwargs["std"] grader = kwargs["grader"] max_workers = kwargs["max_workers"] job_pool = kwargs["job_pool"] @@ -101,13 +102,18 @@ def get_std(): return cls.__process_output_file(std)[1] if job_pool is not None: - std = job_pool.submit(get_std).result() + std_answer = job_pool.submit(get_std).result() else: - std = get_std() + std_answer = get_std() + + with open(std.input_filename, "r", newline="\n", + encoding="utf-8") as input_file: + input_text = input_file.read() def do(file): (file_name, content) = cls.__process_output_file(file) - cls.__compare_two(file_name, content, std, grader) + cls.__compare_two(file_name, content, std_answer, input_text, + grader) if job_pool is not None: job_pool.map(do, files) @@ -121,8 +127,8 @@ def program(cls, std: Optional[Union[str, IO]] = None, std_program: Optional[Union[str, Tuple[str, ...], List[str]]] = None, - grader: Union[str, GraderType] = DEFAULT_GRADER, - max_workers: int = -1, + grader: Union[str, GraderType3] = DEFAULT_GRADER, + max_workers: Optional[int] = -1, job_pool: Optional[ThreadPoolExecutor] = None, stop_on_incorrect=None): """ @@ -182,7 +188,7 @@ def get_std_from_std_program(): elif std is not None: def get_std_from_std_file(): - return cls.__process_output_file(std)[1] + return cls.__process_output_file(cast(Union[str, IO], std))[1] if job_pool is not None: std = job_pool.submit(get_std_from_std_file).result() @@ -197,33 +203,29 @@ def get_std_from_std_file(): "r", newline="\n", encoding="utf-8") as input_file: - - def do(program_name): - timeout = None - if (list_like(program_name) and len(program_name) == 2 - and int_like(program_name[-1])): - program_name, timeout = program_name - if timeout is None: - content = subprocess.check_output( - program_name, - shell=(not list_like(program_name)), - stdin=input_file, - universal_newlines=True, - encoding="utf-8", - ) - else: - content = subprocess.check_output( - program_name, - shell=(not list_like(program_name)), - stdin=input_file, - universal_newlines=True, - timeout=timeout, - encoding="utf-8", - ) - cls.__compare_two(program_name, content, std, grader) - - if job_pool is not None: - job_pool.map(do, programs) + input_text = input_file.read() + + def do(program_name: Union[PrgoramType, Tuple[PrgoramType, float]]): + timeout = None + if isinstance(program_name, tuple) and len(program_name) == 2 and ( + isinstance(program_name[1], float) + or isinstance(program_name[1], int)): + program_name, timeout = cast(Tuple[PrgoramType, float], + program_name) else: - for program in programs: - do(program) + program_name = cast(PrgoramType, program_name) + content = subprocess.check_output( + list(program_name) + if isinstance(program_name, tuple) else program_name, + shell=(not list_like(program_name)), + input=input_text, + universal_newlines=True, + encoding="utf-8", + timeout=timeout) + cls.__compare_two(program_name, content, std, input_text, grader) + + if job_pool is not None: + job_pool.map(do, programs) + else: + for program in programs: + do(program) diff --git a/cyaron/graders/__init__.py b/cyaron/graders/__init__.py index 68276e3..344436b 100644 --- a/cyaron/graders/__init__.py +++ b/cyaron/graders/__init__.py @@ -1,4 +1,4 @@ -from .graderregistry import CYaRonGraders, GraderType +from .graderregistry import CYaRonGraders, GraderType2, GraderType3 from .fulltext import fulltext from .noipstyle import noipstyle \ No newline at end of file diff --git a/cyaron/graders/graderregistry.py b/cyaron/graders/graderregistry.py index 11ae780..1f07d6e 100644 --- a/cyaron/graders/graderregistry.py +++ b/cyaron/graders/graderregistry.py @@ -1,29 +1,51 @@ -from typing import Callable, Tuple, Dict, Union +from typing import Callable, Tuple, Dict, Union, Any -__all__ = ['CYaRonGraders', 'GraderType'] +__all__ = ['CYaRonGraders', 'GraderType2', 'GraderType3'] -GraderType = Callable[[str, str], Tuple[bool, Union[str, None]]] +GraderType2 = Callable[[str, str], Tuple[bool, Any]] +GraderType3 = Callable[[str, str, str], Tuple[bool, Any]] class GraderRegistry: """A registry for grader functions.""" - _registry: Dict[str, GraderType] = {} + _registry: Dict[str, GraderType3] = {} + + def grader2(self, name: str): + """ + This decorator registers a grader function under a specific name in the registry. + + The function being decorated should accept exactly two parameters (excluding + the content input). + """ + + def wrapper(func: GraderType2): + self._registry[name] = lambda content, std, _: func(content, std) + return func + + return wrapper + + grader = grader2 - def grader(self, name: str): - """A decorator to register a grader function.""" + def grader3(self, name: str): + """ + This decorator registers a grader function under a specific name in the registry. + + The function being decorated should accept exactly three parameters. + """ - def wrapper(func: GraderType): + def wrapper(func: GraderType3): self._registry[name] = func return func return wrapper - def invoke(self, grader: Union[str, GraderType], content: str, std: str): + def invoke(self, grader: Union[str, GraderType3], content: str, std: str, + input_content: str): """Invoke a grader function by name or function object.""" if isinstance(grader, str): - return self._registry[grader](content, std) + return self._registry[grader](content, std, input_content) else: - return grader(content, std) + return grader(content, std, input_content) def check(self, name): """Check if a grader is registered.""" diff --git a/cyaron/tests/compare_test.py b/cyaron/tests/compare_test.py index ba4a1c5..c0d49c1 100644 --- a/cyaron/tests/compare_test.py +++ b/cyaron/tests/compare_test.py @@ -123,15 +123,15 @@ def test_file_input_success(self): grader="NOIPStyle") def test_file_input_fail(self): - with open("correct.py", "w") as f: - f.write("print(input())") - with open("std.py", "w") as f: + with open("incorrect.py", "w") as f: f.write("print(input()+'154')") + with open("std.py", "w") as f: + f.write("print(input())") io = IO() io.input_writeln("233") try: with captured_output(): - Compare.program((sys.executable, "correct.py"), + Compare.program((sys.executable, "incorrect.py"), std_program=(sys.executable, "std.py"), input=io, grader="NOIPStyle") @@ -178,10 +178,11 @@ def test_timeout(self): else: self.assertTrue(False) - def test_custom_grader_by_name(self): + def test_custom_grader2_by_name(self): + self.assertEqual(CYaRonGraders.grader, CYaRonGraders.grader2) - @CYaRonGraders.grader("CustomTestGrader") - def custom_test_grader(content: str, std: str): + @CYaRonGraders.grader("CustomTestGrader2") + def custom_test_grader2(content: str, std: str): if content == '1\n' and std == '2\n': return True, None return False, "CustomTestGrader failed" @@ -192,13 +193,38 @@ def custom_test_grader(content: str, std: str): Compare.program("echo 1", std=io, input=IO(), - grader="CustomTestGrader") + grader="CustomTestGrader2") try: Compare.program("echo 2", std=io, input=IO(), - grader="CustomTestGrader") + grader="CustomTestGrader2") + except CompareMismatch as e: + self.assertEqual(e.name, 'echo 2') + self.assertEqual(e.mismatch, "CustomTestGrader failed") + else: + self.fail("Should raise CompareMismatch") + + def test_custom_grader3_by_name(self): + + @CYaRonGraders.grader3("CustomTestGrader3") + def custom_test_grader3(content: str, std: str, input_content: str): + if input_content == '0\n' and content == '1\n' and std == '2\n': + return True, None + return False, "CustomTestGrader failed" + + io = IO() + io.input_writeln("0") + io.output_writeln("2") + + Compare.program("echo 1", std=io, input=io, grader="CustomTestGrader3") + + try: + Compare.program("echo 2", + std=io, + input=io, + grader='CustomTestGrader3') except CompareMismatch as e: self.assertEqual(e.name, 'echo 2') self.assertEqual(e.mismatch, "CustomTestGrader failed") @@ -207,23 +233,21 @@ def custom_test_grader(content: str, std: str): def test_custom_grader_by_function(self): - def custom_test_grader(content: str, std: str): - if content == '1\n' and std == '2\n': + def custom_test_grader(content: str, std: str, input_content: str): + if input_content == '0\n' and content == '1\n' and std == '2\n': return True, None return False, "CustomTestGrader failed" io = IO() + io.input_writeln("0") io.output_writeln("2") - Compare.program("echo 1", - std=io, - input=IO(), - grader=custom_test_grader) + Compare.program("echo 1", std=io, input=io, grader=custom_test_grader) try: Compare.program("echo 2", std=io, - input=IO(), + input=io, grader=custom_test_grader) except CompareMismatch as e: self.assertEqual(e.name, 'echo 2') From b97864a336e174b4e0e72db94c30909c17fbff4a Mon Sep 17 00:00:00 2001 From: "Mr. Python" <2789762371@qq.com> Date: Tue, 18 Feb 2025 23:26:20 +0800 Subject: [PATCH 4/6] Support testlib SPJ --- cyaron/graders/__init__.py | 3 +- cyaron/graders/graderregistry.py | 4 +- cyaron/graders/testlib_checker.py | 61 +++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 cyaron/graders/testlib_checker.py diff --git a/cyaron/graders/__init__.py b/cyaron/graders/__init__.py index 344436b..486c730 100644 --- a/cyaron/graders/__init__.py +++ b/cyaron/graders/__init__.py @@ -1,4 +1,5 @@ from .graderregistry import CYaRonGraders, GraderType2, GraderType3 from .fulltext import fulltext -from .noipstyle import noipstyle \ No newline at end of file +from .noipstyle import noipstyle +from .testlib_checker import TestlibChecker diff --git a/cyaron/graders/graderregistry.py b/cyaron/graders/graderregistry.py index 1f07d6e..1a65f32 100644 --- a/cyaron/graders/graderregistry.py +++ b/cyaron/graders/graderregistry.py @@ -23,7 +23,7 @@ def wrapper(func: GraderType2): return func return wrapper - + grader = grader2 def grader3(self, name: str): @@ -47,7 +47,7 @@ def invoke(self, grader: Union[str, GraderType3], content: str, std: str, else: return grader(content, std, input_content) - def check(self, name): + def check(self, name: str): """Check if a grader is registered.""" return name in self._registry diff --git a/cyaron/graders/testlib_checker.py b/cyaron/graders/testlib_checker.py new file mode 100644 index 0000000..13f5469 --- /dev/null +++ b/cyaron/graders/testlib_checker.py @@ -0,0 +1,61 @@ +import tempfile +import subprocess +import os +import xml.etree.ElementTree as xmlElementTree +from typing import Optional + +STDOUT_DEV = "con" if os.name == "nt" else "/dev/stdout" + +__all__ = ["TestlibChecker"] + + +class TestlibCheckerResult: + + def __init__(self, result: Optional[str], outcome: str, + pctype: Optional[str]): + self.result = result + self.outcome = outcome + self.pctype = pctype + + def __str__(self): + return ' '.join([self.outcome] + + ([] if self.pctype is None else [f'({self.pctype})']) + + ([] if self.result is None else [self.result])) + + +class TestlibChecker: + """ + A grader that uses the testlib checker. + """ + + def __init__(self, checker_path: str): + self.checker_path = checker_path + + def __call__(self, outs: str, ans: str, ins: str): + with tempfile.NamedTemporaryFile( + 'w') as inf, tempfile.NamedTemporaryFile( + 'w') as outf, tempfile.NamedTemporaryFile('w') as ansf: + inf.write(ins) + outf.write(outs) + ansf.write(ans) + inf.flush() + outf.flush() + ansf.flush() + result = subprocess.run((self.checker_path, inf.name, outf.name, + ansf.name, STDOUT_DEV, '-appes'), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False) + checker_output = result.stdout + + result_element = xmlElementTree.fromstring(checker_output) + if result_element.tag != 'result': + raise ValueError("Invalid output from checker") + result_text = result_element.text + result_outcome = result_element.get('outcome') + if result_outcome is None: + raise ValueError("Invalid output from checker") + result_pctype = result_element.get('pctype') + return result_outcome == 'accepted', TestlibCheckerResult( + result_text, result_outcome, result_pctype) From 8ec5bc262b5b90335adf4a51e0a89c38609f7772 Mon Sep 17 00:00:00 2001 From: "Mr. Python" <2789762371@qq.com> Date: Wed, 19 Feb 2025 17:14:02 +0800 Subject: [PATCH 5/6] Check SPJ stderr output --- cyaron/graders/testlib_checker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cyaron/graders/testlib_checker.py b/cyaron/graders/testlib_checker.py index 13f5469..e6e47b8 100644 --- a/cyaron/graders/testlib_checker.py +++ b/cyaron/graders/testlib_checker.py @@ -47,6 +47,9 @@ def __call__(self, outs: str, ans: str, ins: str): stderr=subprocess.PIPE, text=True, check=False) + if result.stderr.strip() != 'See file to check exit message': + raise ValueError("Invalid output from checker: " + + result.stderr) checker_output = result.stdout result_element = xmlElementTree.fromstring(checker_output) From f82ae6498f236e65770c0c6d05f938d6b8a2f8d1 Mon Sep 17 00:00:00 2001 From: "Mr. Python" <2789762371@qq.com> Date: Wed, 19 Feb 2025 22:33:54 +0800 Subject: [PATCH 6/6] Use temp file to get SPJ output --- cyaron/graders/testlib_checker.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/cyaron/graders/testlib_checker.py b/cyaron/graders/testlib_checker.py index e6e47b8..513e7d5 100644 --- a/cyaron/graders/testlib_checker.py +++ b/cyaron/graders/testlib_checker.py @@ -1,10 +1,8 @@ import tempfile import subprocess -import os import xml.etree.ElementTree as xmlElementTree from typing import Optional - -STDOUT_DEV = "con" if os.name == "nt" else "/dev/stdout" +from os.path import join as path_join __all__ = ["TestlibChecker"] @@ -32,17 +30,21 @@ def __init__(self, checker_path: str): self.checker_path = checker_path def __call__(self, outs: str, ans: str, ins: str): - with tempfile.NamedTemporaryFile( - 'w') as inf, tempfile.NamedTemporaryFile( - 'w') as outf, tempfile.NamedTemporaryFile('w') as ansf: + with tempfile.NamedTemporaryFile('w') as inf, \ + tempfile.NamedTemporaryFile('w') as outf, \ + tempfile.NamedTemporaryFile('w') as ansf, \ + tempfile.TemporaryDirectory() as checker_output_dir: inf.write(ins) outf.write(outs) ansf.write(ans) inf.flush() outf.flush() ansf.flush() + checker_output_file = path_join(checker_output_dir, + 'checker_output.xml') + result = subprocess.run((self.checker_path, inf.name, outf.name, - ansf.name, STDOUT_DEV, '-appes'), + ansf.name, checker_output_file, '-appes'), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -50,9 +52,9 @@ def __call__(self, outs: str, ans: str, ins: str): if result.stderr.strip() != 'See file to check exit message': raise ValueError("Invalid output from checker: " + result.stderr) - checker_output = result.stdout - result_element = xmlElementTree.fromstring(checker_output) + result_element = xmlElementTree.parse( + checker_output_file).getroot() if result_element.tag != 'result': raise ValueError("Invalid output from checker") result_text = result_element.text