Fix extra files handling (#29)

* Fix extra files handling

* Add tests

* Run formatters

Author: MasloMaslane

src/sio3pack/packages/package/configuration.py

@@ -13,7 +13,7 @@ def detect(cls) -> dict[str, "CompilerConfig"]:
         # TODO: implement this properly
         return {
             "cpp": cls("g++", "g++-12.2", "g++", ["-std=c++20", "-O3"]),
-            "py": cls("python", "python3.10", "python", []),
+            "python": cls("python", "python3.10", "python", []),
         }
 
 

src/sio3pack/packages/package/model.py

@@ -172,10 +172,6 @@ def get_title(self, lang: str | None = None) -> str:
     def get_statement(self, lang: str | None = None) -> File | None:
         pass
 
-    @wrap_exceptions
-    def get_tests(self) -> list[Test]:
-        pass
-
     @wrap_exceptions
     def get_test(self, test_id: str) -> Test:
         pass

src/sio3pack/packages/sinolpack/model.py

@@ -482,6 +482,9 @@ def _get_special_file_path(self, file_type: str) -> str | None:
             return self.special_files[file_type].path
         return None
 
+    def get_ingen_path(self) -> str | None:
+        return self._get_special_file_path("ingen")
+
     def get_inwer_path(self) -> str | None:
         return self._get_special_file_path("inwer")
 
@@ -543,7 +546,7 @@ def get_extra_execution_files(self) -> list[File]:
             return [
                 LocalFile(os.path.join(self.rootdir, "prog", f))
                 for f in self.config.get("extra_execution_files", [])
-                if os.path.isfile(os.path.join(self.rootdir, f))
+                if os.path.isfile(os.path.join(self.rootdir, "prog", f))
             ]
 
     def get_extra_compilation_files(self) -> list[File]:
@@ -559,7 +562,7 @@ def get_extra_compilation_files(self) -> list[File]:
             return [
                 LocalFile(os.path.join(self.rootdir, "prog", f))
                 for f in self.config.get("extra_compilation_files", [])
-                if os.path.isfile(os.path.join(self.rootdir, f))
+                if os.path.isfile(os.path.join(self.rootdir, "prog", f))
             ]
 
     def _get_limit(self, test: Test, language: str, type: str) -> int:

src/sio3pack/packages/sinolpack/workflows.py

@@ -52,7 +52,7 @@ def get_compile_file_workflow(self, file: File | str) -> tuple[Workflow, str]:
 
     def _get_ingen_workflow(self) -> Workflow:
         workflow = Workflow(
-            "ingen",
+            "Run ingen",
             observable_registers=1,
         )
         ingen_path = self.package.get_ingen_path()

src/sio3pack/workflow/execution/mount_namespace.py

@@ -139,3 +139,9 @@ def to_json(self) -> list[dict]:
         Convert the mount namespace manager to a dictionary.
         """
         return [mount_namespace.to_json() for mount_namespace in self.mount_namespaces]
+
+    def len(self) -> int:
+        """
+        Get the number of mount namespaces.
+        """
+        return len(self.mount_namespaces)

src/sio3pack/workflow/object.py

@@ -17,6 +17,9 @@ def __init__(self, handle: str):
     def __str__(self):
         return f"<Object {self.handle}>"
 
+    def __repr__(self):
+        return f"<Object {self.handle}>"
+
     def replace_templates(self, replacements: dict[str, str]):
         """
         Replace strings in the object with the given replacements.
@@ -113,7 +116,7 @@ def __str__(self):
 
         :return: The string representation of the list.
         """
-        return f"<ObjectList {len(self.objects)} objects>"
+        return f"<ObjectList {self.objects}>"
 
     def union(self, other: "ObjectList"):
         """

tests/packages/sinolpack/test_workflows.py

@@ -7,6 +7,7 @@
 from sio3pack.packages.package.configuration import SIO3PackConfig
 from sio3pack.workflow import ExecutionTask, ScriptTask, Workflow
 from sio3pack.workflow.execution import ObjectReadStream, ObjectWriteStream
+from sio3pack.workflow.execution.filesystems import ObjectFilesystem
 from tests.fixtures import PackageInfo, get_package
 from tests.packages.sinolpack.utils import common_checks
 
@@ -25,9 +26,10 @@ def test_unpack_workflows(get_package):
 
     if package_info.task_id == "abc":
         # We expect 2 workflows: compile checker and outgen
-        assert len(workflows) == 2
+        assert len(workflows) == 3
         assert workflows[0].name == "Compile checker"
-        assert workflows[1].name == "Outgen tests"
+        assert workflows[1].name == "Run ingen"
+        assert workflows[2].name == "Outgen tests"
     elif package_info.task_id == "wer":
         # We expect 3 workflows: compile checker, outgen and inwer
         assert len(workflows) == 3
@@ -166,3 +168,141 @@ def test_custom_workflow(get_package):
 
     assert num_custom == 3, "Should have 3 custom execution tasks"
     assert num_custom_scripts == 3, "Should have 3 custom script tasks"
+
+
+@pytest.mark.parametrize("get_package", ["simple"], indirect=True)
+def test_user_out_workflow(get_package):
+    package_info: PackageInfo = get_package()
+    package = sio3pack.from_file(package_info.path, SIO3PackConfig.detect())
+    common_checks(package_info, package)
+
+    program = LocalFile(os.path.join(package.rootdir, "prog", "abc.cpp"))
+    test = package.tests[0]
+
+    op = package.get_user_out_operation(program, test)
+    workflows = [wf for wf in op.get_workflow()]
+    assert len(workflows) == 1
+    workflow = workflows[0]
+
+    assert len(workflow.observable_objects) == 1, "Should have just user out object"
+    assert workflow.observable_objects[0].handle.startswith("user_out_"), "User out object should be user_out_"
+
+    num_compiles = 0
+    num_runs = 0
+    for task in workflow.tasks:
+        if task.name == f"Compile {program.path} using g++-12.2":
+            assert len(task.processes) == 1, "Compile task should have one process"
+            assert task.processes[0].image == "compiler:g++-12.2", "Compile task should use g++-12.2 image"
+            num_compiles += 1
+        elif task.name.startswith("Run solution for test"):
+            num_runs += 1
+            assert len(task.processes) == 1, "Run task should have one process"
+            proc = task.processes[0]
+            stdin_fd = proc.descriptor_manager.get(0)
+            assert isinstance(stdin_fd, ObjectReadStream), "Run task should have stdin stream"
+            stdout_fd = proc.descriptor_manager.get(1)
+            assert isinstance(stdout_fd, ObjectWriteStream), "Run task should have stdout stream"
+            assert stdout_fd.object.handle.startswith("user_out"), "Run task should have user_out stream"
+
+    assert num_compiles == 1, "Should have one compile task"
+    assert num_runs == 1, "Should have one run task"
+
+
+@pytest.mark.parametrize("get_package", ["simple"], indirect=True)
+def test_test_run_workflow(get_package):
+    package_info: PackageInfo = get_package()
+    package = sio3pack.from_file(package_info.path, SIO3PackConfig.detect())
+    common_checks(package_info, package)
+
+    program = LocalFile(os.path.join(package.rootdir, "prog", "abc.cpp"))
+    test = LocalFile(os.path.join(package.rootdir, "in", "abc0.in"))
+
+    op = package.get_test_run_operation(program, test)
+    workflows = [wf for wf in op.get_workflow()]
+    assert len(workflows) == 1
+    workflow = workflows[0]
+
+    assert len(workflow.observable_objects) == 1, "Should have just user out object"
+    out_obj = workflow.observable_objects[0]
+    assert out_obj.handle == f"test_run_{program.filename}", "User out object should be user_out_<filename>"
+
+    assert len(workflow.tasks) == 2, "Should have two tasks"
+    assert workflow.tasks[0].name == f"Compile {program.path} using g++-12.2", "First task should be compile"
+    assert workflow.tasks[1].name == f"Run solution for test", "Second task should be run"
+    exec_task = workflow.tasks[1]
+    assert isinstance(exec_task, ExecutionTask), "Second task should be an execution task"
+    assert len(exec_task.processes) == 1, "Run task should have one process"
+    proc = exec_task.processes[0]
+    stdin_fd = proc.descriptor_manager.get(0)
+    assert isinstance(stdin_fd, ObjectReadStream), "Run task should have stdin stream"
+    assert stdin_fd.object.handle == test.path, "Run task should have stdin stream"
+    stdout_fd = proc.descriptor_manager.get(1)
+    assert isinstance(stdout_fd, ObjectWriteStream), "Run task should have stdout stream"
+    assert stdout_fd.object.handle == out_obj.handle, "Run task should have stdout stream"
+
+
+@pytest.mark.parametrize("get_package", ["extra_files"], indirect=True)
+def test_extra_files(get_package):
+    package_info: PackageInfo = get_package()
+    package = sio3pack.from_file(package_info.path, SIO3PackConfig.detect())
+    common_checks(package_info, package)
+
+    program = LocalFile(os.path.join(package.rootdir, "prog", "ext.cpp"))
+
+    op = package.get_run_operation(program)
+    workflows = [wf for wf in op.get_workflow()]
+    assert len(workflows) == 1
+    workflow = workflows[0]
+
+    assert len(workflow.external_objects) == 5
+    extlib_h = None
+    extlib_py = None
+    for obj in workflow.external_objects:
+        if obj.handle.endswith("extlib.h"):
+            extlib_h = obj
+        elif obj.handle.endswith("extlib.py"):
+            extlib_py = obj
+    assert extlib_h is not None, "Should have extlib.h as external object"
+    assert extlib_py is not None, "Should have extlib.py as external object"
+
+    for task in workflow.tasks:
+        if isinstance(task, ExecutionTask):
+            if task.name == f"Compile {program.path} using g++-12.2":
+                assert task.filesystem_manager.len() == 2
+                ext_fs = task.filesystem_manager.get_by_id(1)
+                assert isinstance(ext_fs, ObjectFilesystem), "Should have object filesystem with external file"
+                assert ext_fs.object.handle == extlib_h.handle, "Should have extlib.h as external file"
+
+                assert task.mountnamespace_manager.len() == 1, "Should have one mount namespace"
+                assert len(task.mountnamespace_manager.get_by_id(0).mountpoints) == 2, "Should have two mount points"
+
+                proc = task.processes[0]
+                assert "extlib.h" in proc.arguments, "Should have extlib.h in arguments"
+            elif task.name.startswith("Run solution for test"):
+                assert task.filesystem_manager.len() == 2
+                ext_fs = task.filesystem_manager.get_by_id(1)
+                assert isinstance(ext_fs, ObjectFilesystem), "Should have object filesystem with external file"
+                assert ext_fs.object.handle == extlib_py.handle, "Should have extlib.py as external file"
+
+                assert task.mountnamespace_manager.len() == 1, "Should have one mount namespace"
+                assert len(task.mountnamespace_manager.get_by_id(0).mountpoints) == 2, "Should have two mount points"
+
+    # Check that python compilation doesnt have extlib.h in compilation args.
+    program = LocalFile(os.path.join(package.rootdir, "prog", "ext.py"))
+    op = package.get_run_operation(program)
+    workflows = [wf for wf in op.get_workflow()]
+    assert len(workflows) == 1
+
+    for task in workflows[0].tasks:
+        if isinstance(task, ExecutionTask):
+            if task.name == f"Compile {program.path} using python":
+                assert task.filesystem_manager.len() == 2
+                ext_fs = task.filesystem_manager.get_by_id(1)
+                assert isinstance(ext_fs, ObjectFilesystem), "Should have object filesystem with external file"
+                assert ext_fs.object.handle == extlib_py.handle, "Should have extlib.py as external file"
+
+                assert task.mountnamespace_manager.len() == 1, "Should have one mount namespace"
+                assert len(task.mountnamespace_manager.get_by_id(0).mountpoints) == 2, "Should have two mount points"
+
+                proc = task.processes[0]
+                assert "extlib.h" not in proc.arguments, "Should not have extlib.h in arguments"

tests/test_django/test_sio3pack/test_sinolpack.py

@@ -82,7 +82,7 @@ def test_additional_files(get_archived_package):
     package, db_package = _save_and_test_simple(package_info)
     additional_files = package.additional_files
     db_additional_files = SinolpackAdditionalFile.objects.filter(package=db_package)
-    assert db_additional_files.count() == 1
+    assert db_additional_files.count() == 2
     assert len(additional_files) == db_additional_files.count()
 
     for file in additional_files:

tests/test_packages/extra_files/__init__.py

@@ -0,0 +1,2 @@
+TASK_ID = "ext"
+TYPE = "sinolpack"

tests/test_packages/extra_files/config.yml

@@ -0,0 +1,6 @@
+title: Package with extra files
+extra_compilation_files: ['extlib.h']
+extra_compilation_args:
+  cpp:
+    - 'extlib.h'
+extra_execution_files: ['extlib.py']