diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 5e2f16b..892605c 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -3,7 +3,8 @@ name: publish to Pypi on: push: tags: - - v* + - 'v[0-9]+.[0-9]+.[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+-*' jobs: publish: @@ -28,6 +29,15 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true + - name: Set package version + run: | + TAG=${{ github.ref_name }} + VERSION=$(echo "$TAG" | sed 's/^v//') + + echo "tag: $TAG, version: ${VERSION}" + sed -i -e 's|^version = "0.0.0-dev"|version = "'$VERSION'"|' pyproject.toml + + - name: Set build hash run: | HASH=${{ github.sha }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 14ea1da..7169c97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Change Log +## [0.7.0] + +### Features and Enhancements + +- [cli] adjustable container executor per program +- [cli] support custom sandbox executor class for extendable parameters +- [config] add default custom executor for `apple_container` +- [config] set volume type as optional +- [config] fetch another and merge another property by using `fetch_prop(location)` +- [config] toggleable `fetch_prop(location)` by env `SNDK_FETCH_PROP` (value `yes`) +- [config] flatten if the container caller is array and the result of `fetch_prop(location)` helper is an array +- [packaging] actual version will be injected during build and publish pipeline + ## [0.6.0] ### Features and Enhancements diff --git a/README.md b/README.md index 69108d0..7428d5d 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,13 @@ pip show sandock | grep "Location: " | awk '{ print $2 }' | sed 's/lib.*$/bin/g' then create a symbolic link to where your env var `$PATH` located. +**Another way**, if you use [mise](https://mise.jdx.dev/) set the following line into it's pinned version file. + +```toml +[tools] +"pipx:sandock" = { version = "[VERSION]", extras = "yml-config" } +``` + ### 3. Create Configuration File Initialize configuration file, example: @@ -192,81 +199,85 @@ You can find the some of the samples in [examples](./examples/). click here to expand -| Param | Defaults | Description | Required | -| ----------------------------------------------- | -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -| .execution | `{}` | a section, related to the execution with it's adjustable parameters | `no` | -| .execution.docker | `"docker"` | container program that will be executed | `no` | -| .execution.container_name_prefix | `"sandock"` | the prefix of the created container, if it's not the persistent | `no` | -| .execution.property_override_prefix_arg | `"sandbox-arg"` | the prefix of argument name during `run` subcommand that will be overrided some of program property | `no` | -| .execution.alias_program_prefix | `""` | the prefix that will be added in generated alias subcommand | `no` | -| .backup | `{}` | a section, related to backup configuration parameters | `no` | -| .backup.restic | `{}` | a sub section, related to the used restic container for backup | `no` | -| .backup.restic.image | `"restic/restic:0.18.0"` | restic image version | `no` | -| .backup.restic.compression | `"auto"` | backup compression type | `no` | -| .backup.restic.no_snapshot_unless_changed | `True` | will not create a new backup snapshot if there isn't new changes | `no` | -| .backup.restic.extra_args | `[]` | Additional (global) restic argument in each execution | `no` | -| .backup.path | `"${HOME}/.sandock_vol_backup"` | backup (local) path | `no` | -| .backup.no_password | `False` | set to `True` for no password configured in backup repository | `no` | -| .backup.volume_labels | `{}` | Key-value pattern for list of that matched with volume labels for `--all` argument during backup, it will use **AND** operation, so the more it filled the more specific it becomes execution | `no` | -| .backup.volume_excludes | `[]` | List of volume that will be execluded to backup | `no` | -| .config | `{}` | a section, related to how `sandock` interact with configuration | `no` | -| .config.current_dir_conf | `True` | enable/disable current directory configuration file ([Dot Config](#dot-config)) | `no` | -| .config.current_dir_conf_excludes | `[]` | add some folder to be excluded in current directory config reads, you can put a [full match](https://docs.python.org/3/library/re.html#re.fullmatch) regex pattern | `no` | -| .config.includes | `[]` | load external configuration files, it will be merged into the main configuration for `programs`, `volumes`, `images` and `networks` | `no` | -| .programs | `{}` | list of programs are defined here | `yes` | -| .programs | `{}` | list of programs are defined here | `yes` | -| .programs[name].image | | container image that will be loaded, this also will be set as a reference of image name for the build/custom one | `yes` | -| .programs[name].exec | | path of executeable inside container that will be ran as **entrypoint**, this is will be the main one | `yes` | -| .programs[name].extends | `[]` | extending from another program config, ensure the config name is exists | `no` | -| .programs[name].aliases | `{}` | the maps of any other executeable inside container, during subcommand **alias** by the argument **--generate**, this will generate alias by pattern "[program_name]-[alias]" | `no` | -| .programs[name].interactive | `True` | interactive mode (**-it** ~> keep STDIN and provide pseudo TTY ) | `no` | -| .programs[name].allow_home_dir | `False` | allow ran in (top of) home directory if auto sandbox mount enabled | `no` | -| .programs[name].name | | name of created container, if not set then then pattern will be generated is "[execution.container_name_prefix]-[program_name]-[timestamp]" | `no` | -| .programs[name].network | | name of network name that will be used, if it's one of defined in `.networks` then it will be create first (if not exists), you can set with "none" for no network connectivity allowed | `no` | -| .programs[name].hostname | | container hostname | `no` | -| .programs[name].build | `{}` | a subsection, define how a container build. the definition is same as defined in section `.images[name]`, if this not defined assuming the image already exists in the local container engine or it will be pulled automatically from container registry | `no` | -| .programs[name].user | | a subsection, if set then it will define the user and group id related config in the container side | `no` | -| .programs[name].user.uid | `0` | user id in container | `no` | -| .programs[name].user.gid | `0` | group id in container | `no` | -| .programs[name].user.keep_id | `False` | set the same uid and gid as the executor/host, this cannot be combined with .uid and .gid | `no` | -| .programs[name].workdir | | set the working directory | `no` | -| .programs[name].platform | | container platform type, if set, it's also affecting platform type for custom image build | `no` | -| .programs[name].persist | `{}` | a subsection, define whether its a temporary container or will be kept exists | `no` | -| .programs[name].persist.enable | `False` | enable/disable persist container | `no` | -| .programs[name].persist.auto_start | `True` | enable/disable auto start the container if the status other than **running** | `no` | -| .programs[name].sandbox_mount | `{}` | a subsection, define how the current directory to be (auto) mounted | `no` | -| .programs[name].sandbox_mount.enable | `True` | enable/disable current working directory to be auto mounted | `no` | -| .programs[name].sandbox_mount.read_only | `False` | enable/disable current directory mount as read only mode mounted | `no` | -| .programs[name].sandbox_mount.current_dir_mount | `"/sandbox"` | the path of mount point inside container, this also will be set as **--workdir** if the specific configuration was not set | `no` | -| .programs[name].env | `{}` | maps of environment variable that will be injected into container | `no` | -| .programs[name].volumes | `[]` | list of inline volume mounting definition, `${VOL_DIR}` will dynamically replaced by normalized current path | `no` | -| .programs[name].ports | `[]` | list of inline port mapping | `no` | -| .programs[name].cap_add | `[]` | list of capabilities that will be added | `no` | -| .programs[name].cap_drop | `[]` | list of capabilities that will be dropped | `no` | -| .programs[name].extra_run_args | `[]` | list of argument that will be executed during **run** in container cli, since there are some unique arguments per provider | `no` | -| .programs[name].pre_exec_cmds | `[]` | list of commands that will be execute before running the container | `no` | -| .volumes | `{}` | list of volume that will be created by `sandock`, all of volume will have label `created_by.sandock` with value `true` | `no` | -| .volumes[name].driver | `"local"` | volume driver, ensure it's supported by the container engine | `no` | -| .volumes[name].extends | `[]` | extending from another volume config, ensure the config name is exists | `no` | -| .volumes[name].driver_opts | `{}` | key-value configuration of driver options | `no` | -| .volumes[name].labels | `{}` | key-value label that will be attach to the created volume | `no` | -| .images | `{}` | list of container image build definition | `no` | -| .images[name].extends | `[]` | extending from another image config, ensure the config name is exists | `no` | -| .images[name].context | | path/location during the build time | `no` | -| .images[name].dockerfile_inline | | docker file inline declaration, this cannot be mixed with `.dockerFile` | `no` | -| .images[name].dockerFile | | path of `Dockerfile`, this cannot be mixed with `.dockerfile_inline` | `no` | -| .images[name].depends_on | | set dependency of another custom image build, to be ensured exists/created first | `no` | -| .images[name].args | `{}` | kv that will be injected as build args | `no` | -| .images[name].extra_build_args | `[]` | list of additional command argument that will be provided during build time | `no` | -| .images[name].dump | `{}` | automatically dump custom build image options | `no` | -| .images[name].dump.enable | `False` | a toggle | `no` | -| .images[name].dump.cleanup_prev | `True` | Cleanup previous dumped image file if use the standard pattern | `no` | -| .images[name].dump.store | `${HOME}/.sandock_dump_images/${image}:${platform}${hash}.tar` | a path pattern where the dumped custom image stored | `no` | -| .networks | `{}` | list of custom network declaration | `no` | -| .networks[name].extends | `[]` | extending from another network config, ensure the config name is exists | `no` | -| .networks[name].driver | `"bridge"` | driver type | `no` | -| .networks[name].driver_opts | `{}` | additional network driver options | `no` | -| .networks[name].params | `{}` | additional extra parameters in building network | `no` | +| Param | Defaults | Description | Required | +| ----------------------------------------------- | ---------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| .execution | `{}` | a section, related to the execution with it's adjustable parameters | `no` | +| .execution.docker | `"docker"` | container program that will be executed | `no` | +| .execution.container_name_prefix | `"sandock"` | the prefix of the created container, if it's not the persistent | `no` | +| .execution.property_override_prefix_arg | `"sandbox-arg"` | the prefix of argument name during `run` subcommand that will be overrided some of program property | `no` | +| .execution.alias_program_prefix | `""` | the prefix that will be added in generated alias subcommand | `no` | +| .executors | `{"apple_container": {"load_cls": "sandock.executors.AppleContainerExec", "bin_path": "container"}}` | a section, list of container external executor it's need set for `bin_path` or `load_cls` (or both) | `no` | +| .executors.[name].bin_path | | container executor cli that will be executed | `no` | +| .executors.[name].load_cls | | custom sandbox executor class | `no` | +| .backup | `{}` | a section, related to backup configuration parameters | `no` | +| .backup.restic | `{}` | a sub section, related to the used restic container for backup | `no` | +| .backup.restic.image | `"restic/restic:0.18.0"` | restic image version | `no` | +| .backup.restic.compression | `"auto"` | backup compression type | `no` | +| .backup.restic.no_snapshot_unless_changed | `True` | will not create a new backup snapshot if there isn't new changes | `no` | +| .backup.restic.extra_args | `[]` | Additional (global) restic argument in each execution | `no` | +| .backup.path | `"${HOME}/.sandock_vol_backup"` | backup (local) path | `no` | +| .backup.no_password | `False` | set to `True` for no password configured in backup repository | `no` | +| .backup.volume_labels | `{}` | Key-value pattern for list of that matched with volume labels for `--all` argument during backup, it will use **AND** operation, so the more it filled the more specific it becomes execution | `no` | +| .backup.volume_excludes | `[]` | List of volume that will be execluded to backup | `no` | +| .config | `{}` | a section, related to how `sandock` interact with configuration | `no` | +| .config.current_dir_conf | `True` | enable/disable current directory configuration file ([Dot Config](#dot-config)) | `no` | +| .config.current_dir_conf_excludes | `[]` | add some folder to be excluded in current directory config reads, you can put a [full match](https://docs.python.org/3/library/re.html#re.fullmatch) regex pattern | `no` | +| .config.includes | `[]` | load external configuration files, it will be merged into the main configuration for `programs`, `volumes`, `images` and `networks` | `no` | +| .programs | `{}` | list of programs are defined here | `yes` | +| .programs | `{}` | list of programs are defined here | `yes` | +| .programs[name].image | | container image that will be loaded, this also will be set as a reference of image name for the build/custom one | `yes` | +| .programs[name].exec | | path of executeable inside container that will be ran as **entrypoint**, this is will be the main one | `yes` | +| .programs[name].extends | `[]` | extending from another program config, ensure the config name is exists | `no` | +| .programs[name].executor | | set the container executor | `no` | +| .programs[name].aliases | `{}` | the maps of any other executeable inside container, during subcommand **alias** by the argument **--generate**, this will generate alias by pattern "[program_name]-[alias]" | `no` | +| .programs[name].interactive | `True` | interactive mode (**-it** ~> keep STDIN and provide pseudo TTY ) | `no` | +| .programs[name].allow_home_dir | `False` | allow ran in (top of) home directory if auto sandbox mount enabled | `no` | +| .programs[name].name | | name of created container, if not set then then pattern will be generated is "[execution.container_name_prefix]-[program_name]-[timestamp]" | `no` | +| .programs[name].network | | name of network name that will be used, if it's one of defined in `.networks` then it will be create first (if not exists), you can set with "none" for no network connectivity allowed | `no` | +| .programs[name].hostname | | container hostname | `no` | +| .programs[name].build | `{}` | a subsection, define how a container build. the definition is same as defined in section `.images[name]`, if this not defined assuming the image already exists in the local container engine or it will be pulled automatically from container registry | `no` | +| .programs[name].user | | a subsection, if set then it will define the user and group id related config in the container side | `no` | +| .programs[name].user.uid | `0` | user id in container | `no` | +| .programs[name].user.gid | `0` | group id in container | `no` | +| .programs[name].user.keep_id | `False` | set the same uid and gid as the executor/host, this cannot be combined with .uid and .gid | `no` | +| .programs[name].workdir | | set the working directory | `no` | +| .programs[name].platform | | container platform type, if set, it's also affecting platform type for custom image build | `no` | +| .programs[name].persist | `{}` | a subsection, define whether its a temporary container or will be kept exists | `no` | +| .programs[name].persist.enable | `False` | enable/disable persist container | `no` | +| .programs[name].persist.auto_start | `True` | enable/disable auto start the container if the status other than **running** | `no` | +| .programs[name].sandbox_mount | `{}` | a subsection, define how the current directory to be (auto) mounted | `no` | +| .programs[name].sandbox_mount.enable | `True` | enable/disable current working directory to be auto mounted | `no` | +| .programs[name].sandbox_mount.read_only | `False` | enable/disable current directory mount as read only mode mounted | `no` | +| .programs[name].sandbox_mount.current_dir_mount | `"/sandbox"` | the path of mount point inside container, this also will be set as **--workdir** if the specific configuration was not set | `no` | +| .programs[name].env | `{}` | maps of environment variable that will be injected into container | `no` | +| .programs[name].volumes | `[]` | list of inline volume mounting definition, `${VOL_DIR}` will dynamically replaced by normalized current path | `no` | +| .programs[name].ports | `[]` | list of inline port mapping | `no` | +| .programs[name].cap_add | `[]` | list of capabilities that will be added | `no` | +| .programs[name].cap_drop | `[]` | list of capabilities that will be dropped | `no` | +| .programs[name].extra_run_args | `[]` | list of argument that will be executed during **run** in container cli, since there are some unique arguments per provider | `no` | +| .programs[name].pre_exec_cmds | `[]` | list of commands that will be execute before running the container | `no` | +| .volumes | `{}` | list of volume that will be created by `sandock`, all of volume will have label `created_by.sandock` with value `true` | `no` | +| .volumes[name].driver | `"local"` | volume driver, ensure it's supported by the container engine | `no` | +| .volumes[name].extends | `[]` | extending from another volume config, ensure the config name is exists | `no` | +| .volumes[name].driver_opts | `{}` | key-value configuration of driver options | `no` | +| .volumes[name].labels | `{}` | key-value label that will be attach to the created volume | `no` | +| .images | `{}` | list of container image build definition | `no` | +| .images[name].extends | `[]` | extending from another image config, ensure the config name is exists | `no` | +| .images[name].context | | path/location during the build time | `no` | +| .images[name].dockerfile_inline | | docker file inline declaration, this cannot be mixed with `.dockerFile` | `no` | +| .images[name].dockerFile | | path of `Dockerfile`, this cannot be mixed with `.dockerfile_inline` | `no` | +| .images[name].depends_on | | set dependency of another custom image build, to be ensured exists/created first | `no` | +| .images[name].args | `{}` | kv that will be injected as build args | `no` | +| .images[name].extra_build_args | `[]` | list of additional command argument that will be provided during build time | `no` | +| .images[name].dump | `{}` | automatically dump custom build image options | `no` | +| .images[name].dump.enable | `False` | a toggle | `no` | +| .images[name].dump.cleanup_prev | `True` | Cleanup previous dumped image file if use the standard pattern | `no` | +| .images[name].dump.store | `${HOME}/.sandock_dump_images/${image}:${platform}${hash}.tar` | a path pattern where the dumped custom image stored | `no` | +| .networks | `{}` | list of custom network declaration | `no` | +| .networks[name].extends | `[]` | extending from another network config, ensure the config name is exists | `no` | +| .networks[name].driver | `"bridge"` | driver type | `no` | +| .networks[name].driver_opts | `{}` | additional network driver options | `no` | +| .networks[name].params | `{}` | additional extra parameters in building network | `no` | @@ -292,6 +303,34 @@ dot config configuration file ordered by the format: See how it can be done in variable **CONFIG_FORMAT_DECODER_MAPS** inside [sandbox.config](./sandock/config.py). +### Reuseable Property + +You can use `fetch_prop(location.to.declaration)`, the feature is similar with [Gitlab's reference](https://docs.gitlab.com/ci/yaml/yaml_optimization/#reference-tags). by following rules: + +- to reduce uneeded call when not used this feature is disable by default, set env `SNDK_FETCH_PROP` by value `yes` +- must provide the full path/location +- if the caller is member of list and it will include another property as list then it will be flatten. sample: + + ```yaml + programs: + satu: + volumes: + - here:/here + - there:/there + + dua: + volumes: + - dir_top:/top + - fetch_prop(programs.satu.volumes) + ``` + + then value of `.programs.dua.volumes` is + ```yaml + - dir_top:/top + - here:/here + - there:/there + ``` + ## Commands > [!NOTE] diff --git a/pyproject.toml b/pyproject.toml index a57fb2d..5e50003 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sandock" -version = "0.6.0" +version = "0.0.0-dev" description = "CLI tool for sandbox execution using container approach" authors = [{ name = "Imam Omar Mochtar", email = "iomarmochtar@gmail.com" }] license = { text = "MIT" } diff --git a/pyproject.toml-e b/pyproject.toml-e new file mode 100644 index 0000000..5e50003 --- /dev/null +++ b/pyproject.toml-e @@ -0,0 +1,50 @@ +[project] +name = "sandock" +version = "0.0.0-dev" +description = "CLI tool for sandbox execution using container approach" +authors = [{ name = "Imam Omar Mochtar", email = "iomarmochtar@gmail.com" }] +license = { text = "MIT" } +readme = "README.md" +requires-python = ">=3.9" + +[project.scripts] +sandock = "sandock.cli:main" + +[project.urls] +Repository = "https://github.com/iomarmochtar/sandock" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[project.optional-dependencies] +yml-config = [ "pyyaml (>=6.0.2,<7.0.0)" ] + +[tool.poetry.group.dev] +optional = true + +[tool.poetry.group.dev.dependencies] +black = "^25.1.0" +mypy = "^1.15.0" +types-pyyaml = "^6.0.12.20250402" +coverage = "^7.8.0" +flake8 = "^7.2.0" +flake8-pyproject = "^1.2.3" +pyyaml = "^6.0.2" + +[tool.black] +target-version = ['py39'] + +[tool.mypy] +python_version = "3.9" +mypy_path = "sandock" +exclude = "tests" +strict = true +pretty = true +show_error_codes = true +show_column_numbers = true + +[tool.flake8] +max-line-length = 88 +extend-ignore = ["E501", "E252"] +exclude = ["build", "dist", "__pycache__"] \ No newline at end of file diff --git a/sandock/cli.py b/sandock/cli.py index d611c8e..3f8afd8 100644 --- a/sandock/cli.py +++ b/sandock/cli.py @@ -2,17 +2,26 @@ import sys import logging import subprocess -from typing import List, Tuple, Dict, Any, Optional +import importlib +from typing import List, Tuple, Dict, Any, Optional, Type from argparse import ArgumentParser, Namespace, REMAINDER, ArgumentTypeError from importlib.metadata import metadata from .config import MainConfig, load_config_file, main_config_finder -from .shared import log, SANDBOX_DEBUG_ENV, CONFIG_PATH_ENV, run_shell +from .config.program import Program +from .shared import log, SANDBOX_DEBUG_ENV, CONFIG_PATH_ENV, KV, run_shell from .sandbox import SandboxExec from .volume import VolumeMgr from .exceptions import SandboxBaseException, SandboxExecConfig, SandboxVolumeExec from ._version import __version__, __build_hash__ +def import_sandbox_dynamic_class(full_class_path: str) -> Type[SandboxExec]: + module_path, class_name = full_class_path.rsplit(".", 1) + module = importlib.import_module(module_path) + + return getattr(module, class_name) # type: ignore[no-any-return] + + def parse_arg_key_value(s: str) -> Tuple[str, str]: if "=" not in s: raise ArgumentTypeError(f"Invalid format: '{s}', expected KEY=VALUE") @@ -304,6 +313,27 @@ def main(self) -> None: class CmdRun(BaseCommand): description = "run program" + program_cfg: Program + + def __init__(self, *args, **kwargs) -> None: # type: ignore[no-untyped-def] + super().__init__(*args, **kwargs) + program_cfg = self.config.programs.get(self.args.program) + if not program_cfg: + raise SandboxExecConfig(f"`{self.args.program}` is not defined") + + self.program_cfg = program_cfg + + @staticmethod + def register_arguments(parser: ArgumentParser) -> None: + parser.add_argument( + "program", + ) + + parser.add_argument( + "program_args", + nargs=REMAINDER, + help="arguments that will be forwarded, excluded for the override args", + ) @property def overrides_args(self) -> ArgumentParser: @@ -352,6 +382,7 @@ def overrides_args(self) -> ArgumentParser: self.override_arg(name="recreate-img"), action="store_true", default=False, + dest="hook_recreate_img", help="recreate the used container image", ) @@ -364,18 +395,6 @@ def overrides_args(self) -> ArgumentParser: return oparser - @staticmethod - def register_arguments(parser: ArgumentParser) -> None: - parser.add_argument( - "program", - ) - - parser.add_argument( - "program_args", - nargs=REMAINDER, - help="arguments that will be forwarded, excluded for the override args", - ) - def override_properties(self, args: List[str]) -> Dict[str, Any]: """ convert the override argument to Program's property @@ -401,13 +420,14 @@ def override_properties(self, args: List[str]) -> Dict[str, Any]: result[arg_name] = v return result - @property - def remainder_args(self) -> Tuple[List[str], Dict[str, str]]: + def apply_overrides(self) -> Tuple[List[str], KV]: """ capture argument that will be forwarded to program and read for sandbox-exec """ program_args = [] + hooks = {} snbx_args = [] + overrides = {} for remainder in self.args.program_args: if remainder.startswith(self.override_arg()): snbx_args.append(remainder) @@ -415,12 +435,54 @@ def remainder_args(self) -> Tuple[List[str], Dict[str, str]]: program_args.append(remainder) - return (program_args, self.override_properties(args=snbx_args)) + for k, v in self.override_properties(args=snbx_args).items(): + if k.startswith("hook_"): + hooks[k] = v + continue + + if hasattr(self.program_cfg, k): + log.debug(f"overriding value {v} in property {k}") + setattr(self.program_cfg, k, v) + overrides[k] = v + continue + + # persist container cannot be renamed in preventing unexpected behaviour + # (eg: need gc the stopped one with different name) + if self.program_cfg.persist.enable and "name" in overrides: + raise SandboxExecConfig("name of persist program cannot be overrided") + + return program_args, hooks + + @property + def executor_cls(self) -> Type[SandboxExec]: + """ + return sandbox class that will be use + """ + program_exec = self.program_cfg.executor + if not program_exec: + return SandboxExec + + executor = self.config.executors.get(program_exec) + if not executor: + raise SandboxExecConfig(f"unknown executor `{program_exec}` in {self.args.program}'s config") + + if not executor.load_cls: + return SandboxExec + + log.debug(f"using custom sandbox exec class ~> {executor.load_cls}") + return import_sandbox_dynamic_class(full_class_path=executor.load_cls) def main(self) -> None: - program_args, overrides = self.remainder_args - log.debug(f"overrides args ~> {overrides}") - snbx = SandboxExec(name=self.args.program, cfg=self.config, overrides=overrides) + # apply the program configuration overrides, the rest of it will be the arguments to + # container executeable + program_args, hooks = self.apply_overrides() + + snbx = self.executor_cls(name=self.args.program, program=self.program_cfg, cfg=self.config) + # run hooks if any + for hook, v in hooks.items(): + getattr(snbx, hook)(v) + + # forward any arguments to the container executeable snbx.do(args=program_args) @@ -434,7 +496,7 @@ def reraise_if_debug(e: Exception) -> None: def main(args: Optional[List[str]] = None) -> None: meta = metadata("sandock") - cmds = dict(list=CmdList, alias=CmdAlias, run=CmdRun, volume=CmdVolume) + cmds: Dict[str, Type[BaseCommand]] = dict(list=CmdList, alias=CmdAlias, run=CmdRun, volume=CmdVolume) parser = ArgumentParser( description="A wrapper in running command inside container sandboxed environment", epilog=f"Author: {meta['author']} <{meta['author-email']}>", diff --git a/sandock/config/__init__.py b/sandock/config/__init__.py index 4c19d83..4d22190 100644 --- a/sandock/config/__init__.py +++ b/sandock/config/__init__.py @@ -1,8 +1,9 @@ import os +import re from dataclasses import dataclass, field -from typing import Dict, Optional +from typing import Dict, Optional, Any from pathlib import Path -from ..shared import dict_merge, log, KV, CONFIG_PATH_ENV +from ..shared import dict_merge, log, fetch_prop, flatten_list, KV, CONFIG_PATH_ENV, FETCH_PROP_ENABLE_ENV from ..exceptions import SandboxExecConfig from .image import ImageBuild from .program import Program @@ -10,10 +11,19 @@ from .backup import Backup from ._helpers import read_config, build_if_set, dot_config_finder +DEFAULT_CUSTOM_EXECUTORS = dict( + apple_container=dict( + bin_path="container", + load_cls="sandock.executors.AppleContainerExec" + ) +) + +fetch_prop_re = re.compile(r"^fetch_prop\(([^)]+)\)$") + @dataclass class Volume(object): - driver: str = "local" + driver: Optional[str] = None driver_opts: Dict[str, str] = field(default_factory=dict) labels: Dict[str, str] = field(default_factory=dict) @@ -33,6 +43,16 @@ class Execution(object): alias_program_prefix: str = "" +@dataclass +class Executor(object): + bin_path: Optional[str] = None + load_cls: Optional[str] = None + + def __post_init__(self) -> None: + if self.bin_path is None and self.load_cls is None: + raise ValueError("one of `bin_path` or `load_cls` must be set on executor") + + @dataclass class MainConfig(object): execution: Execution = field(default_factory=Execution) @@ -42,6 +62,7 @@ class MainConfig(object): volumes: Dict[str, Volume] = field(default_factory=dict) images: Dict[str, ImageBuild] = field(default_factory=dict) networks: Dict[str, Network] = field(default_factory=dict) + executors: Dict[str, Executor] = field(default_factory=dict) def __post_init__(self) -> None: build_if_set(self, attr="config", cls=Configuration) @@ -50,9 +71,11 @@ def __post_init__(self) -> None: build_if_set(self, attr="execution", cls=Execution) build_if_set(self, attr="backup", cls=Backup) + self.executors = dict_merge(self.executors, DEFAULT_CUSTOM_EXECUTORS) # configuration that use kv format if the value set as dict cls_mapper = dict( - programs=Program, volumes=Volume, networks=Network, images=ImageBuild + programs=Program, volumes=Volume, networks=Network, images=ImageBuild, + executors=Executor ) for name, prop_cls in cls_mapper.items(): @@ -94,6 +117,43 @@ def __post_init__(self) -> None: if not self.programs: raise ValueError("no program configured") + if self.fetch_prop_enable: + self.resolve_fetch_prop(target=self) + + @property + def fetch_prop_enable(self) -> bool: + return os.getenv(FETCH_PROP_ENABLE_ENV) == "yes" + + def resolve_fetch_prop(self, target: Any=None) -> Any: + if isinstance(target, dict): + for k, v in target.items(): + target[k] = self.resolve_fetch_prop(v) + return target + + elif isinstance(target, list): + return [self.resolve_fetch_prop(i) for i in target] + + elif isinstance(target, str): + match = fetch_prop_re.match(target.strip()) + if match: + path = match.group(1).strip() + return fetch_prop(path=path, obj=self) + return target + + elif hasattr(target, "__dict__"): + for attr, val in vars(target).items(): + if not val: + continue + + val = self.resolve_fetch_prop(val) + if isinstance(val, list): + val = flatten_list(items=val) + + setattr(target, attr, val) + return target + + return target + def load_config_file(path: str) -> MainConfig: """ diff --git a/sandock/config/program.py b/sandock/config/program.py index 650f910..29e8531 100644 --- a/sandock/config/program.py +++ b/sandock/config/program.py @@ -43,6 +43,7 @@ class Program(object): interactive: bool = True allow_home_dir: bool = False name: Optional[str] = None + executor: Optional[str] = None network: Optional[str] = None hostname: Optional[str] = None build: Optional[ImageBuild] = None diff --git a/sandock/executors/__init__.py b/sandock/executors/__init__.py new file mode 100644 index 0000000..7c3d563 --- /dev/null +++ b/sandock/executors/__init__.py @@ -0,0 +1,4 @@ +from .apple_container import AppleContainerExec + + +__all__ = ["AppleContainerExec"] diff --git a/sandock/executors/apple_container.py b/sandock/executors/apple_container.py new file mode 100644 index 0000000..c491574 --- /dev/null +++ b/sandock/executors/apple_container.py @@ -0,0 +1,25 @@ +from typing import List +from ..shared import KV, list_remove_element +from ..sandbox import SandboxExec, CONTAINER_STATE_RUNNING + + +class AppleContainerExec(SandboxExec): + @property + def docker_bin(self) -> str: + return "container" + + def run_container_cmd(self) -> List[str]: + """ + some parameters are run supported + """ + cmds = super().run_container_cmd() + return list_remove_element(source=cmds, elem="--hostname") + + def inspect_container_cmd(self) -> str: + return f"{self.docker_bin} inspect {self.container_name}" + + def _check_running_container(self, container_info: List[KV]) -> bool: + return container_info[0].get("status") == CONTAINER_STATE_RUNNING + + def container_start_cmd(self) -> str: + return f"{self.docker_bin} start {self.container_name}" diff --git a/sandock/sandbox.py b/sandock/sandbox.py index 649fbb2..f6ee2bc 100644 --- a/sandock/sandbox.py +++ b/sandock/sandbox.py @@ -3,12 +3,12 @@ import tempfile import re from datetime import datetime -from typing import List, Dict, Any, Optional, Callable, Tuple +from typing import List, Optional from pathlib import Path from .config import MainConfig from .config.program import Program from .config.image import ImageBuild, DEFAULT_DUMP_IMAGE_STORE -from .shared import log, run_shell, file_hash, ensure_home_dir_special_prefix +from .shared import log, run_shell, file_hash, ensure_home_dir_special_prefix, KV from .exceptions import SandboxExecution VOL_LABEL_CREATED_BY = "created_by.sandock" @@ -19,42 +19,16 @@ class SandboxExec(object): + name: str cfg: MainConfig - program_name: str program: Program - interactive: bool container_name: str def __init__( - self, name: str, cfg: MainConfig, overrides: Dict[str, Any] = {} + self, name: str, program: Program, cfg: MainConfig ) -> None: - self.cfg = cfg self.name = name - program = self.cfg.programs.get(name) - if not program: - raise SandboxExecution(f"`{name}` is not defined") - - # persist container cannot be renamed in preventing unexpected behaviour - # (eg: need gc the stopped one with different name) - if program.persist.enable and "name" in overrides: - raise SandboxExecution("name of persist program cannot be overrided") - - hooks: List[Tuple[Callable[...], Any]] = [] # type: ignore[misc] - # apply program's attribute overrides - for k, v in overrides.items(): - if not hasattr(program, k): - # it might be an internal/hook method - method = f"hook_{k}" - if hasattr(self, method): - log.debug(f"hook detected for method {method}") - hooks.append((getattr(self, method), v)) - continue - - log.warning(f"program doesn't has property {k}") - continue - - log.debug(f"overriding value {v} in property {k}") - setattr(program, k, v) + self.cfg = cfg self.program = program # prevent if it's run on homedir, we don't want unintended breach except their aware @@ -68,15 +42,12 @@ def __init__( ) self.container_name = self.generate_container_name() - # run hooks if any - for method_hook, arg in hooks: - method_hook(arg) - def hook_recreate_img(self, create: bool=False) -> None: + def hook_recreate_img(self, execute: bool=False) -> None: """ register for pre-exec cmd to delete image run the related container """ - if not create: + if not execute: return log.debug("[hook] registring for image deletion: {self.program.image}") @@ -86,7 +57,16 @@ def hook_recreate_img(self, create: bool=False) -> None: @property def docker_bin(self) -> str: - return self.cfg.execution.docker_bin + default_bin = self.cfg.execution.docker_bin + custom_executor = self.program.executor + if not custom_executor: + return default_bin + + executor = self.cfg.executors.get(custom_executor) + if not executor: + raise SandboxExecution(f"Executor `{custom_executor}` is not defined") + + return executor.bin_path if executor.bin_path else default_bin @property def current_timestamp(self) -> float: @@ -120,7 +100,7 @@ def exec_path(self) -> str: def run_container_cmd(self) -> List[str]: """ - docker run command builder + container run command builder """ command = [ self.docker_bin, @@ -217,13 +197,18 @@ def ensure_volume(self, name: str) -> None: if VOL_LABEL_CREATED_BY not in vol.labels: vol.labels[VOL_LABEL_CREATED_BY] = "true" - vol_ops = " ".join([f"--opt {k}={v}" for k, v in vol.driver_opts.items()]) - vol_labels = " ".join([f"--label {k}='{v}'" for k, v in vol.labels.items()]) - vol_create_cmd = ( - f"{self.docker_bin} volume create " - f"--driver={vol.driver} {vol_ops} {vol_labels}" - f" {name}" - ) + vol_create_cmd = [f"{self.docker_bin} volume create"] + + if vol.driver: + vol_create_cmd.append(f"--driver={vol.driver}") + + if vol.driver_opts: + vol_create_cmd.append(" ".join([f"--opt {k}={v}" for k, v in vol.driver_opts.items()])) + + if vol.labels: + vol_create_cmd.append(" ".join([f"--label {k}='{v}'" for k, v in vol.labels.items()])) + + vol_create_cmd.append(name) run_shell(vol_create_cmd, capture_output=False) def ensure_network(self) -> None: @@ -388,6 +373,15 @@ def ensure_custom_image(self, image_name: Optional[str] = None) -> None: ): os.remove(docker_file_path) + def inspect_container_cmd(self) -> str: + return f"{self.docker_bin} container inspect {self.container_name}" + + def container_start_cmd(self) -> str: + return f"{self.docker_bin} container start {self.container_name}" + + def _check_running_container(self, container_info: List[KV]) -> bool: + return bool(container_info[0].get("State", {}).get("Status") == CONTAINER_STATE_RUNNING) + @property def attach_container(self) -> bool: """ @@ -396,8 +390,7 @@ def attach_container(self) -> bool: if not self.program.persist.enable: return False - cmd = f"{self.docker_bin} container inspect {self.container_name}" - inspect_result = run_shell(cmd, check_err=False) + inspect_result = run_shell(self.inspect_container_cmd(), check_err=False) if inspect_result.returncode != 0: # it might be the uncreated container err_msg = str(inspect_result.stderr) @@ -414,13 +407,11 @@ def attach_container(self) -> bool: ) return False - is_running: bool = ( - container_info[0].get("State", {}).get("Status") == CONTAINER_STATE_RUNNING - ) + is_running = self._check_running_container(container_info) if not is_running and self.program.persist.auto_start: log.warning("persist container is down, starting container") run_shell( - f"{self.docker_bin} container start {self.container_name}", + self.container_start_cmd(), capture_output=False, ) @@ -461,4 +452,4 @@ def do(self, args: List[str] = []) -> None: run_container_cmd.extend(args) log.debug(f"starting container cmd ~> {run_container_cmd}") - run_shell(command=" ".join(run_container_cmd), capture_output=False) + run_shell(run_container_cmd, capture_output=False) diff --git a/sandock/shared.py b/sandock/shared.py index 8a5843a..1f13ddf 100644 --- a/sandock/shared.py +++ b/sandock/shared.py @@ -9,6 +9,7 @@ KV = Dict[str, Any] CONFIG_PATH_ENV = "SNDK_CFG" SANDBOX_DEBUG_ENV = "SNDK_DEBUG" +FETCH_PROP_ENABLE_ENV = "SNDK_FETCH_PROP" class LogColorFormatter(logging.Formatter): @@ -67,7 +68,8 @@ def run_shell( # type: ignore[no-untyped-def] | cmd_args ) if isinstance(command, list): - command = " ".join(command) + # filter for empty elem, this might from the result of inline if-else + command = " ".join([x for x in command if x]) log.debug(f"shell cmd: {command}, check_err: {check_err}, cmd_args: {cmd_args}") call_cmd = subprocess.run(command, **cmd_args) @@ -84,6 +86,19 @@ def run_shell( # type: ignore[no-untyped-def] return call_cmd +def list_remove_element(source: List[str], elem: str) -> List[str]: + """ + remove the related member and the next to it + """ + # return as is if not found + try: + elem_idx = source.index(elem) + except ValueError: + return source + + return source[:elem_idx] + source[elem_idx+2:] + + def dict_merge(dict1: KV, dict2: KV) -> KV: """ deep merge between dict1 and dicgt2 @@ -117,3 +132,44 @@ def file_hash(fpath: str, max_chars: Optional[int] = None) -> str: hex_digest = sha256(fh.read().encode("utf-8")).hexdigest() return hex_digest[:max_chars] if max_chars is not None else hex_digest + + +def flatten_list(items: List[Any]) -> List[Any]: + """ + Recursively flatten a nested list into a single list. + """ + result = [] + for item in items: + if isinstance(item, list): + result.extend(flatten_list(item)) + else: + result.append(item) + return result + + +def fetch_prop(path: str, obj: Union[object, List[Any], KV], separator: str=".") -> Any: + """ + Fetch a nested property from an object or dictionary using the given separator char. + Supports dicts, lists, and normal Python objects. + """ + keys = path.split(separator) + current = obj + + for key in keys: + if isinstance(current, dict): + if key not in current: + raise KeyError(f"Key `{key}` not found in dict at `{path}`") + current = current[key] + elif isinstance(current, list): + try: + index = int(key) + current = current[index] + except (ValueError, IndexError): + raise KeyError(f"Invalid list index `{key}` in path `{path}`") + else: + # Handle Python object + if not hasattr(current, key): + raise KeyError(f"Attribute `{key}` not found in object at `{path}`") + current = getattr(current, key) + + return current diff --git a/tests/helpers.py b/tests/helpers.py index a51aac7..6275b9a 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -7,7 +7,7 @@ from unittest import mock from contextlib import contextmanager from typing import Iterator, List -from sandock.shared import log, SANDBOX_DEBUG_ENV, CONFIG_PATH_ENV, KV +from sandock.shared import log, SANDBOX_DEBUG_ENV, CONFIG_PATH_ENV, KV, FETCH_PROP_ENABLE_ENV from sandock.config import MainConfig, Program @@ -77,5 +77,6 @@ class BaseTestCase(unittest.TestCase): def setUp(self) -> None: # cleanup any env that used in app logic - for env_var in [SANDBOX_DEBUG_ENV, CONFIG_PATH_ENV]: + known_envs = [SANDBOX_DEBUG_ENV, CONFIG_PATH_ENV, FETCH_PROP_ENABLE_ENV] + for env_var in known_envs: os.environ.pop(env_var, None) diff --git a/tests/test_cli.py b/tests/test_cli.py index a6e805f..5097a39 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -32,6 +32,8 @@ CmdVolume, main as cli_main, ) +from sandock.sandbox import SandboxExec +from sandock.executors import AppleContainerExec from sandock.volume import VolumeMgr, BackupSnapshot from helpers import mock_shell_exec @@ -170,8 +172,86 @@ def test_main(self) -> None: class CmdRunTest(SkeltonCmdTest): cls = CmdRun + @contextmanager + def obj(self, ns_props: Optional[dict]={}, **kwargs) -> Iterator[CmdRun]: + kwargs.setdefault("args", Namespace(program="pydev", **ns_props)) + + with super().obj(**kwargs) as o: + yield o + + def test_executor_cls_default(self) -> None: + with self.obj() as o: + self.assertEqual(o.executor_cls, SandboxExec) + + def test_executor_cls_unknown_executor(self) -> None: + with self.obj( + cfg=dummy_main_cfg(program_kwargs={"executor": "special_exec"}), + ) as o: + with self.assertRaisesRegex(SandboxExecConfig, "unknown executor `special_exec` in pydev's config"): + o.executor_cls + + def test_executor_cls_not_mentioned_custom_cls(self) -> None: + """ + it's registered on configuration but not specified for custom loader class + """ + with self.obj( + cfg=dummy_main_cfg( + program_kwargs={"executor": "special_exec"}, + executors={"special_exec": {"bin_path": "/usr/bin/special_exec"}} + ), + ) as o: + self.assertEqual(o.executor_cls, SandboxExec) + + def test_executor_cls_custom_cls(self) -> None: + with self.obj( + cfg=dummy_main_cfg( + program_kwargs={"executor": "buah_apple"}, + executors={"buah_apple": {"load_cls": "sandock.executors.AppleContainerExec"}} + ), + ) as o: + self.assertEqual(o.executor_cls, AppleContainerExec) + + # TODO: [x] validation: program is not defined + def test_init_program_not_defined(self) -> None: + """ + the provided program is not listed on configuration + """ + with self.assertRaisesRegex(SandboxExecConfig, "`another_app` is not defined"): + with self.obj(args=Namespace(program="another_app")): + pass + + # TODO: [x] validation: name of persist command canot be overrided + def test_apply_overrides_persist_cannot_overrided(self) -> None: + with self.assertRaisesRegex( + SandboxExecConfig, "name of persist program cannot be overrided" + ): + with self.obj( + ns_props=dict(program_args=["--sandbox-arg-name=rubydev"]), + cfg=dummy_main_cfg(program_kwargs={"persist": {"enable": True}}), + ) as o: + o.apply_overrides() + + + # TODO: [x] running hook test (recreate_img) + @mock.patch("sandock.cli.SandboxExec") + def test_main_hook_recreate_img(self, sandbox_exec_mock: MagicMock) -> None: + remote = MagicMock() + sandbox_exec_mock.return_value = remote + + provided_args = ["--sandbox-arg-recreate-img"] + with self.obj( + ns_props=dict(program_args=provided_args), + ) as o: + o.main() + + remote.hook_recreate_img.assert_called_once_with(True) + remote.do.assert_called_once_with(args=[]) + @mock.patch("sandock.cli.SandboxExec") - def test_main(self, sandbox_exec_mock: MagicMock) -> None: + def test_main_overrided_params(self, sandbox_exec_mock: MagicMock) -> None: + """ + overrding program parameters + """ remote = MagicMock() sandbox_exec_mock.return_value = remote @@ -181,29 +261,27 @@ def test_main(self, sandbox_exec_mock: MagicMock) -> None: "--sandbox-arg-ports=8081:8081", "--version"] with self.obj( - args=Namespace(program="pydev", program_args=provided_args), + ns_props=dict(program_args=provided_args), ) as o: o.main() - self.assertDictEqual( - sandbox_exec_mock.call_args[1]["overrides"], - dict( + # TODO: [x] testing override parameters + self.assertEqual( + sandbox_exec_mock.call_args[1]["program"], + dummy_program_cfg( hostname="change_host", - allow_home_dir=False, - recreate_img=False, - ports=["8080:8080", "8081:8081"]), - ) - remote.do.assert_called_once() - self.assertListEqual( - remote.do.call_args[1]["args"], - ["--version"], - msg="the forwarded argument to container's program", + ports=[ + "8080:8080", + "8081:8081" + ] + ), ) + remote.hook_recreate_img.assert_called_once_with(False) + remote.do.assert_called_once_with(args=["--version"]) + def test_overrides_properties_kv(self) -> None: - with self.obj( - args=Namespace(program="pydev"), - ) as o: + with self.obj() as o: result = o.override_properties( args=[ "--sandbox-arg-env=DEBUG=true", @@ -213,13 +291,10 @@ def test_overrides_properties_kv(self) -> None: ) expected_env = dict(DEBUG="true", APP_ENV="dev") - ov_props = dict(allow_home_dir=False, recreate_img=True, env=expected_env) + ov_props = dict(allow_home_dir=False, hook_recreate_img=True, env=expected_env) self.assertDictEqual(result, ov_props) - with self.obj( - args=Namespace(program="pydev"), - ) as o: - + with self.obj() as o: # wrongly formatted key value provided with mock.patch("sys.exit") as sys_exit: result = o.override_properties(args=["--sandbox-arg-env=NOVALUE"]) @@ -232,9 +307,7 @@ def test_overrides_properties_print_help( self, argparse_print_help: MagicMock, sys_exit: MagicMock ) -> None: # print help if provided with sandbox arg help arams - with self.obj( - args=Namespace(program="pydev"), - ) as o: + with self.obj() as o: o.override_properties(args=["--sandbox-arg-help"]) argparse_print_help.assert_called_once() diff --git a/tests/test_config.py b/tests/test_config.py index b6710c0..945359d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -15,6 +15,7 @@ Network, Configuration, Execution, + Executor, MainConfig, load_config_file, main_config_finder, @@ -28,7 +29,7 @@ class VolumeTest(BaseTestCase): def test_defaults(self) -> None: o = Volume() - self.assertEqual(o.driver, "local") + self.assertEqual(o.driver, None) self.assertEqual(o.driver_opts, {}) self.assertEqual(o.labels, {}) @@ -56,6 +57,16 @@ def test_validations(self) -> None: ): ImageBuild(dockerFile="Dockerfile", dockerfile_inline="FROM ubuntu:22.04") +class ExecutorTest(BaseTestCase): + def test_must_set(self) -> None: + with self.assertRaisesRegex( + ValueError, "one of `bin_path` or `load_cls` must be set on executor" + ): + Executor() + + def test_set_value(self) -> None: + self.assertEqual(Executor(bin_path="container").bin_path, "container") + self.assertEqual(Executor(load_cls="lib.SomeClass").load_cls, "lib.SomeClass") class ContainerUserTest(BaseTestCase): def test_defaults(self) -> None: @@ -205,6 +216,7 @@ def test_defaults(self) -> None: self.assertTrue(o.interactive, True) self.assertFalse(o.allow_home_dir) self.assertIsNone(o.name) + self.assertIsNone(o.executor) self.assertIsNone(o.network) self.assertIsNone(o.hostname) self.assertIsNone(o.user) @@ -383,6 +395,75 @@ def test_networks(self) -> None: ), ) + @mock.patch.dict(os.environ, dict(SNDK_FETCH_PROP="yes")) + def test_resolve_fetch_prop_enable(self) -> None: + o = self.obj( + programs=dict( + p1=Program( + image="python:3.14", + exec="python3", + volumes=[ + "./:/opt/mount1", + "another:/opt/another", + ] + ), + p2=Program( + image="python:3.11", + exec="python3", + volumes=[ + "top:/top", + "fetch_prop(programs.p1.volumes)", + "above:/above", + "fetch_prop(programs.p1.volumes.0)", + ] + ) + ) + ) + + self.assertListEqual( + o.programs["p2"].volumes, + [ + "top:/top", + "./:/opt/mount1", + "another:/opt/another", + "above:/above", + "./:/opt/mount1" + ] + ) + + + def test_resolve_fetch_prop_disable(self) -> None: + o = self.obj( + programs=dict( + p1=Program( + image="python:3.14", + exec="python3", + volumes=[ + "./:/opt/mount1", + "another:/opt/another", + ] + ), + p2=Program( + image="python:3.11", + exec="python3", + volumes=[ + "top:/top", + "fetch_prop(programs.p1.volumes)", + "above:/above", + ] + ) + ) + ) + + self.assertListEqual( + o.programs["p2"].volumes, + [ + "top:/top", + "fetch_prop(programs.p1.volumes)", + "above:/above", + ] + ) + class HelpersTest(BaseTestCase): def test_load_config_file(self) -> None: diff --git a/tests/test_executors_apple_container.py b/tests/test_executors_apple_container.py new file mode 100644 index 0000000..49fdcf0 --- /dev/null +++ b/tests/test_executors_apple_container.py @@ -0,0 +1,194 @@ + +from unittest import mock +from sandock.executors import AppleContainerExec +from sandock.exceptions import SandboxExecution +from test_sandbox import SandboxExecTest +from helpers import ( + dummy_main_cfg, + mock_shell_exec +) + +# majority of the behaviours are same +class AppleContainerExecTest(SandboxExecTest): + default_executor: str = "container" + exec_cls: object = AppleContainerExec + + + def test_attach_container_status_start(self) -> None: + + with mock_shell_exec( + side_effects=[ + dict(returncode=0, stdout='[{"status": "running"}]') + ] + ) as rs: # inspect container + o = self.obj(program_kwargs=dict(persist=dict(enable=True), name="pydev")) + + self.assertTrue(o.attach_container) + self.assertEqual(rs.call_count, 1) + self.assertEqual( + rs.call_args_list[0].args[0], + "container inspect pydev", + ) + + def test_attach_container_status_stop_auto_start(self) -> None: + + side_effects = [ + dict( + returncode=0, stdout='[{"status": "stopped"}]' + ), # inspect container + dict(returncode=0), # start container + ] + with mock_shell_exec(side_effects=side_effects) as rs: + o = self.obj(program_kwargs=dict(persist=dict(enable=True), name="pydev")) + + self.assertTrue(o.attach_container) + self.assertEqual(rs.call_count, 2) + self.assertEqual( + rs.call_args_list[0].args[0], + f"{self.default_executor} inspect pydev", + ) + + self.assertEqual( + rs.call_args_list[1].args[0], + f"{self.default_executor} start pydev", + ) + + # auto start disable + with mock_shell_exec( + side_effects=[ + dict(returncode=0, stdout='[{"status": "stopped"}]') + ] + ) as rs: # inspect container + o = self.obj( + program_kwargs=dict( + persist=dict(enable=True, auto_start=False), name="pydev" + ) + ) + + self.assertFalse(o.attach_container) + self.assertEqual(rs.call_count, 1) + self.assertEqual( + rs.call_args_list[0].args[0], + f"{self.default_executor} inspect pydev", + ) + + def test_attach_container(self) -> None: + with mock_shell_exec() as rs: + o = self.obj() + self.assertFalse( + o.attach_container, msg="non persist program will not attach" + ) + + rs.assert_not_called() + + # apple's container return empty list json when container not exists + with mock_shell_exec( + side_effects=[dict(returncode=0, stdout="[]")] + ) as rs: + o = self.obj(program_kwargs=dict(persist=dict(enable=True), name="pydev")) + + self.assertFalse( + o.attach_container, msg="non persist program will not attach" + ) + + self.assertEqual(rs.call_count, 1) + self.assertEqual( + rs.call_args_list[0].args[0], + f"{self.default_executor} inspect pydev", + ) + + with mock_shell_exec( + side_effects=[dict(returncode=1, stderr="unexpected error")] + ) as rs: + o = self.obj(program_kwargs=dict(persist=dict(enable=True), name="pydev")) + + with self.assertRaisesRegex( + SandboxExecution, + "error during check container status: unexpected error", + ): + o.attach_container + + def test_attach_container_empy_info(self) -> None: + + with mock_shell_exec( + side_effects=[dict(returncode=0, stdout="[]")] + ) as rs: # inspect container + o = self.obj(program_kwargs=dict(persist=dict(enable=True), name="pydev")) + + self.assertFalse(o.attach_container) + self.assertEqual(rs.call_count, 1) + self.assertEqual( + rs.call_args_list[0].args[0], + f"{self.default_executor} inspect pydev", + ) + + def test_run_container_cmd_extended(self) -> None: + cfg = dummy_main_cfg( + program_kwargs=dict( + name="mypydev", + persist=dict(enable=True), + platform="linux/amd64", + hostname="imah", + executor="apple_container", + network="host", + user=dict(keep_id=True), + aliases=dict(sh="/bin/bash"), + sandbox_mount=dict(read_only=True), + volumes=[ + "output_${VOL_DIR}:/output", + "~/share_to_container:/shared:ro", + ], + env=dict(PYTHON_PATH="/shared"), + extra_run_args=["--env-file=~/common_container_envs"], + ), + execution=dict(docker_bin="podman"), + ) + + with mock.patch.multiple( + AppleContainerExec, + current_dir="/path/to/repo", + home_dir="/home/dir", + current_uid=1000, + current_gid=1000, + ): + o = self.obj(cfg=cfg) + + self.assertListEqual( + o.run_container_cmd(), + [ + "container", + "run", + "--entrypoint", + "python3", + "--name", + "mypydev", + "-it", + "--platform", + "linux/amd64", + "--network", + "host", + "-u", + "1000:1000", + "-v", + "output_path_to_repo:/output", + "-v", + "~/share_to_container:/shared:ro", + "-v", + "/path/to/repo:/sandbox:ro", + "--workdir", + "/sandbox", + "-e PYTHON_PATH='/shared'", + "--env-file=~/common_container_envs", + "python:3.11", + ], + ) + + def test_run_container_cmd_set_executor(self) -> None: + """ + skipped: no way to custom the binary path/name + """ + + def test_docker_bin_executor_not_defined(self) -> None: + """ + skipped: the bin_path is hard coded on the defined class + """ \ No newline at end of file diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index 053aa2b..8b9ddd7 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -43,29 +43,24 @@ def sample_cfg_program_dependent_images() -> MainConfig: class SandboxExecTest(BaseTestCase): + default_executor: str = "docker" + exec_cls: object = SandboxExec + def obj(self, program_kwargs: dict = {}, **kwargs) -> SandboxExec: """ automatically inject default values """ + cfg = kwargs.get("cfg", dummy_main_cfg(program_kwargs=program_kwargs)) kwargs = ( - dict(name="pydev", cfg=dummy_main_cfg(program_kwargs=program_kwargs)) + dict(name="pydev", program=cfg.programs["pydev"], cfg=cfg) | kwargs ) - return SandboxExec(**kwargs) + return self.exec_cls(**kwargs) + # TODO: move this to cli test + # TODO: [x] keep to validate directory execution check def test_init_validations(self) -> None: - with self.assertRaisesRegex(SandboxExecution, "`ruby33` is not defined"): - self.obj(name="ruby33") - - with self.assertRaisesRegex( - SandboxExecution, "name of persist program cannot be overrided" - ): - self.obj( - cfg=dummy_main_cfg(program_kwargs=dict(persist=dict(enable=True))), - overrides=dict(name="pydev_ov"), - ) - with self.assertRaisesRegex( SandboxExecution, "cannot be ran on top of home directory when the program's sandbox mount is enabled", @@ -76,14 +71,6 @@ def test_init_validations(self) -> None: ): self.obj() - def test_init_overrides(self) -> None: - o = self.obj( - overrides=dict(exec="/bin/bash", no_prop="not_found", name="conda") - ) - - self.assertEqual(o.program.exec, "/bin/bash") - self.assertEqual(o.program.name, "conda") - def test_generate_container_name(self) -> None: with mock.patch.object(SandboxExec, "current_timestamp", "123.456"): self.assertEqual( @@ -121,11 +108,42 @@ def test_run_container_cmd_defaults(self) -> None: current_timestamp="123.456", ): o = self.obj() + self.assertListEqual( + o.run_container_cmd(), + [ + self.default_executor, + "run", + "--entrypoint", + "python3", + "--name", + "sandock-pydev-123.456", + "--rm", + "-it", + "-v", + "/path/to/repo:/sandbox", + "--workdir", + "/sandbox", + "python:3.11", + ], + ) + + def test_run_container_cmd_set_executor(self) -> None: + with mock.patch.multiple( + SandboxExec, + current_dir="/path/to/repo", + home_dir="/home/dir", + current_timestamp="123.456", + ): + cfg = dummy_main_cfg( + program_kwargs={"executor": "podman_latest"}, + executors={"podman_latest": {"bin_path": "/opt/podman_latest/bin/podman"}} + ) + o = self.obj(cfg=cfg) self.assertListEqual( o.run_container_cmd(), [ - "docker", + "/opt/podman_latest/bin/podman", "run", "--entrypoint", "python3", @@ -141,6 +159,13 @@ def test_run_container_cmd_defaults(self) -> None: ], ) + def test_docker_bin_executor_not_defined(self) -> None: + cfg = dummy_main_cfg( + program_kwargs={"executor": "podman_latest"}, + ) + with self.assertRaisesRegex(SandboxExecution, "Executor `podman_latest` is not defined"): + self.obj(cfg=cfg).docker_bin + def test_run_container_cmd_extended(self) -> None: cfg = dummy_main_cfg( program_kwargs=dict( @@ -169,7 +194,7 @@ def test_run_container_cmd_extended(self) -> None: current_uid=1000, current_gid=1000, ): - o = self.obj(cfg=cfg, overrides=dict(exec="sh")) + o = self.obj(cfg=cfg) self.assertListEqual( o.run_container_cmd(), @@ -177,7 +202,7 @@ def test_run_container_cmd_extended(self) -> None: "podman", "run", "--entrypoint", - "/bin/bash", + "python3", "--name", "mypydev", "-it", @@ -229,7 +254,7 @@ def test_ensure_volume_exists(self) -> None: rs.assert_called() self.assertEqual(rs.call_count, 1) self.assertEqual( - rs.call_args_list[0].args[0], "docker volume inspect myhome" + rs.call_args_list[0].args[0], f"{self.default_executor} volume inspect myhome" ) def test_ensure_volume_auto_create(self) -> None: @@ -237,7 +262,11 @@ def test_ensure_volume_auto_create(self) -> None: mentioned but it's not exist, then auto create it """ cfg = dummy_main_cfg( - volumes=dict(myhome=dict(labels={"backup.container.mochtar.net": "true"})), + volumes=dict(myhome=dict( + driver="local", + driver_opts={"device": "tmpfs"}, + labels={"backup.container.mochtar.net": "true"}) + ), ) side_effects = [ @@ -251,11 +280,11 @@ def test_ensure_volume_auto_create(self) -> None: rs.assert_called() self.assertEqual(rs.call_count, 2) self.assertEqual( - rs.call_args_list[0].args[0], "docker volume inspect myhome" + rs.call_args_list[0].args[0], f"{self.default_executor} volume inspect myhome" ) self.assertEqual( rs.call_args_list[1].args[0], - "docker volume create --driver=local --label backup.container.mochtar.net='true' --label created_by.sandock='true' myhome", + f"{self.default_executor} volume create --driver=local --opt device=tmpfs --label backup.container.mochtar.net='true' --label created_by.sandock='true' myhome", ) def test_ensure_network_unmanaged(self) -> None: @@ -294,7 +323,7 @@ def test_ensure_network_exists(self) -> None: rs.assert_called() self.assertEqual(rs.call_count, 1) self.assertEqual( - rs.call_args_list[0].args[0], "docker network inspect mynet" + rs.call_args_list[0].args[0], f"{self.default_executor} network inspect mynet" ) def test_ensure_network_auto_create(self) -> None: @@ -316,11 +345,11 @@ def test_ensure_network_auto_create(self) -> None: rs.assert_called() self.assertEqual(rs.call_count, 2) self.assertEqual( - rs.call_args_list[0].args[0], "docker network inspect mynet" + rs.call_args_list[0].args[0], f"{self.default_executor} network inspect mynet" ) self.assertEqual( rs.call_args_list[1].args[0], - "docker network create --driver=bridge mynet", + f"{self.default_executor} network create --driver=bridge mynet", ) def test_ensure_custom_image_not_defined(self) -> None: @@ -344,7 +373,7 @@ def test_ensure_custom_image_exists(self) -> None: rs.assert_called() self.assertEqual(rs.call_count, 1) self.assertEqual( - rs.call_args_list[0].args[0], "docker image inspect custom_pydev" + rs.call_args_list[0].args[0], f"{self.default_executor} image inspect custom_pydev" ) @mock.patch.dict(os.environ, dict(HOME="/home/user1")) @@ -374,8 +403,8 @@ def test_ensure_custom_image_escape_homedir( self.assertListEqual( extract_first_call_arg_list(m=rs), [ - "docker image inspect pydev:base", - "docker build -t pydev:base -f /home/user1/path/to/Dockerfile /home/user1/path/to", + f"{self.default_executor} image inspect pydev:base", + f"{self.default_executor} build -t pydev:base -f /home/user1/path/to/Dockerfile /home/user1/path/to", ], ) @@ -404,10 +433,10 @@ def test_ensure_custom_image_auto_create(self) -> None: self.assertListEqual( extract_first_call_arg_list(m=rs), [ - "docker image inspect custom_pydev_base", - f'docker build -t custom_pydev_base -f {docker_file_temp} --build-arg USER="another" --platform=linux/amd64 /tmp/dst', - "docker image inspect custom_pydev", - f"docker build -t custom_pydev -f {docker_file_temp} --progress=quite --platform=linux/amd64 /tmp/dst", + f"{self.default_executor} image inspect custom_pydev_base", + f'{self.default_executor} build -t custom_pydev_base -f {docker_file_temp} --build-arg USER="another" --platform=linux/amd64 /tmp/dst', + f"{self.default_executor} image inspect custom_pydev", + f"{self.default_executor} build -t custom_pydev -f {docker_file_temp} --progress=quite --platform=linux/amd64 /tmp/dst", ], ) @@ -448,8 +477,8 @@ def test_ensure_custom_image_auto_create_dumped_exists( self.assertListEqual( extract_first_call_arg_list(m=rs), [ - "docker image inspect pydev:base", - "docker image load -i /path/to/dumped/image.tar", + f"{self.default_executor} image inspect pydev:base", + f"{self.default_executor} image load -i /path/to/dumped/image.tar", ], ) @@ -494,9 +523,9 @@ def test_ensure_custom_image_auto_create_dumped_not_exists( mktemp=mock.Mock(return_value=docker_file_temp), ): expected_executed_shell_cmds = [ - "docker image inspect pydev:base", - f"docker build -t pydev:base -f {docker_file_temp} --platform=linux/amd64 /tmp/dst", - "docker image save pydev:base --output /path/to/dumped/image.tar", + f"{self.default_executor} image inspect pydev:base", + f"{self.default_executor} build -t pydev:base -f {docker_file_temp} --platform=linux/amd64 /tmp/dst", + f"{self.default_executor} image save pydev:base --output /path/to/dumped/image.tar", ] # scenario: use the same file pattern @@ -566,7 +595,7 @@ def test_attach_container(self) -> None: self.assertEqual(rs.call_count, 1) self.assertEqual( rs.call_args_list[0].args[0], - "docker container inspect pydev", + f"{self.default_executor} container inspect pydev", ) with mock_shell_exec( @@ -591,7 +620,7 @@ def test_attach_container_empy_info(self) -> None: self.assertEqual(rs.call_count, 1) self.assertEqual( rs.call_args_list[0].args[0], - "docker container inspect pydev", + f"{self.default_executor} container inspect pydev", ) def test_attach_container_status_stop_auto_start(self) -> None: @@ -609,12 +638,12 @@ def test_attach_container_status_stop_auto_start(self) -> None: self.assertEqual(rs.call_count, 2) self.assertEqual( rs.call_args_list[0].args[0], - "docker container inspect pydev", + f"{self.default_executor} container inspect pydev", ) self.assertEqual( rs.call_args_list[1].args[0], - "docker container start pydev", + f"{self.default_executor} container start pydev", ) # auto start disable @@ -633,7 +662,7 @@ def test_attach_container_status_stop_auto_start(self) -> None: self.assertEqual(rs.call_count, 1) self.assertEqual( rs.call_args_list[0].args[0], - "docker container inspect pydev", + f"{self.default_executor} container inspect pydev", ) def test_attach_container_status_start(self) -> None: @@ -655,7 +684,7 @@ def test_attach_container_status_start(self) -> None: def test_exec_container_cmd(self) -> None: self.assertListEqual( self.obj(program_kwargs=dict(name="pydev")).exec_container_cmd(), - ["docker", "exec", "-it", "pydev", "python3"], + [self.default_executor, "exec", "-it", "pydev", "python3"], ) def test_do_docker_run(self) -> None: @@ -688,7 +717,7 @@ def test_do_docker_run(self) -> None: ), ): - o = self.obj(cfg=cfg, overrides=dict(recreate_img=False)) + o = self.obj(cfg=cfg) o.do(args=["--version"]) self.assertEqual(o.ensure_custom_image.call_count, 1) @@ -708,7 +737,7 @@ def test_do_docker_run(self) -> None: def test_do_hook_recreate_img(self) -> None: """ - run recreate image as the first execute on pre-exec + register hook for image removal before running container command """ shell_side_effects = [ dict(returncode=0), # pre cmd from deleting image @@ -735,11 +764,12 @@ def test_do_hook_recreate_img(self) -> None: ensure_volume=mock.MagicMock(), exec_container_cmd=mock.MagicMock(), run_container_cmd=mock.MagicMock( - return_value=["docker", "run", "bla", "bla"] + return_value=[self.default_executor, "run", "bla", "bla"] ), ): - o = self.obj(cfg=cfg, overrides=dict(recreate_img=True)) + o = self.obj(cfg=cfg) + o.hook_recreate_img(execute=True) o.do(args=["--version"]) self.assertEqual(o.ensure_custom_image.call_count, 1) @@ -750,12 +780,12 @@ def test_do_hook_recreate_img(self) -> None: # only for the shell command under "do" methods rs.assert_called() self.assertEqual(rs.call_count, 4) - self.assertEqual(rs.call_args_list[0].args[0], "docker image rm custom_python3") + self.assertEqual(rs.call_args_list[0].args[0], f"{self.default_executor} image rm custom_python3") self.assertEqual(rs.call_args_list[1].args[0], "whoami") self.assertEqual(rs.call_args_list[2].args[0], "cd /tmp") self.assertEqual( rs.call_args_list[3].args[0], - "docker run bla bla --version", + f"{self.default_executor} run bla bla --version", ) def test_do_docker_exec(self) -> None: @@ -776,14 +806,14 @@ def test_do_docker_exec(self) -> None: # mocks methods and properties with mock.patch.multiple( - SandboxExec, + self.exec_cls, current_dir="/path/to/repo", attach_container=True, ensure_custom_image=mock.MagicMock(), ensure_network=mock.MagicMock(), ensure_volume=mock.MagicMock(), exec_container_cmd=mock.MagicMock( - return_value=["docker", "exec", "bla", "bla"] + return_value=[self.default_executor, "exec", "bla", "bla"] ), run_container_cmd=mock.MagicMock(), ): @@ -803,7 +833,7 @@ def test_do_docker_exec(self) -> None: self.assertEqual(rs.call_args_list[1].args[0], "cd /tmp") self.assertEqual( rs.call_args_list[2].args[0], - "docker exec bla bla --version", + f"{self.default_executor} exec bla bla --version", ) if __name__ == "__main__": diff --git a/tests/test_shared.py b/tests/test_shared.py index 2930304..620587f 100644 --- a/tests/test_shared.py +++ b/tests/test_shared.py @@ -6,7 +6,7 @@ from inspect import cleandoc from subprocess import CalledProcessError from sandock import shared -from helpers import BaseTestCase +from helpers import dummy_main_cfg, BaseTestCase class RunShellTest(BaseTestCase): @@ -58,6 +58,92 @@ def test_ensure_home_dir_special_prefix(self) -> None: "/home/sweet_home/path/to.yml", ) +class UtilitiesTest(BaseTestCase): + sample_dict = { + "satu": { + "sub1_satu": "hello", + "sub2_satu": "word", + }, + "dua": [ + "elem1", + "elem2", + ] + } + + sample_obj = dummy_main_cfg(program_kwargs=dict( + volumes=[ + "temp1:/opt/temp1", + "temp2:/opt/temp2", + ] + )) + + def test_flatten_list(self) -> None: + l1 = [ + "first:top", + "first:bottom", + [ + "sub1:top", + "sub1:bottom" + ] + ] + + l2 = [ + "second:top", + l1, + "second:bottom" + ] + + self.assertListEqual(shared.flatten_list(l2), [ + "second:top", + "first:top", + "first:bottom", + "sub1:top", + "sub1:bottom", + "second:bottom" + ]) + + def test_fetch_prop_dict(self) -> None: + self.assertEqual( + shared.fetch_prop(path="satu.sub1_satu", obj=self.sample_dict), + "hello", + msg="fetch value" + ) + + with self.assertRaisesRegex( + KeyError, "Key `sub1_not_exists` not found in dict at `satu.sub1_not_exists`", + msg="key not found"): + shared.fetch_prop(path="satu.sub1_not_exists", obj=self.sample_dict) + + + def test_fetch_prop_list(self) -> None: + self.assertEqual( + shared.fetch_prop(path="dua.1", obj=self.sample_dict), + "elem2", + msg="fetch list" + ) + + with self.assertRaisesRegex( + KeyError, "Invalid list index `10` in path `dua.10`", + msg="index not found"): + shared.fetch_prop(path="dua.10", obj=self.sample_dict) + + def test_fetch_prop_obj(self) -> None: + self.assertEqual( + shared.fetch_prop(path="programs.pydev.image", obj=self.sample_obj), + "python:3.11", + ) + + self.assertListEqual( + shared.fetch_prop(path="programs.pydev.volumes", obj=self.sample_obj), + [ + "temp1:/opt/temp1", + "temp2:/opt/temp2", + ] + ) + + with self.assertRaisesRegex( + KeyError, "Attribute `not_exists` not found in object at `programs.pydev.not_exists`"): + shared.fetch_prop(path="programs.pydev.not_exists", obj=self.sample_obj) if __name__ == "__main__": unittest.main() diff --git a/tests/test_volume.py b/tests/test_volume.py index 49df570..94b05b7 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -256,8 +256,7 @@ def gen_expected_cmd(vol_name: str, exclude_params: List[str] = []) -> str: "--entrypoint=restic -v /home/user1/.sandock_vol_backup:/backup_repo " f"-v {vol_name}:{mount_source}:ro restic/restic:0.18.0 --repo=/backup_repo " f"--compression=auto --no-cache --insecure-no-password backup " - f'--skip-if-unchanged --group-by=paths {("" if not vol_name in exclude_params else f"--exclude-file={mount_source}/.sandock_backup_ignore")} ' - f"{mount_source}" + f'--skip-if-unchanged --group-by=paths {("" if not vol_name in exclude_params else f"--exclude-file={mount_source}/.sandock_backup_ignore ")}{mount_source}' ) mock_file_exists_in_vol.side_effect = [