From b69cd311096569c2b7ae6b70694f8d5bf86e51b6 Mon Sep 17 00:00:00 2001 From: Abel Feng Date: Wed, 30 Oct 2024 18:06:02 +0800 Subject: [PATCH] vmm: support youki when create container in vm Signed-off-by: Abel Feng --- Makefile | 12 +- vmm/common/src/lib.rs | 1 - vmm/task/Cargo.lock | 655 ++++++++++++++++++++++++++++++++++---- vmm/task/Cargo.toml | 9 +- vmm/task/src/container.rs | 266 +++++++++------- vmm/task/src/debug.rs | 7 +- vmm/task/src/io.rs | 13 +- vmm/task/src/main.rs | 6 +- vmm/task/src/task.rs | 26 +- vmm/task/src/youki.rs | 648 +++++++++++++++++++++++++++++++++++++ 10 files changed, 1449 insertions(+), 194 deletions(-) create mode 100644 vmm/task/src/youki.rs diff --git a/Makefile b/Makefile index 60b3273f..147e9e0a 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,14 @@ INSTALL_DIR := /var/lib/kuasar BIN_DIR := /usr/local/bin SYSTEMD_SERVICE_DIR := /usr/lib/systemd/system SYSTEMD_CONF_DIR := /etc/sysconfig +ENABLE_YOUKI ?= false +RUNC_FEATURES = +VMM_TASK_FEATURES = + +ifeq ($(ENABLE_YOUKI), true) + RUNC_FEATURES = youki + VMM_TASK_FEATURES = youki +endif .PHONY: vmm wasm quark clean all install-vmm install-wasm install-quark install \ bin/vmm-sandboxer bin/vmm-task bin/vmlinux.bin bin/kuasar.img bin/kuasar.initrd \ @@ -21,7 +29,7 @@ bin/vmm-sandboxer: @mkdir -p bin && cp vmm/sandbox/target/release/${HYPERVISOR} bin/vmm-sandboxer bin/vmm-task: - @cd vmm/task && cargo build --release --target=${ARCH}-unknown-linux-musl + @cd vmm/task && cargo build --release --target=${ARCH}-unknown-linux-musl --features=${VMM_TASK_FEATURES} @mkdir -p bin && cp vmm/task/target/${ARCH}-unknown-linux-musl/release/vmm-task bin/vmm-task bin/vmlinux.bin: @@ -45,7 +53,7 @@ bin/quark-sandboxer: @mkdir -p bin && cp quark/target/release/quark-sandboxer bin/quark-sandboxer bin/runc-sandboxer: - @cd runc && cargo build --release + @cd runc && cargo build --release --features=${RUNC_FEATURES} @mkdir -p bin && cp runc/target/release/runc-sandboxer bin/runc-sandboxer wasm: bin/wasm-sandboxer diff --git a/vmm/common/src/lib.rs b/vmm/common/src/lib.rs index b1d8418f..498f90c3 100644 --- a/vmm/common/src/lib.rs +++ b/vmm/common/src/lib.rs @@ -21,7 +21,6 @@ pub mod mount; pub mod storage; pub const KUASAR_STATE_DIR: &str = "/run/kuasar/state"; -pub const YOUKI_DIR: &str = "/run/kuasar/youki"; pub const IO_FILE_PREFIX: &str = "io"; pub const STORAGE_FILE_PREFIX: &str = "storage"; diff --git a/vmm/task/Cargo.lock b/vmm/task/Cargo.lock index f64120bd..c4e990a8 100644 --- a/vmm/task/Cargo.lock +++ b/vmm/task/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aho-corasick" version = "1.0.2" @@ -26,6 +32,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.71" @@ -54,7 +75,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.85", ] [[package]] @@ -65,7 +86,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.85", ] [[package]] @@ -140,7 +161,7 @@ dependencies = [ "cc", "cfg-if 1.0.0", "libc", - "miniz_oxide", + "miniz_oxide 0.7.2", "object", "rustc-demangle", ] @@ -169,6 +190,12 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "byteorder" version = "1.4.3" @@ -191,11 +218,24 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +[[package]] +name = "caps" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190baaad529bcfbde9e1a19022c42781bdb6ff9de25721abdb8fd98c0807730b" +dependencies = [ + "libc", + "thiserror", +] + [[package]] name = "cc" -version = "1.0.79" +version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +dependencies = [ + "shlex", +] [[package]] name = "cfg-if" @@ -227,6 +267,19 @@ dependencies = [ "regex", ] +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "windows-targets 0.52.6", +] + [[package]] name = "cmake" version = "0.1.50" @@ -260,7 +313,7 @@ dependencies = [ "libc", "log", "nix 0.23.2", - "oci-spec", + "oci-spec 0.5.8", "pin-project-lite", "prost 0.10.4", "prost-types 0.10.1", @@ -290,7 +343,7 @@ dependencies = [ "libc", "log", "nix 0.28.0", - "oci-spec", + "oci-spec 0.5.8", "page_size", "pin-project-lite", "prctl", @@ -317,6 +370,21 @@ dependencies = [ "ttrpc-codegen 0.4.2", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crossbeam" version = "0.8.2" @@ -390,8 +458,18 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core 0.20.10", + "darling_macro 0.20.10", ] [[package]] @@ -404,21 +482,46 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.10.0", "syn 1.0.109", ] +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.85", +] + [[package]] name = "darling_macro" version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ - "darling_core", + "darling_core 0.14.4", "quote", "syn 1.0.109", ] +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core 0.20.10", + "quote", + "syn 2.0.85", +] + [[package]] name = "derive-new" version = "0.5.9" @@ -436,7 +539,16 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3" dependencies = [ - "derive_builder_macro", + "derive_builder_macro 0.11.2", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro 0.20.2", ] [[package]] @@ -445,22 +557,44 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" dependencies = [ - "darling", + "darling 0.14.4", "proc-macro2", "quote", "syn 1.0.109", ] +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.10", + "proc-macro2", + "quote", + "syn 2.0.85", +] + [[package]] name = "derive_builder_macro" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68" dependencies = [ - "derive_builder_core", + "derive_builder_core 0.11.2", "syn 1.0.109", ] +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core 0.20.2", + "syn 2.0.85", +] + [[package]] name = "either" version = "1.8.1" @@ -510,6 +644,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" + [[package]] name = "fixedbitset" version = "0.2.0" @@ -522,6 +662,22 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flate2" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +dependencies = [ + "crc32fast", + "miniz_oxide 0.8.0", +] + [[package]] name = "fnv" version = "1.0.7" @@ -584,7 +740,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.85", ] [[package]] @@ -695,6 +851,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -719,6 +881,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "0.2.9" @@ -801,6 +969,29 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -870,6 +1061,15 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +[[package]] +name = "js-sys" +version = "0.3.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -878,9 +1078,48 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" + +[[package]] +name = "libcgroups" +version = "0.4.1" +source = "git+https://github.com/containers/youki.git#6717cbcb2d883a5e5ebdf1e4fef3eac59c2b7091" +dependencies = [ + "fixedbitset 0.5.7", + "nix 0.28.0", + "oci-spec 0.7.0", + "procfs", + "serde", + "thiserror", + "tracing", +] + +[[package]] +name = "libcontainer" +version = "0.4.1" +source = "git+https://github.com/containers/youki.git#6717cbcb2d883a5e5ebdf1e4fef3eac59c2b7091" +dependencies = [ + "caps", + "chrono", + "fastrand 2.1.1", + "libc", + "libcgroups", + "nc", + "nix 0.28.0", + "oci-spec 0.7.0", + "once_cell", + "prctl", + "procfs", + "regex", + "rust-criu", + "safe-path", + "serde", + "serde_json", + "thiserror", + "tracing", +] [[package]] name = "linux-raw-sys" @@ -888,6 +1127,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + [[package]] name = "lock_api" version = "0.4.10" @@ -912,9 +1157,9 @@ checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" [[package]] name = "memchr" -version = "2.5.0" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memoffset" @@ -958,6 +1203,15 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "0.8.11" @@ -975,6 +1229,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "nc" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34566634a278b9af0f62b872339d884ea689982514825ba306705f264038144e" +dependencies = [ + "cc", +] + [[package]] name = "netlink-packet-core" version = "0.7.0" @@ -1128,6 +1391,15 @@ dependencies = [ "memoffset 0.9.1", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.15.0" @@ -1140,9 +1412,9 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" dependencies = [ "libc", ] @@ -1162,18 +1434,34 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98135224dd4faeb24c05a2fac911ed53ea6b09ecb09d7cada1cb79963ab2ee34" dependencies = [ - "derive_builder", + "derive_builder 0.11.2", "getset", "serde", "serde_json", "thiserror", ] +[[package]] +name = "oci-spec" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cee185ce7cf1cce45e194e34cd87c0bad7ff0aa2e8917009a2da4f7b31fb363" +dependencies = [ + "derive_builder 0.20.2", + "getset", + "regex", + "serde", + "serde_json", + "strum", + "strum_macros", + "thiserror", +] + [[package]] name = "once_cell" -version = "1.18.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "os_pipe" @@ -1215,7 +1503,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.48.0", ] [[package]] @@ -1285,7 +1573,7 @@ checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.85", ] [[package]] @@ -1352,13 +1640,39 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.59" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" +dependencies = [ + "bitflags 2.6.0", + "chrono", + "flate2", + "hex", + "lazy_static", + "procfs-core", + "rustix 0.38.25", +] + +[[package]] +name = "procfs-core" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" +dependencies = [ + "bitflags 2.6.0", + "chrono", + "hex", +] + [[package]] name = "prost" version = "0.8.0" @@ -1533,9 +1847,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.28" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -1581,9 +1895,21 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.4" +version = "1.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", @@ -1592,9 +1918,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rtnetlink" @@ -1624,7 +1950,7 @@ dependencies = [ "libc", "log", "nix 0.25.1", - "oci-spec", + "oci-spec 0.5.8", "os_pipe", "path-absolutize", "rand", @@ -1638,6 +1964,18 @@ dependencies = [ "uuid", ] +[[package]] +name = "rust-criu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4737b28406b3395359f485127073117a11cedc8942738b69ba6ab9a79432acbc" +dependencies = [ + "anyhow", + "libc", + "protobuf 3.2.0", + "protobuf-codegen 3.2.0", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -1654,16 +1992,44 @@ dependencies = [ "errno", "io-lifetimes", "libc", - "linux-raw-sys", + "linux-raw-sys 0.3.8", "windows-sys", ] +[[package]] +name = "rustix" +version = "0.38.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys 0.4.14", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" + [[package]] name = "ryu" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "safe-path" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "980abdd3220aa19b67ca3ea07b173ca36383f18ae48cde696d90c8af39447ffb" +dependencies = [ + "libc", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -1687,7 +2053,7 @@ checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.85", ] [[package]] @@ -1701,6 +2067,12 @@ dependencies = [ "serde", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook" version = "0.3.15" @@ -1763,6 +2135,31 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.85", +] + [[package]] name = "syn" version = "1.0.109" @@ -1776,9 +2173,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" dependencies = [ "proc-macro2", "quote", @@ -1799,9 +2196,9 @@ checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" dependencies = [ "autocfg", "cfg-if 1.0.0", - "fastrand", + "fastrand 1.9.0", "redox_syscall", - "rustix", + "rustix 0.37.19", "windows-sys", ] @@ -1816,22 +2213,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.85", ] [[package]] @@ -1882,7 +2279,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.85", ] [[package]] @@ -2031,11 +2428,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-attributes", @@ -2044,20 +2440,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.24" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.85", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", ] @@ -2207,12 +2603,14 @@ dependencies = [ "ipnetwork", "lazy_static", "libc", + "libcontainer", "log", "netlink-packet-core", "netlink-packet-route", "netlink-sys 0.7.0", "nix 0.28.0", - "oci-spec", + "oci-spec 0.5.8", + "os_pipe", "pin-project-lite", "protobuf 3.2.0", "rtnetlink", @@ -2222,6 +2620,7 @@ dependencies = [ "signal-hook-tokio", "time", "tokio", + "tokio-pipe", "tokio-vsock", "ttrpc", "vmm-common", @@ -2253,6 +2652,61 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.85", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" + [[package]] name = "which" version = "4.4.0" @@ -2295,13 +2749,22 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.0", ] [[package]] @@ -2310,13 +2773,29 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -2325,38 +2804,86 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + [[package]] name = "windows_aarch64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + [[package]] name = "windows_i686_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + [[package]] name = "windows_i686_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + [[package]] name = "windows_x86_64_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + [[package]] name = "windows_x86_64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" diff --git a/vmm/task/Cargo.toml b/vmm/task/Cargo.toml index d9b4e350..0e8e1347 100644 --- a/vmm/task/Cargo.toml +++ b/vmm/task/Cargo.toml @@ -12,7 +12,7 @@ vmm-common = { path = "../common" } log = "0.4" nix = { version = "0.28.0", features = ["sched", "term", "time", "hostname", "signal", "mount", "uio", "socket"] } libc = "0.2.95" -time = { version = "=0.3.23", features = ["serde", "std"] } +time = { version = "=0.3.7", features = ["serde", "std"] } serde = { version = "1.0.133", features = ["derive"] } serde_json = "1.0.74" oci-spec = "0.5.4" @@ -39,7 +39,12 @@ ttrpc = { version = "0.7", features = ["async"] } containerd-sandbox = { git = "https://github.com/kuasar-io/rust-extensions.git" } containerd-shim = { git = "https://github.com/kuasar-io/rust-extensions.git", features = ["async"] } runc = { git = "https://github.com/kuasar-io/rust-extensions.git", features = ["async"] } -libcontainer = { git="https://github.com/containers/youki.git", version="0.3.0", default-features = false, features=["v1"]} +libcontainer = { git="https://github.com/containers/youki.git", version="0.4.1", optional = true, default-features = false, features = ["v1", "v2", "systemd"] } +os_pipe = "1.0.0" +tokio-pipe = "0.2.12" [patch.crates-io] ttrpc = { git = "https://github.com/kuasar-io/ttrpc-rust.git", branch = "v0.7.1-kuasar" } + +[features] +youki = ["libcontainer"] diff --git a/vmm/task/src/container.rs b/vmm/task/src/container.rs index 181cafa9..2ca86b49 100644 --- a/vmm/task/src/container.rs +++ b/vmm/task/src/container.rs @@ -15,12 +15,8 @@ limitations under the License. */ use std::{ - convert::TryFrom, - io::SeekFrom, - os::unix::prelude::ExitStatusExt, - path::{Path, PathBuf}, - process::ExitStatus, - sync::Arc, + convert::TryFrom, io::SeekFrom, os::unix::prelude::ExitStatusExt, path::Path, + process::ExitStatus, sync::Arc, }; use async_trait::async_trait; @@ -46,15 +42,10 @@ use containerd_shim::{ util::read_spec, ExitSignal, }; -use libcontainer::{ - container::{builder::ContainerBuilder, Container}, - signal::Signal, - syscall::syscall::SyscallType, -}; use log::{debug, error}; use nix::{sys::signalfd::signal::kill, unistd::Pid}; use oci_spec::runtime::{LinuxResources, Process, Spec}; -use runc::Spawner; +use runc::{options::GlobalOpts, Runc, Spawner}; use serde::Deserialize; use tokio::{ fs::{remove_file, File}, @@ -62,7 +53,7 @@ use tokio::{ process::Command, sync::Mutex, }; -use vmm_common::{mount::get_mount_type, storage::Storage, KUASAR_STATE_DIR, YOUKI_DIR}; +use vmm_common::{mount::get_mount_type, storage::Storage, KUASAR_STATE_DIR}; use crate::{ device::rescan_pci_bus, @@ -86,12 +77,14 @@ pub(crate) struct KuasarFactory { } pub struct KuasarExecFactory { + runtime: Runc, bundle: String, io_uid: u32, io_gid: u32, } pub struct KuasarExecLifecycle { + runtime: Runc, bundle: String, container_id: String, io_uid: u32, @@ -101,6 +94,7 @@ pub struct KuasarExecLifecycle { } pub struct KuasarInitLifecycle { + runtime: Runc, opts: Options, bundle: String, exit_signal: Arc, @@ -116,7 +110,7 @@ pub struct Log { impl ContainerFactory for KuasarFactory { async fn create( &self, - _ns: &str, + ns: &str, req: &CreateTaskRequest, ) -> containerd_shim::Result { rescan_pci_bus().await?; @@ -141,9 +135,18 @@ impl ContainerFactory for KuasarFactory { if opts.compute_size() > 0 { debug!("create options: {:?}", &opts); } + let runtime = opts.binary_name.as_str(); // As the rootfs is already mounted when handling the storage, the root in spec is one of the // storage mount point. so no need to mount rootfs anymore + let runc = create_runc( + runtime, + ns, + &bundle, + &opts, + Some(Arc::new(ShimExecutor::default())), + )?; + let id = req.id(); let stdio = match read_io(&bundle, req.id(), None).await { @@ -155,7 +158,11 @@ impl ContainerFactory for KuasarFactory { // that needs to be converted to the serial file path let stdio = convert_stdio(&stdio).await?; - let mut init = InitProcess::new(id, stdio, KuasarInitLifecycle::new(opts.clone(), &bundle)); + let mut init = InitProcess::new( + id, + stdio, + KuasarInitLifecycle::new(runc.clone(), opts.clone(), &bundle), + ); self.do_create(&mut init).await?; let container = KuasarContainer { @@ -163,6 +170,7 @@ impl ContainerFactory for KuasarFactory { bundle: bundle.to_string(), init, process_factory: KuasarExecFactory { + runtime: runc, bundle: bundle.to_string(), io_uid: opts.io_uid, io_gid: opts.io_gid, @@ -189,46 +197,41 @@ impl KuasarFactory { let opts = &init.lifecycle.opts; let bundle = &init.lifecycle.bundle; let pid_path = Path::new(bundle).join(INIT_PID_FILE); - let mut _no_pivot_root = opts.no_pivot_root; + let mut no_pivot_root = opts.no_pivot_root; // pivot_root could not work with initramfs - // TODO youki not support no_pivot_root yet match get_mount_type("/") { Ok(m_type) => { if m_type == *"rootfs" { - _no_pivot_root = true; + no_pivot_root = true; } } Err(e) => debug!("get mount type failed {}", e), }; - - let (socket, pio, socket_path) = if stdio.terminal { + let mut create_opts = runc::options::CreateOpts::new() + .pid_file(&pid_path) + .no_pivot(no_pivot_root) + .no_new_keyring(opts.no_new_keyring) + .detach(false); + let (socket, pio) = if stdio.terminal { let s = ConsoleSocket::new().await?; - let path = s.path.to_owned(); - (Some(s), None, Some(path)) + create_opts.console_socket = Some(s.path.to_owned()); + (Some(s), None) } else { let pio = create_io(&id, opts.io_uid, opts.io_gid, stdio)?; - (None, Some(pio), None) - }; - - let resp = ContainerBuilder::new(id.to_string(), SyscallType::default()) - .with_pid_file(Some(pid_path.clone())) - .map_err(other_error!(e, "failed to set youki create pid file"))? - .with_console_socket(socket_path) - .with_root_path(PathBuf::from(YOUKI_DIR)) - .map_err(other_error!(e, "failed to set youki create root path"))? - .as_init(bundle) - .with_systemd(false) - .with_detach(false) - .build(); + create_opts.io = pio.io.as_ref().cloned(); + (None, Some(pio)) + }; + + let resp = init + .lifecycle + .runtime + .create(&id, bundle, Some(&create_opts)) + .await; if let Err(e) = resp { if let Some(s) = socket { s.clean().await; } - return Err(other!( - "youki create container {} failed {}", - id.to_string(), - e - )); + return Err(runtime_error(bundle, e, "OCI runtime create failed").await); } copy_io_or_console(init, socket, pio, init.lifecycle.exit_signal.clone()).await?; let pid = read_file_to_str(pid_path).await?.parse::()?; @@ -237,6 +240,25 @@ impl KuasarFactory { } } +// runtime_error will read the OCI runtime logfile retrieving OCI runtime error +pub async fn runtime_error(bundle: &str, r_err: runc::error::Error, msg: &str) -> Error { + match get_last_runtime_error(bundle).await { + Err(e) => other!( + "{}: unable to retrieve OCI runtime error ({}): {}", + msg, + e, + r_err + ), + Ok(rt_msg) => { + if rt_msg.is_empty() { + other!("{}: empty msg in log file: {}", msg, r_err) + } else { + other!("{}: {}", msg, rt_msg) + } + } + } +} + async fn get_last_runtime_error(bundle: &str) -> Result { let log_path = Path::new(bundle).join("log.json"); let mut rt_msg = String::new(); @@ -304,6 +326,7 @@ impl ProcessFactory for KuasarExecFactory { wait_chan_tx: vec![], console: None, lifecycle: Arc::from(KuasarExecLifecycle { + runtime: self.runtime.clone(), bundle: self.bundle.to_string(), container_id: req.id.to_string(), io_uid: self.io_uid, @@ -319,12 +342,9 @@ impl ProcessFactory for KuasarExecFactory { #[async_trait] impl ProcessLifecycle for KuasarInitLifecycle { async fn start(&self, p: &mut InitProcess) -> containerd_shim::Result<()> { - let mut container = Container::load(PathBuf::from(YOUKI_DIR).join(p.id.as_str())) - .map_err(other_error!(e, "youki load container error"))?; - - container - .start() - .map_err(other_error!(e, "youki start container error"))?; + if let Err(e) = self.runtime.start(p.id.as_str()).await { + return Err(runtime_error(&p.lifecycle.bundle, e, "OCI runtime start failed").await); + } p.state = Status::RUNNING; Ok(()) } @@ -335,34 +355,37 @@ impl ProcessLifecycle for KuasarInitLifecycle { signal: u32, all: bool, ) -> containerd_shim::Result<()> { - let mut container = Container::load(PathBuf::from(YOUKI_DIR).join(p.id.as_str())) - .map_err(other_error!(e, "youki load container error"))?; - - let kill_signal: Signal = - signal.to_string().as_str().try_into().map_err(|e| { - Error::Other(format!("kill signal {} transfer error: {}", signal, e)) - })?; - - container - .kill(kill_signal, all) - .map_err(other_error!(e, "youki kill container error")) - .map_err(|e| check_kill_error(e.to_string())) + if let Err(r_err) = self + .runtime + .kill( + p.id.as_str(), + signal, + Some(&runc::options::KillOpts { all }), + ) + .await + { + let e = runtime_error(&p.lifecycle.bundle, r_err, "OCI runtime kill failed").await; + + return Err(check_kill_error(e.to_string())); + } + Ok(()) } async fn delete(&self, p: &mut InitProcess) -> containerd_shim::Result<()> { - let mut container = Container::load(PathBuf::from(YOUKI_DIR).join(p.id.as_str())) - .map_err(other_error!(e, "youki load container error"))?; - - container - .delete(true) - .or_else(|e| { - if !e.to_string().to_lowercase().contains("does not exist") { - Err(e) - } else { - Ok(()) - } - }) - .map_err(other_error!(e, "youki delete container error"))?; + if let Err(e) = self + .runtime + .delete( + p.id.as_str(), + Some(&runc::options::DeleteOpts { force: true }), + ) + .await + { + if !e.to_string().to_lowercase().contains("does not exist") { + return Err( + runtime_error(&p.lifecycle.bundle, e, "OCI runtime delete failed").await, + ); + } + } self.exit_signal.signal(); Ok(()) } @@ -400,11 +423,12 @@ impl ProcessLifecycle for KuasarInitLifecycle { } async fn ps(&self, p: &InitProcess) -> Result> { - let container = Container::load(PathBuf::from(YOUKI_DIR).join(p.id.as_str())) - .map_err(other_error!(e, "youki load container error"))?; - - Ok(container - .pid() + let pids = self + .runtime + .ps(&p.id) + .await + .map_err(other_error!(e, "failed to execute runc ps"))?; + Ok(pids .iter() .map(|&x| ProcessInfo { pid: x as u32, @@ -415,13 +439,14 @@ impl ProcessLifecycle for KuasarInitLifecycle { } impl KuasarInitLifecycle { - pub fn new(opts: Options, bundle: &str) -> Self { + pub fn new(runtime: Runc, opts: Options, bundle: &str) -> Self { let work_dir = Path::new(bundle).join("work"); let mut opts = opts; if opts.criu_path().is_empty() { opts.criu_path = work_dir.to_string_lossy().to_string(); } Self { + runtime, opts, bundle: bundle.to_string(), exit_signal: Default::default(), @@ -433,47 +458,34 @@ impl KuasarInitLifecycle { impl ProcessLifecycle for KuasarExecLifecycle { async fn start(&self, p: &mut ExecProcess) -> containerd_shim::Result<()> { rescan_pci_bus().await?; - let pid_path = Path::new(self.bundle.as_str()).join(format!("{}.pid", &p.id)); - let (socket, pio, socket_path) = if p.stdio.terminal { + let bundle = self.bundle.to_string(); + let pid_path = Path::new(&bundle).join(format!("{}.pid", &p.id)); + let mut exec_opts = runc::options::ExecOpts { + io: None, + pid_file: Some(pid_path.to_owned()), + console_socket: None, + detach: true, + }; + let (socket, pio) = if p.stdio.terminal { let s = ConsoleSocket::new().await?; - let path = s.path.to_owned(); - (Some(s), None, Some(path)) + exec_opts.console_socket = Some(s.path.to_owned()); + (Some(s), None) } else { let pio = create_io(&p.id, self.io_uid, self.io_gid, &p.stdio)?; - (None, Some(pio), None) + exec_opts.io = pio.io.as_ref().cloned(); + (None, Some(pio)) }; - - let probe_path = format!("{}/{}-process.json", self.bundle, &p.id); - let spec_str = serde_json::to_string(&self.spec) - .map_err(other_error!(e, "failed to marshall exec spec to string"))?; - tokio::fs::write(&probe_path, &spec_str) - .await - .map_err(other_error!(e, "failed to write spec to process.json"))?; - //TODO checkpoint support - let exec_result = ContainerBuilder::new(self.container_id.clone(), SyscallType::default()) - .with_root_path(PathBuf::from(YOUKI_DIR)) - .map_err(other_error!(e, "failed to set youki root path"))? - .with_console_socket(socket_path) - .with_pid_file(Some(pid_path.clone())) - .map_err(other_error!(e, "failed to set process pid file"))? - .as_tenant() - .with_detach(true) - .with_process(Some(&probe_path)) - .build(); + let exec_result = self + .runtime + .exec(&self.container_id, &self.spec, Some(&exec_opts)) + .await; if let Err(e) = exec_result { if let Some(s) = socket { s.clean().await; } - let _ = tokio::fs::remove_file(&probe_path).await; - return Err(other!( - "youki exec container {} failed {}", - self.container_id, - e - )); + return Err(runtime_error(&bundle, e, "OCI runtime exec failed").await); } - let _ = tokio::fs::remove_file(&probe_path).await; - copy_io_or_console(p, socket, pio, p.lifecycle.exit_signal.clone()).await?; let pid = read_file_to_str(pid_path).await?.parse::()?; p.pid = pid; @@ -535,6 +547,44 @@ fn get_spec_from_request( } } +const DEFAULT_RUNC_ROOT: &str = "/run/containerd/runc"; +const DEFAULT_COMMAND: &str = "runc"; + +pub fn create_runc( + runtime: &str, + namespace: &str, + bundle: impl AsRef, + opts: &Options, + spawner: Option>, +) -> containerd_shim::Result { + let runtime = if runtime.is_empty() { + DEFAULT_COMMAND + } else { + runtime + }; + let root = opts.root.as_str(); + let root = Path::new(if root.is_empty() { + DEFAULT_RUNC_ROOT + } else { + root + }) + .join(namespace); + + let log = bundle.as_ref().join("log.json"); + let mut gopts = GlobalOpts::default() + .command(runtime) + .root(root) + .log(log) + .log_json() + .systemd_cgroup(opts.systemd_cgroup); + if let Some(s) = spawner { + gopts.custom_spawner(s); + } + gopts + .build() + .map_err(other_error!(e, "unable to create runc instance")) +} + #[derive(Default, Debug)] pub struct ShimExecutor {} diff --git a/vmm/task/src/debug.rs b/vmm/task/src/debug.rs index 094db50c..19c2c31c 100644 --- a/vmm/task/src/debug.rs +++ b/vmm/task/src/debug.rs @@ -26,7 +26,11 @@ use containerd_shim::{ }; use futures::StreamExt; use log::{debug, error}; -use nix::{pty::openpty, unistd::setsid}; +use nix::{ + pty::openpty, + sys::signal::{kill, Signal}, + unistd::{setsid, Pid}, +}; use tokio::process::Command; use tokio_vsock::VsockStream; @@ -79,6 +83,7 @@ pub async fn debug_console(stream: VsockStream) -> Result<()> { } } if let Some(id) = child.id() { + kill(Pid::from_raw(id as i32), Signal::SIGKILL).unwrap_or_default(); let exit_status = wait_pid(id as i32, s).await; debug!("debug console shell exit with {}", exit_status) } diff --git a/vmm/task/src/io.rs b/vmm/task/src/io.rs index 19a2c3fe..4ec5dd97 100644 --- a/vmm/task/src/io.rs +++ b/vmm/task/src/io.rs @@ -27,12 +27,12 @@ use std::{ task::{Context, Poll}, }; +#[cfg(not(feature = "youki"))] +use containerd_shim::util::IntoOption; use containerd_shim::{ asynchronous::{console::ConsoleSocket, processes::ProcessTemplate, util::asyncify}, io::Stdio, - io_error, other, - util::IntoOption, - Console, Error, ExitSignal, Result, + io_error, other, Console, Error, ExitSignal, Result, }; use log::{debug, error, warn}; use nix::{ @@ -42,7 +42,9 @@ use nix::{ termios::tcgetattr, }, }; -use runc::io::{IOOption, Io, NullIo, PipedIo, FIFO}; +use runc::io::Io; +#[cfg(not(feature = "youki"))] +use runc::io::{IOOption, NullIo, PipedIo, FIFO}; use tokio::{ fs::{File, OpenOptions}, io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf}, @@ -63,6 +65,7 @@ pub struct ProcessIO { const VSOCK: &str = "vsock"; const STREAMING: &str = "streaming"; +#[cfg(not(feature = "youki"))] pub fn create_io( id: &str, io_uid: u32, @@ -145,7 +148,7 @@ pub(crate) async fn copy_io_or_console

( Ok(()) } -async fn copy_console

( +pub async fn copy_console

( p: &ProcessTemplate

, console_socket: &ConsoleSocket, stdio: &Stdio, diff --git a/vmm/task/src/main.rs b/vmm/task/src/main.rs index f93e563c..cd6a8311 100644 --- a/vmm/task/src/main.rs +++ b/vmm/task/src/main.rs @@ -44,7 +44,6 @@ use vmm_common::{ api::{sandbox_ttrpc::create_sandbox_service, streaming_ttrpc::create_streaming}, mount::mount, ETC_RESOLV, IPC_NAMESPACE, KUASAR_STATE_DIR, PID_NAMESPACE, RESOLV_FILENAME, UTS_NAMESPACE, - HOSTNAME_FILENAME, SANDBOX_NS_PATH, YOUKI_DIR, }; use crate::{ @@ -56,6 +55,7 @@ use crate::{ }; mod config; +#[cfg(not(feature = "youki"))] mod container; mod debug; mod device; @@ -69,6 +69,8 @@ mod streaming; mod task; mod util; mod vsock; +#[cfg(feature = "youki")] +mod youki; const NAMESPACE: &str = "k8s.io"; @@ -173,8 +175,6 @@ async fn initialize() -> anyhow::Result<()> { } } - tokio::fs::create_dir_all(YOUKI_DIR).await.expect("failed to create youki dir"); - late_init_call().await?; Ok(()) diff --git a/vmm/task/src/task.rs b/vmm/task/src/task.rs index 22dfef70..3e94aa10 100644 --- a/vmm/task/src/task.rs +++ b/vmm/task/src/task.rs @@ -31,18 +31,28 @@ use log::{debug, error}; use oci_spec::runtime::{LinuxNamespaceType, Spec}; use tokio::sync::{mpsc::Sender, Mutex}; -use crate::{ - container::{KuasarContainer, KuasarFactory}, - sandbox::SandboxResources, - NAMESPACE, -}; +#[cfg(not(feature = "youki"))] +use crate::container::{KuasarContainer, KuasarFactory}; +#[cfg(feature = "youki")] +use crate::youki::{YoukiContainer, YoukiFactory}; +use crate::{sandbox::SandboxResources, NAMESPACE}; + +#[cfg(not(feature = "youki"))] +type Factory = KuasarFactory; +#[cfg(not(feature = "youki"))] +type RealContainer = KuasarContainer; + +#[cfg(feature = "youki")] +type Factory = YoukiFactory; +#[cfg(feature = "youki")] +type RealContainer = YoukiContainer; pub(crate) async fn create_task_service( tx: Sender<(String, Box)>, -) -> anyhow::Result> { +) -> anyhow::Result> { let sandbox = Arc::new(Mutex::new(SandboxResources::new().await)); let task = TaskService { - factory: KuasarFactory::new(sandbox), + factory: Factory::new(sandbox), containers: Arc::new(Default::default()), namespace: NAMESPACE.to_string(), exit: Arc::new(Default::default()), @@ -54,7 +64,7 @@ pub(crate) async fn create_task_service( Ok(task) } -async fn process_exits(s: Subscription, task: &TaskService) { +async fn process_exits(s: Subscription, task: &TaskService) { let containers = task.containers.clone(); let mut s = s; tokio::spawn(async move { diff --git a/vmm/task/src/youki.rs b/vmm/task/src/youki.rs new file mode 100644 index 00000000..ef4a1638 --- /dev/null +++ b/vmm/task/src/youki.rs @@ -0,0 +1,648 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +use std::{ + convert::TryFrom, + os::fd::{FromRawFd, IntoRawFd, OwnedFd, RawFd}, + path::{Path, PathBuf}, + sync::Arc, +}; + +use async_trait::async_trait; +use containerd_shim::{ + api::{CreateTaskRequest, ExecProcessRequest, Options, Status}, + asynchronous::{ + console::ConsoleSocket, + container::{ContainerFactory, ContainerTemplate, ProcessFactory}, + processes::{ProcessLifecycle, ProcessTemplate}, + }, + io::Stdio, + io_error, other, other_error, + protos::{ + api::ProcessInfo, + cgroups::metrics::Metrics, + protobuf::{CodedInputStream, Message}, + }, + util::read_spec, + Error, ExitSignal, Result, +}; +use libcontainer::{ + container::{builder::ContainerBuilder, Container}, + error::LibcontainerError, + signal::Signal, + syscall::syscall::SyscallType, +}; +use log::{debug, warn}; +use nix::{ + sys::signal::kill, + unistd::{Gid, Pid, Uid}, +}; +use oci_spec::runtime::{LinuxResources, Process, Spec}; +use runc::io::{IOOption, Io, NullIo}; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + process::Command, + sync::Mutex, + task::spawn_blocking, +}; +use vmm_common::{storage::Storage, KUASAR_STATE_DIR}; + +use crate::{ + device::rescan_pci_bus, + io::{convert_stdio, copy_io_or_console, ProcessIO}, + sandbox::SandboxResources, + util::{read_io, read_storages}, +}; + +pub type ExecProcess = ProcessTemplate; +pub type InitProcess = ProcessTemplate; + +pub type YoukiContainer = ContainerTemplate; + +pub const YOUKI_DIR: &str = "/run/kuasar/youki"; +const STORAGE_ANNOTATION: &str = "io.kuasar.storages"; + +#[derive(Clone)] +pub(crate) struct YoukiFactory { + sandbox: Arc>, +} + +#[async_trait] +impl ContainerFactory for YoukiFactory { + async fn create( + &self, + _ns: &str, + req: &CreateTaskRequest, + ) -> containerd_shim::Result { + rescan_pci_bus().await?; + let bundle = format!("{}/{}", KUASAR_STATE_DIR, req.id); + let spec: Spec = read_spec(&bundle).await?; + let annotations = spec.annotations().clone().unwrap_or_default(); + let storages = if let Some(storage_str) = annotations.get(STORAGE_ANNOTATION) { + serde_json::from_str::>(storage_str)? + } else { + read_storages(&bundle, req.id()).await? + }; + self.sandbox + .lock() + .await + .add_storages(req.id(), storages) + .await?; + let mut opts = Options::new(); + if let Some(any) = req.options.as_ref() { + let mut input = CodedInputStream::from_bytes(any.value.as_ref()); + opts.merge_from(&mut input)?; + } + if opts.compute_size() > 0 { + debug!("create options: {:?}", &opts); + } + + let id = req.id(); + + let stdio = match read_io(&bundle, req.id(), None).await { + Ok(io) => Stdio::new(&io.stdin, &io.stdout, &io.stderr, io.terminal), + Err(_) => Stdio::new(req.stdin(), req.stdout(), req.stderr(), req.terminal()), + }; + + // for qemu, the io path is pci address for virtio-serial + // that needs to be converted to the serial file path + let stdio = convert_stdio(&stdio).await?; + + let init = self.do_create(id, &stdio, &opts, &bundle).await?; + let container = YoukiContainer { + id: id.to_string(), + bundle: bundle.to_string(), + init, + process_factory: YoukiExecFactory { + bundle: bundle.to_string(), + io_uid: opts.io_uid, + io_gid: opts.io_gid, + }, + processes: Default::default(), + }; + Ok(container) + } + + async fn cleanup(&self, _ns: &str, c: &YoukiContainer) -> containerd_shim::Result<()> { + self.sandbox.lock().await.defer_storages(&c.id).await?; + Ok(()) + } +} + +impl YoukiFactory { + pub fn new(sandbox: Arc>) -> Self { + Self { sandbox } + } + + async fn do_create( + &self, + id: &str, + stdio: &Stdio, + opts: &Options, + bundle: &str, + ) -> Result { + // youki seems not support no_pivot_root and no_new_keyring option yet. + + let (socket, pio, _container_io) = if stdio.terminal { + let s = ConsoleSocket::new().await?; + (Some(s), None, None) + } else { + let (pio, container_io) = create_io(&id, opts.io_uid, opts.io_gid, stdio)?; + (None, Some(pio), Some(container_io)) + }; + let id_clone = id.to_string(); + let opts_clone = opts.clone(); + let bundle_clone = bundle.to_string(); + let socket_path = socket.as_ref().map(|p| p.path.clone()); + let resp = spawn_blocking(move || { + let builder = ContainerBuilder::new(id_clone, SyscallType::default()) + .with_console_socket(socket_path) + .with_root_path(PathBuf::from(YOUKI_DIR)) + .map_err(other_error!(e, "failed to set youki create root path"))?; + // if let Some(f) = container_io { + // if let Some(p) = f.stdin { + // builder = builder.with_stdin(p); + // } + // if let Some(p) = f.stdout { + // builder = builder.with_stdout(p); + // } + // if let Some(p) = f.stderr { + // builder = builder.with_stderr(p); + // } + // } + + let resp = builder + .as_init(bundle_clone) + .with_systemd(opts_clone.systemd_cgroup) + .build() + .map_err(other_error!(e, "failed to build youki container ")); + resp + }) + .await + .map_err(other_error!(e, "failed to wait container building "))?; + + match resp { + Ok(mut c) => { + let pid = if let Some(p) = c.pid() { + p.as_raw() + } else { + if let Err(e) = c.delete(true) { + warn!("failed to cleanup container {}", e); + } + return Err(other!("failed to get pid of the youki container {}", id)); + }; + let mut init = InitProcess::new( + id, + stdio.clone(), + YoukiInitLifecycle::new(Arc::new(Mutex::new(c)), opts.clone(), bundle), + ); + let exit_signal = init.lifecycle.exit_signal.clone(); + copy_io_or_console(&mut init, socket, pio, exit_signal).await?; + init.pid = pid; + return Ok(init); + } + Err(e) => { + if let Some(s) = socket { + s.clean().await; + } + return Err(other!("failed to create container {}", e)); + } + } + } +} + +pub struct YoukiExecFactory { + bundle: String, + io_uid: u32, + io_gid: u32, +} + +#[async_trait] +impl ProcessFactory for YoukiExecFactory { + async fn create(&self, req: &ExecProcessRequest) -> Result { + let p = get_spec_from_request(req)?; + let stdio = match read_io(&self.bundle, req.id(), Some(req.exec_id())).await { + // terminal is still determined from request + Ok(io) => Stdio::new(&io.stdin, &io.stdout, &io.stderr, req.terminal()), + Err(_) => Stdio::new(req.stdin(), req.stdout(), req.stderr(), req.terminal()), + }; + let stdio = convert_stdio(&stdio).await?; + Ok(ExecProcess { + state: Status::CREATED, + id: req.exec_id.to_string(), + stdio: stdio, + pid: 0, + exit_code: 0, + exited_at: None, + wait_chan_tx: vec![], + console: None, + lifecycle: Arc::from(YoukiExecLifecycle { + bundle: self.bundle.to_string(), + container_id: req.id.to_string(), + io_uid: self.io_uid, + io_gid: self.io_gid, + spec: p, + exit_signal: Default::default(), + }), + stdin: Arc::new(Mutex::new(None)), + }) + } +} + +pub struct YoukiInitLifecycle { + youki_container: Arc>, + exit_signal: Arc, +} + +#[async_trait] +impl ProcessLifecycle for YoukiInitLifecycle { + async fn start(&self, p: &mut InitProcess) -> containerd_shim::Result<()> { + p.lifecycle + .youki_container + .lock() + .await + .start() + .map_err(other_error!(e, "failed to start container "))?; + p.state = Status::RUNNING; + Ok(()) + } + + async fn kill( + &self, + p: &mut InitProcess, + signal: u32, + all: bool, + ) -> containerd_shim::Result<()> { + let signal = Signal::try_from(signal as i32) + .map_err(other_error!(e, "failed to parse kill signal "))?; + p.lifecycle + .youki_container + .lock() + .await + .kill(signal, all) + .or_else(|e| { + if let LibcontainerError::IncorrectStatus = e { + Ok(()) + } else { + Err(e) + } + }) + .map_err(other_error!(e, "failed to kill container "))?; + Ok(()) + } + + async fn delete(&self, p: &mut InitProcess) -> containerd_shim::Result<()> { + p.lifecycle + .youki_container + .lock() + .await + .delete(true) + .map_err(other_error!(e, "failed to delete container "))?; + self.exit_signal.signal(); + Ok(()) + } + + #[cfg(target_os = "linux")] + async fn update(&self, p: &mut InitProcess, resources: &LinuxResources) -> Result<()> { + if p.pid <= 0 { + return Err(other!( + "failed to update resources because init process is {}", + p.pid + )); + } + containerd_shim::cgroup::update_resources(p.pid as u32, resources) + } + + #[cfg(not(target_os = "linux"))] + async fn update(&self, _p: &mut InitProcess, _resources: &LinuxResources) -> Result<()> { + Err(Error::Unimplemented("update resource".to_string())) + } + + #[cfg(target_os = "linux")] + async fn stats(&self, p: &InitProcess) -> Result { + if p.pid <= 0 { + return Err(other!( + "failed to collect metrics because init process is {}", + p.pid + )); + } + containerd_shim::cgroup::collect_metrics(p.pid as u32) + } + + #[cfg(not(target_os = "linux"))] + async fn stats(&self, _p: &InitProcess) -> Result { + Err(Error::Unimplemented("process stats".to_string())) + } + + async fn ps(&self, p: &InitProcess) -> Result> { + Ok(vec![ProcessInfo { + pid: p.pid as u32, + ..Default::default() + }]) + } +} + +impl YoukiInitLifecycle { + pub fn new(youki_container: Arc>, opts: Options, bundle: &str) -> Self { + let work_dir = Path::new(bundle).join("work"); + let mut opts = opts; + if opts.criu_path().is_empty() { + opts.criu_path = work_dir.to_string_lossy().to_string(); + } + Self { + youki_container: youki_container, + exit_signal: Default::default(), + } + } +} + +pub struct YoukiExecLifecycle { + bundle: String, + container_id: String, + io_uid: u32, + io_gid: u32, + spec: Process, + exit_signal: Arc, +} + +#[async_trait] +impl ProcessLifecycle for YoukiExecLifecycle { + async fn start(&self, p: &mut ExecProcess) -> containerd_shim::Result<()> { + rescan_pci_bus().await?; + let (socket, pio, _container_io) = if p.stdio.terminal { + let s = ConsoleSocket::new().await?; + (Some(s), None, None) + } else { + let (pio, container_io) = create_io(&p.id, self.io_uid, self.io_gid, &p.stdio)?; + //exec_opts.io = pio.io.as_ref().cloned(); + (None, Some(pio), Some(container_io)) + }; + + let probe_path = format!("{}/{}-process.json", self.bundle, &p.id); + let spec_str = serde_json::to_string(&self.spec) + .map_err(other_error!(e, "failed to marshall exec spec to string"))?; + tokio::fs::write(&probe_path, &spec_str) + .await + .map_err(other_error!(e, "failed to write spec to process.json"))?; + + let container_id = self.container_id.clone(); + let socket_path = socket.as_ref().map(|p| p.path.clone()); + let probe_path_clone = probe_path.clone(); + let exec_result = spawn_blocking(move || { + let builder = ContainerBuilder::new(container_id, SyscallType::default()) + .with_root_path(PathBuf::from(YOUKI_DIR)) + .map_err(other_error!(e, "failed to set youki root path"))? + .with_console_socket(socket_path); + + // if let Some(f) = container_io { + // if let Some(fd) = f.stdin { + // builder = builder.with_stdin(fd); + // } + // if let Some(fd) = f.stdout { + // builder = builder.with_stdout(fd); + // } + // if let Some(fd) = f.stderr { + // builder = builder.with_stderr(fd); + // } + // } + + let exec_result = builder + .as_tenant() + .with_detach(true) + .with_process(Some(&probe_path_clone)) + .build() + .map_err(other_error!( + e, + "failed to exec process in youki container " + )); + exec_result + }) + .await + .map_err(other_error!(e, "failed to wait exec thread "))?; + tokio::fs::remove_file(&probe_path) + .await + .unwrap_or_default(); + match exec_result { + Ok(pid) => { + copy_io_or_console(p, socket, pio, p.lifecycle.exit_signal.clone()).await?; + p.pid = pid.as_raw(); + p.state = Status::RUNNING; + } + Err(e) => { + if let Some(s) = socket { + s.clean().await; + } + return Err(other!("failed to start youki exec: {}", e)); + } + } + Ok(()) + } + + async fn kill( + &self, + p: &mut ExecProcess, + signal: u32, + _all: bool, + ) -> containerd_shim::Result<()> { + if p.pid <= 0 { + Err(Error::FailedPreconditionError( + "process not created".to_string(), + )) + } else if p.exited_at.is_some() { + Err(Error::NotFoundError("process already finished".to_string())) + } else { + // TODO this is kill from nix crate, it is os specific, maybe have annotated with target os + kill( + Pid::from_raw(p.pid), + nix::sys::signal::Signal::try_from(signal as i32).unwrap(), + ) + .map_err(Into::into) + } + } + + async fn delete(&self, _p: &mut ExecProcess) -> containerd_shim::Result<()> { + self.exit_signal.signal(); + Ok(()) + } + + async fn update(&self, _p: &mut ExecProcess, _resources: &LinuxResources) -> Result<()> { + Err(Error::Unimplemented("exec update".to_string())) + } + + async fn stats(&self, _p: &ExecProcess) -> Result { + Err(Error::Unimplemented("exec stats".to_string())) + } + + async fn ps(&self, _p: &ExecProcess) -> Result> { + Err(Error::Unimplemented("exec ps".to_string())) + } +} + +pub fn create_io( + _id: &str, + io_uid: u32, + io_gid: u32, + stdio: &Stdio, +) -> containerd_shim::Result<(ProcessIO, ContainerPipeIo)> { + if stdio.is_null() { + let nio = NullIo::new().map_err(io_error!(e, "new Null Io"))?; + let pio = ProcessIO { + io: Some(Arc::new(nio)), + copy: false, + }; + return Ok((pio, ContainerPipeIo::default())); + } + let stdout = stdio.stdout.as_str(); + let scheme_path = stdout.trim().split("://").collect::>(); + let _scheme = if scheme_path.len() <= 1 { + // no scheme specified + // default schema to fifo + "fifo" + } else { + scheme_path[0] + }; + + let mut pio = ProcessIO { + io: None, + copy: false, + }; + + let opt = IOOption { + open_stdin: !stdio.stdin.is_empty(), + open_stdout: !stdio.stdout.is_empty(), + open_stderr: !stdio.stderr.is_empty(), + }; + let (io, container_io) = + PipedIo::new(io_uid, io_gid, &opt).map_err(io_error!(e, "new PipedIo"))?; + pio.io = Some(Arc::new(io)); + pio.copy = true; + Ok((pio, container_io)) +} + +impl Io for PipedIo { + fn stdin(&self) -> Option> { + self.stdin.and_then(|pipe| { + tokio_pipe::PipeWrite::from_raw_fd_checked(pipe) + .map(|x| Box::new(x) as Box) + .ok() + }) + } + + fn stdout(&self) -> Option> { + self.stdout.and_then(|pipe| { + tokio_pipe::PipeRead::from_raw_fd_checked(pipe) + .map(|x| Box::new(x) as Box) + .ok() + }) + } + + fn stderr(&self) -> Option> { + self.stderr.and_then(|pipe| { + tokio_pipe::PipeRead::from_raw_fd_checked(pipe) + .map(|x| Box::new(x) as Box) + .ok() + }) + } + + // Note that this internally use [`std::fs::File`]'s `try_clone()`. + // Thus, the files passed to commands will be not closed after command exit. + fn set(&self, _cmd: &mut Command) -> std::io::Result<()> { + Ok(()) + } + + fn close_after_start(&self) {} +} + +/// Struct to represent a pipe that can be used to transfer stdio inputs and outputs. +/// +/// With this Io driver, methods of [crate::Runc] may capture the output/error messages. +/// When one side of the pipe is closed, the state will be represented with [`None`]. +#[derive(Debug)] +pub struct Pipe { + rd: RawFd, + wr: RawFd, +} + +#[derive(Debug, Default)] +pub struct PipedIo { + stdin: Option, + stdout: Option, + stderr: Option, +} + +#[derive(Debug, Default)] +pub struct ContainerPipeIo { + stdin: Option, + stdout: Option, + stderr: Option, +} + +impl Pipe { + fn new() -> std::io::Result { + let (rd, wr) = os_pipe::pipe()?; + Ok(Self { + rd: rd.into_raw_fd(), + wr: wr.into_raw_fd(), + }) + } +} + +impl PipedIo { + pub fn new(uid: u32, gid: u32, opts: &IOOption) -> std::io::Result<(Self, ContainerPipeIo)> { + let mut res = Self::default(); + let mut container_pipes = ContainerPipeIo::default(); + if opts.open_stdin { + let pipe = Self::create_pipe(uid, gid, true)?; + res.stdin = Some(pipe.wr); + container_pipes.stdin = Some(unsafe { OwnedFd::from_raw_fd(pipe.rd) }); + } + if opts.open_stdout { + let pipe = Self::create_pipe(uid, gid, false)?; + res.stdout = Some(pipe.rd); + container_pipes.stdout = Some(unsafe { OwnedFd::from_raw_fd(pipe.wr) }); + } + if opts.open_stderr { + let pipe = Self::create_pipe(uid, gid, false)?; + res.stderr = Some(pipe.rd); + container_pipes.stderr = Some(unsafe { OwnedFd::from_raw_fd(pipe.wr) }); + } + Ok((res, container_pipes)) + } + + fn create_pipe(uid: u32, gid: u32, stdin: bool) -> std::io::Result { + let pipe = Pipe::new()?; + let uid = Some(Uid::from_raw(uid)); + let gid = Some(Gid::from_raw(gid)); + if stdin { + nix::unistd::fchown(pipe.rd, uid, gid)?; + } else { + nix::unistd::fchown(pipe.wr, uid, gid)?; + } + Ok(pipe) + } +} + +fn get_spec_from_request( + req: &ExecProcessRequest, +) -> containerd_shim::Result { + if let Some(val) = req.spec.as_ref() { + let mut p = serde_json::from_slice::(&val.value)?; + p.set_terminal(Some(req.terminal)); + Ok(p) + } else { + Err(Error::InvalidArgument("no spec in request".to_string())) + } +}