From 2b596b52c96fc923a61269facedc5a24fb557687 Mon Sep 17 00:00:00 2001 From: Jake Correnti Date: Tue, 10 Jun 2025 11:44:39 -0400 Subject: [PATCH 1/5] cargo: Use rust-vmm/kvm instead of virtee/kvm-* Signed-off-by: Jake Correnti --- Cargo.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2d71b4d..71193bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,8 +14,9 @@ rust-version = "1.86" [dependencies] bitflags = "2.4.2" -kvm-bindings = { git = "https://github.com/virtee/kvm-bindings.git", branch = "main", features = ["fam-wrappers"] } -kvm-ioctls = { git = "https://github.com/virtee/kvm-ioctls.git", branch = "main" } +iocuddle = "0.1.1" +kvm-bindings = { version = "0.12.0", features = ["fam-wrappers"] } +kvm-ioctls = "0.22.0" libc = "0.2.155" uuid = "1.8.0" vmm-sys-util = "0.12.1" From 9b764823168623cb141ddce21e153081f135af01 Mon Sep 17 00:00:00 2001 From: Jake Correnti Date: Tue, 10 Jun 2025 11:45:59 -0400 Subject: [PATCH 2/5] launch: Update API Update the API to use a `Launcher` type instead of a `TdxVm` and `TdxVcpu`. This is more elegant and is more in line with what virtee/sev is providing users as an interface. Signed-off-by: Jake Correnti --- src/launch/linux.rs | 6 ++ src/launch/mod.rs | 177 +++++++++++++++++++++----------------------- 2 files changed, 91 insertions(+), 92 deletions(-) diff --git a/src/launch/linux.rs b/src/launch/linux.rs index fcaa6e5..0f07375 100644 --- a/src/launch/linux.rs +++ b/src/launch/linux.rs @@ -60,6 +60,12 @@ impl From for TdxError { } } +impl From for TdxError { + fn from(err: std::io::Error) -> Self { + TdxError::from(err.raw_os_error().unwrap()) + } +} + impl From for TdxError { fn from(errno: i32) -> Self { match errno { diff --git a/src/launch/mod.rs b/src/launch/mod.rs index 2e32cf3..b7747cb 100644 --- a/src/launch/mod.rs +++ b/src/launch/mod.rs @@ -4,11 +4,20 @@ mod bindings; mod linux; use bindings::{kvm_tdx_capabilities, kvm_tdx_init_mem_region, kvm_tdx_init_vm}; -use kvm_bindings::kvm_enable_cap; use linux::{Cmd, CmdId, TdxError, NR_CPUID_CONFIGS}; use bitflags::bitflags; -use kvm_ioctls::VmFd; +use iocuddle::*; + +use std::os::unix::io::RawFd; + +const KVM: Group = Group::new(0xAE); +const ENC_OP: Ioctl = unsafe { KVM.write_read(0xBA) }; +const GET_CAPABILITIES: Ioctl> = unsafe { ENC_OP.lie() }; +const INIT_VM: Ioctl> = unsafe { ENC_OP.lie() }; +const INIT_VCPU: Ioctl> = unsafe { ENC_OP.lie() }; +const INIT_MEM_REGION: Ioctl> = unsafe { ENC_OP.lie() }; +const FINALIZE: Ioctl> = unsafe { ENC_OP.lie() }; // Defined in linux/arch/x86/include/uapi/asm/kvm.h pub const KVM_X86_TDX_VM: u64 = 5; @@ -42,24 +51,41 @@ pub fn vec_with_array_field(count: usize) -> Vec { vec_with_size_in_bytes(vec_size_bytes) } -/// Handle to the TDX VM file descriptor -pub struct TdxVm {} +pub struct MemRegion { + pub gpa: u64, + pub nr_pages: u64, + pub attributes: u32, + pub source_addr: u64, +} -impl TdxVm { - /// Create a new TDX VM with KVM - pub fn new(vm_fd: &VmFd) -> Result { - let mut cap: kvm_enable_cap = kvm_enable_cap { - cap: kvm_bindings::KVM_CAP_X2APIC_API, - ..Default::default() - }; - cap.args[0] = (1 << 0) | (1 << 1); - vm_fd.enable_cap(&cap).unwrap(); +impl MemRegion { + pub fn new(gpa: u64, nr_pages: u64, attributes: u32, source_addr: u64) -> Self { + Self { + gpa, + nr_pages, + attributes, + source_addr, + } + } +} + +pub type Result = std::result::Result; + +#[derive(Clone, Default)] +pub struct Launcher { + vm_fd: RawFd, + vcpu_fds: Vec, +} - Ok(Self {}) +impl Launcher { + pub fn new(vm_fd: RawFd) -> Self { + Self { + vm_fd, + vcpu_fds: Vec::new(), + } } - /// Retrieve information about the Intel TDX module - pub fn get_capabilities(&self, fd: &VmFd) -> Result { + pub fn get_capabilities(&mut self) -> Result { let mut caps = kvm_tdx_capabilities::default(); let mut defaults = Vec::with_capacity(NR_CPUID_CONFIGS); @@ -83,9 +109,7 @@ impl TdxVm { let mut cmd: Cmd = Cmd::from(CmdId::GetCapabilities, &cpuid_entries[0]); - unsafe { - fd.encrypt_op(&mut cmd)?; - } + GET_CAPABILITIES.ioctl(&mut self.vm_fd, &mut cmd).unwrap(); Ok(TdxCapabilities { attributes: AttributesFlags::from_bits_truncate(cpuid_entries[0].supported_attrs), @@ -100,13 +124,7 @@ impl TdxVm { }) } - /// Do additional VM initialization that is specific to Intel TDX - pub fn init_vm( - &self, - fd: &VmFd, - caps: &TdxCapabilities, - cpuid: kvm_bindings::CpuId, - ) -> Result<(), TdxError> { + pub fn init_vm(&mut self, caps: &TdxCapabilities, cpuid: kvm_bindings::CpuId) -> Result<()> { let mut defaults: Vec = cpuid.as_slice().to_vec(); defaults.resize( kvm_bindings::KVM_MAX_CPUID_ENTRIES, @@ -131,9 +149,7 @@ impl TdxVm { Self::tdx_filter_cpuid(&mut entries[0].cpuid, &caps.cpuid_configs); let mut cmd: Cmd = Cmd::from(CmdId::InitVm, &entries[0]); - unsafe { - fd.encrypt_op(&mut cmd)?; - } + INIT_VM.ioctl(&mut self.vm_fd, &mut cmd).unwrap(); Ok(()) } @@ -182,13 +198,49 @@ impl TdxVm { None } - /// Complete measurement of the initial TD contents and mark it ready to run - pub fn finalize(&self, fd: &VmFd) -> Result<(), TdxError> { + pub fn finalize(&mut self) -> Result<()> { let mut cmd: Cmd = Cmd::from(CmdId::FinalizeVm, &0); - unsafe { - fd.encrypt_op(&mut cmd)?; + FINALIZE.ioctl(&mut self.vm_fd, &mut cmd).unwrap(); + + Ok(()) + } + + pub fn add_vcpu_fd(&mut self, fd: RawFd) { + self.vcpu_fds.push(fd); + } + + pub fn init_vcpus(&mut self, hob_address: u64) -> Result<()> { + let mut cmd: Cmd = Cmd::from(CmdId::InitVcpu, &hob_address); + for fd in self.vcpu_fds.iter_mut() { + INIT_VCPU.ioctl(fd, &mut cmd).unwrap(); + } + Ok(()) + } + + pub fn init_mem_region(&mut self, region: MemRegion) -> Result<()> { + if self.vcpu_fds.is_empty() { + return Err(TdxError { + code: -22, + message: String::from("missing vcpu fds"), + }); } + const TDVF_SECTION_ATTRIBUTES_MR_EXTEND: u32 = 1u32 << 0; + let mem_region = kvm_tdx_init_mem_region { + source_addr: region.source_addr, + gpa: region.gpa, + nr_pages: region.nr_pages, + }; + + let mut cmd: Cmd = Cmd::from(CmdId::InitMemRegion, &mem_region); + + // determines if we also extend the measurement + cmd.flags = (region.attributes & TDVF_SECTION_ATTRIBUTES_MR_EXTEND > 0) as u32; + + INIT_MEM_REGION + .ioctl(&mut self.vcpu_fds[0], &mut cmd) + .unwrap(); + Ok(()) } } @@ -319,62 +371,3 @@ pub struct TdxCapabilities { pub xfam: XFAMFlags, pub cpuid_configs: Vec, } - -/// Manually create the wrapper for KVM_MEMORY_ENCRYPT_OP since `kvm_ioctls` doesn't -/// support `.encrypt_op` for vcpu fds -use vmm_sys_util::*; -ioctl_iowr_nr!( - KVM_MEMORY_ENCRYPT_OP, - kvm_bindings::KVMIO, - 0xba, - std::os::raw::c_ulong -); - -pub struct TdxVcpu {} - -impl TdxVcpu { - pub fn init(fd: &kvm_ioctls::VcpuFd, hob_address: u64) -> Result<(), TdxError> { - let mut cmd: Cmd = Cmd::from(CmdId::InitVcpu, &hob_address); - let ret = unsafe { ioctl::ioctl_with_mut_ptr(fd, KVM_MEMORY_ENCRYPT_OP(), &mut cmd) }; - if ret < 0 { - // can't return `ret` because it will just return -1 and not give the error - // code. `cmd.error` will also just be 0. - return Err(TdxError::from(errno::Error::last())); - } - Ok(()) - } - - /// Encrypt a memory continuous region - pub fn init_mem_region( - fd: &kvm_ioctls::VcpuFd, - gpa: u64, - nr_pages: u64, - attributes: u32, - source_addr: u64, - ) -> Result<(), TdxError> { - const TDVF_SECTION_ATTRIBUTES_MR_EXTEND: u32 = 1u32 << 0; - let mem_region = kvm_tdx_init_mem_region { - source_addr, - gpa, - nr_pages, - }; - - let mut cmd: Cmd = Cmd::from(CmdId::InitMemRegion, &mem_region); - - // determines if we also extend the measurement - cmd.flags = if attributes & TDVF_SECTION_ATTRIBUTES_MR_EXTEND > 0 { - 1 - } else { - 0 - }; - - let ret = unsafe { ioctl::ioctl_with_mut_ptr(fd, KVM_MEMORY_ENCRYPT_OP(), &mut cmd) }; - if ret < 0 { - // can't return `ret` because it will just return -1 and not give the error - // code. `cmd.error` will also just be 0. - return Err(TdxError::from(errno::Error::last())); - } - - Ok(()) - } -} From 1fbe55f2ccbead4629b1304a2d382bb11e2248f0 Mon Sep 17 00:00:00 2001 From: Jake Correnti Date: Tue, 10 Jun 2025 11:47:09 -0400 Subject: [PATCH 3/5] tests: update launch test Update the launch test to use the new `Launcher` API. Signed-off-by: Jake Correnti --- tests/launch.rs | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/launch.rs b/tests/launch.rs index 1c579a2..b95fafe 100644 --- a/tests/launch.rs +++ b/tests/launch.rs @@ -3,9 +3,11 @@ use kvm_ioctls::Kvm; use vmm_sys_util::*; -use tdx::launch::{TdxVcpu, TdxVm}; +use tdx::launch::*; use tdx::tdvf; +use std::os::unix::io::AsRawFd; + // `mov eax,1000h` will set the value in the register eax (and rax since they both share the bottom 32 bits) to 1000h // `jmp *%rax` will jump the program to the address that rax contains, which in this case will be 1000h const FIRMWARE: &[u8; 7] = &[ @@ -16,7 +18,6 @@ const FIRMWARE: &[u8; 7] = &[ #[test] fn launch() { const KVM_CAP_GUEST_MEMFD: u32 = 234; - const KVM_CAP_MEMORY_MAPPING: u32 = 236; // create vm let kvm_fd = Kvm::new().unwrap(); @@ -31,12 +32,12 @@ fn launch() { cap.args[0] = 24; vm_fd.enable_cap(&cap).unwrap(); - let tdx_vm = TdxVm::new(&vm_fd).unwrap(); - let _caps = tdx_vm.get_capabilities(&vm_fd).unwrap(); + let mut launcher = Launcher::new(vm_fd.as_raw_fd()); + let caps = launcher.get_capabilities().unwrap(); let cpuid = kvm_fd .get_supported_cpuid(kvm_bindings::KVM_MAX_CPUID_ENTRIES) .unwrap(); - let _ = tdx_vm.init_vm(&vm_fd, &_caps, cpuid).unwrap(); + launcher.init_vm(&caps, cpuid).unwrap(); // get tdvf sections let mut firmware = std::fs::File::open("/usr/share/edk2/ovmf/OVMF.inteltdx.fd").unwrap(); @@ -45,6 +46,8 @@ fn launch() { // create vcpu let mut vcpufd = vm_fd.create_vcpu(10).unwrap(); + launcher.add_vcpu_fd(vcpufd.as_raw_fd()); + let mut cpuid = kvm_fd .get_supported_cpuid(kvm_bindings::KVM_MAX_CPUID_ENTRIES) .unwrap(); @@ -55,7 +58,8 @@ fn launch() { } } vcpufd.set_cpuid2(&cpuid).unwrap(); - TdxVcpu::init(&vcpufd, hob_section.memory_address).unwrap(); + + launcher.init_vcpus(hob_section.memory_address).unwrap(); // map memory to guest if !check_extension(KVM_CAP_GUEST_MEMFD) { @@ -118,15 +122,12 @@ fn launch() { }; vm_fd.set_memory_attributes(attr).unwrap(); - if check_extension(KVM_CAP_MEMORY_MAPPING) { - // TODO(jakecorrenti): the current CentOS SIG doesn't support the KVM_MEMORY_MAPPING or - // KVM_TDX_EXTEND_MEMORY ioctls, which is what we would typically use here. - } else { - TdxVcpu::init_mem_region(&vcpufd, guest_addr, 1, 1, firmware_userspace).unwrap(); - } + launcher + .init_mem_region(MemRegion::new(guest_addr, 1, 1, firmware_userspace)) + .unwrap(); // finalize measurement - tdx_vm.finalize(&vm_fd).unwrap(); + launcher.finalize().unwrap(); // run the vCPU From 49b21833b36e546cfaa5aa83455c0716af694d1b Mon Sep 17 00:00:00 2001 From: Jake Correnti Date: Tue, 10 Jun 2025 12:16:23 -0400 Subject: [PATCH 4/5] launch: Update Error type and handling Update Error type to be more idiomatic Rust. Consequently, update error handling where necessary. Signed-off-by: Jake Correnti --- src/launch/linux.rs | 49 ++++++++++++++++++--------------------------- src/launch/mod.rs | 25 +++++++++++++---------- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/src/launch/linux.rs b/src/launch/linux.rs index 0f07375..8205f39 100644 --- a/src/launch/linux.rs +++ b/src/launch/linux.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +use std::io::Error as IoError; use std::marker::PhantomData; pub const NR_CPUID_CONFIGS: usize = 24; @@ -49,38 +50,26 @@ impl<'a, T: 'a> Cmd<'a, T> { } #[derive(Debug)] -pub struct TdxError { - pub code: i32, - pub message: String, +pub enum Error { + GetCapabilities(IoError), + InitVm(IoError), + InitVcpu(IoError), + InitMemRegion(IoError), + Finalize(IoError), + MissingVcpuFds, } -impl From for TdxError { - fn from(kvm_err: kvm_ioctls::Error) -> Self { - TdxError::from(kvm_err.errno()) - } -} - -impl From for TdxError { - fn from(err: std::io::Error) -> Self { - TdxError::from(err.raw_os_error().unwrap()) - } -} - -impl From for TdxError { - fn from(errno: i32) -> Self { - match errno { - 7 => TdxError { - code: 7, - message: String::from("Invalid value for NR_CPUID_CONFIGS"), - }, - 25 => TdxError { - code: 25, - message: String::from("Inappropriate ioctl for device. Ensure the proper VM type is being used for the ioctl"), - }, - _ => TdxError { - code: errno, - message: format!("errno: {}", errno), - }, +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::GetCapabilities(io_err) => { + write!(f, "KVM_TDX_CAPABILITIES failed: {io_err}") + } + Error::InitVm(io_err) => write!(f, "KVM_TDX_INIT_VM failed: {io_err}"), + Error::InitVcpu(io_err) => write!(f, "KVM_TDX_INIT_VCPU failed: {io_err}"), + Error::InitMemRegion(io_err) => write!(f, "KVM_TDX_INIT_MEM_REGION failed: {io_err}"), + Error::Finalize(io_err) => write!(f, "KVM_TDX_FINALIZE failed: {io_err}"), + Error::MissingVcpuFds => write!(f, "Launcher contains zero vCPU file descriptors"), } } } diff --git a/src/launch/mod.rs b/src/launch/mod.rs index b7747cb..4753ed6 100644 --- a/src/launch/mod.rs +++ b/src/launch/mod.rs @@ -4,7 +4,7 @@ mod bindings; mod linux; use bindings::{kvm_tdx_capabilities, kvm_tdx_init_mem_region, kvm_tdx_init_vm}; -use linux::{Cmd, CmdId, TdxError, NR_CPUID_CONFIGS}; +use linux::{Cmd, CmdId, Error, NR_CPUID_CONFIGS}; use bitflags::bitflags; use iocuddle::*; @@ -69,7 +69,7 @@ impl MemRegion { } } -pub type Result = std::result::Result; +pub type Result = std::result::Result; #[derive(Clone, Default)] pub struct Launcher { @@ -109,7 +109,9 @@ impl Launcher { let mut cmd: Cmd = Cmd::from(CmdId::GetCapabilities, &cpuid_entries[0]); - GET_CAPABILITIES.ioctl(&mut self.vm_fd, &mut cmd).unwrap(); + GET_CAPABILITIES + .ioctl(&mut self.vm_fd, &mut cmd) + .map_err(Error::GetCapabilities)?; Ok(TdxCapabilities { attributes: AttributesFlags::from_bits_truncate(cpuid_entries[0].supported_attrs), @@ -149,7 +151,9 @@ impl Launcher { Self::tdx_filter_cpuid(&mut entries[0].cpuid, &caps.cpuid_configs); let mut cmd: Cmd = Cmd::from(CmdId::InitVm, &entries[0]); - INIT_VM.ioctl(&mut self.vm_fd, &mut cmd).unwrap(); + INIT_VM + .ioctl(&mut self.vm_fd, &mut cmd) + .map_err(Error::InitVm)?; Ok(()) } @@ -200,7 +204,9 @@ impl Launcher { pub fn finalize(&mut self) -> Result<()> { let mut cmd: Cmd = Cmd::from(CmdId::FinalizeVm, &0); - FINALIZE.ioctl(&mut self.vm_fd, &mut cmd).unwrap(); + FINALIZE + .ioctl(&mut self.vm_fd, &mut cmd) + .map_err(Error::Finalize)?; Ok(()) } @@ -212,17 +218,14 @@ impl Launcher { pub fn init_vcpus(&mut self, hob_address: u64) -> Result<()> { let mut cmd: Cmd = Cmd::from(CmdId::InitVcpu, &hob_address); for fd in self.vcpu_fds.iter_mut() { - INIT_VCPU.ioctl(fd, &mut cmd).unwrap(); + INIT_VCPU.ioctl(fd, &mut cmd).map_err(Error::InitVcpu)?; } Ok(()) } pub fn init_mem_region(&mut self, region: MemRegion) -> Result<()> { if self.vcpu_fds.is_empty() { - return Err(TdxError { - code: -22, - message: String::from("missing vcpu fds"), - }); + return Err(Error::MissingVcpuFds); } const TDVF_SECTION_ATTRIBUTES_MR_EXTEND: u32 = 1u32 << 0; @@ -239,7 +242,7 @@ impl Launcher { INIT_MEM_REGION .ioctl(&mut self.vcpu_fds[0], &mut cmd) - .unwrap(); + .map_err(Error::InitMemRegion)?; Ok(()) } From b4b3c18cd8fe5fa2f1452ad1bcc3befb979cb9cc Mon Sep 17 00:00:00 2001 From: Jake Correnti Date: Tue, 10 Jun 2025 13:19:05 -0400 Subject: [PATCH 5/5] launch: Document API methods Signed-off-by: Jake Correnti --- src/launch/mod.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/launch/mod.rs b/src/launch/mod.rs index 4753ed6..7727f41 100644 --- a/src/launch/mod.rs +++ b/src/launch/mod.rs @@ -51,10 +51,18 @@ pub fn vec_with_array_field(count: usize) -> Vec { vec_with_size_in_bytes(vec_size_bytes) } +/// Represents the memory region to be initialized by KVM_TDX_INIT_MEM_REGION pub struct MemRegion { + /// starting guest address of private memory pub gpa: u64, + + /// number of pages to initialize pub nr_pages: u64, + + /// memory attributes of region pub attributes: u32, + + /// address of userspace provided data pub source_addr: u64, } @@ -71,13 +79,18 @@ impl MemRegion { pub type Result = std::result::Result; +/// Launcher facilitates the correct execution of the TDX command #[derive(Clone, Default)] pub struct Launcher { + /// Raw FD associated with the KVM VM fd vm_fd: RawFd, + + /// Collection of Raw FDs associated with the vCPUs created with KVM vcpu_fds: Vec, } impl Launcher { + /// Initialize a new Launcher pub fn new(vm_fd: RawFd) -> Self { Self { vm_fd, @@ -85,6 +98,8 @@ impl Launcher { } } + /// Retrieve the TDX capabilities that KVM supports with the TDX module loaded + /// in the system. pub fn get_capabilities(&mut self) -> Result { let mut caps = kvm_tdx_capabilities::default(); @@ -126,6 +141,10 @@ impl Launcher { }) } + /// Perform TDX specific VM initialization. + /// + /// Note: this must be called after after calling `KVM_CREATE_VM` and before + /// creating any vCPUs. pub fn init_vm(&mut self, caps: &TdxCapabilities, cpuid: kvm_bindings::CpuId) -> Result<()> { let mut defaults: Vec = cpuid.as_slice().to_vec(); defaults.resize( @@ -202,6 +221,7 @@ impl Launcher { None } + /// Complete measurement of the initial TD contents and mark it ready to run pub fn finalize(&mut self) -> Result<()> { let mut cmd: Cmd = Cmd::from(CmdId::FinalizeVm, &0); FINALIZE @@ -211,10 +231,13 @@ impl Launcher { Ok(()) } + /// Add RawFd associated with a vCPU to the Launcher pub fn add_vcpu_fd(&mut self, fd: RawFd) { self.vcpu_fds.push(fd); } + /// Perform TDX specific vCPU initialization for each vCPU fd provided to + /// the Launcher pub fn init_vcpus(&mut self, hob_address: u64) -> Result<()> { let mut cmd: Cmd = Cmd::from(CmdId::InitVcpu, &hob_address); for fd in self.vcpu_fds.iter_mut() { @@ -223,6 +246,8 @@ impl Launcher { Ok(()) } + /// Initialize @nr_pages TDX guest private memory starting from @gpa with + /// userspace provided data from @source_addr pub fn init_mem_region(&mut self, region: MemRegion) -> Result<()> { if self.vcpu_fds.is_empty() { return Err(Error::MissingVcpuFds);