From 1a5a9f49d2b4bb27c3fa5666a122b8104dd3cab5 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:08 +0200 Subject: [PATCH 01/51] rust: alloc: implement `ReallocFunc` `ReallocFunc` is an abstraction for the kernel's realloc derivates, such as `krealloc`, `vrealloc` and `kvrealloc`. All of the named functions share the same function signature and implement the same semantics. The `ReallocFunc` abstractions provides a generalized wrapper around those, to trivialize the implementation of `Kmalloc`, `Vmalloc` and `KVmalloc` in subsequent patches. Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-5-dakr@kernel.org [ Added temporary `allow(dead_code)` for `dangling_from_layout` to clean warning in `rusttest` target as discussed in the list (but it is needed earlier, i.e. in this patch already). - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 9 +++++ rust/kernel/alloc/allocator.rs | 70 ++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 998779cc69769f..95d402feb6ffd9 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -187,3 +187,12 @@ pub unsafe trait Allocator { let _ = unsafe { Self::realloc(Some(ptr), Layout::new::<()>(), layout, Flags(0)) }; } } + +#[allow(dead_code)] +/// Returns a properly aligned dangling pointer from the given `layout`. +pub(crate) fn dangling_from_layout(layout: Layout) -> NonNull { + let ptr = layout.align() as *mut u8; + + // SAFETY: `layout.align()` (and hence `ptr`) is guaranteed to be non-zero. + unsafe { NonNull::new_unchecked(ptr) } +} diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 3b1c735ba409fb..e5bac90046a43a 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -1,10 +1,20 @@ // SPDX-License-Identifier: GPL-2.0 //! Allocator support. +//! +//! Documentation for the kernel's memory allocators can found in the "Memory Allocation Guide" +//! linked below. For instance, this includes the concept of "get free page" (GFP) flags and the +//! typical application of the different kernel allocators. +//! +//! Reference: use super::{flags::*, Flags}; use core::alloc::{GlobalAlloc, Layout}; use core::ptr; +use core::ptr::NonNull; + +use crate::alloc::AllocError; +use crate::bindings; struct Kmalloc; @@ -36,6 +46,66 @@ pub(crate) unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: F unsafe { bindings::krealloc(ptr as *const core::ffi::c_void, size, flags.0) as *mut u8 } } +/// # Invariants +/// +/// One of the following `krealloc`, `vrealloc`, `kvrealloc`. +struct ReallocFunc( + unsafe extern "C" fn(*const core::ffi::c_void, usize, u32) -> *mut core::ffi::c_void, +); + +#[expect(dead_code)] +impl ReallocFunc { + /// # Safety + /// + /// This method has the same safety requirements as [`Allocator::realloc`]. + /// + /// # Guarantees + /// + /// This method has the same guarantees as `Allocator::realloc`. Additionally + /// - it accepts any pointer to a valid memory allocation allocated by this function. + /// - memory allocated by this function remains valid until it is passed to this function. + unsafe fn call( + &self, + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError> { + let size = aligned_size(layout); + let ptr = match ptr { + Some(ptr) => { + if old_layout.size() == 0 { + ptr::null() + } else { + ptr.as_ptr() + } + } + None => ptr::null(), + }; + + // SAFETY: + // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that + // `ptr` is NULL or valid. + // - `ptr` is either NULL or valid by the safety requirements of this function. + // + // GUARANTEE: + // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc`. + // - Those functions provide the guarantees of this function. + let raw_ptr = unsafe { + // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. + self.0(ptr.cast(), size, flags.0).cast() + }; + + let ptr = if size == 0 { + crate::alloc::dangling_from_layout(layout) + } else { + NonNull::new(raw_ptr).ok_or(AllocError)? + }; + + Ok(NonNull::slice_from_raw_parts(ptr, size)) + } +} + // SAFETY: TODO. unsafe impl GlobalAlloc for Kmalloc { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { From 6797ab52daa350f3e03a473dc7857a90faa5a5c3 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:09 +0200 Subject: [PATCH 02/51] rust: alloc: make `allocator` module public Subsequent patches implement allocators such as `Kmalloc`, `Vmalloc`, `KVmalloc`; we need them to be available outside of the kernel crate as well. Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-6-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 95d402feb6ffd9..2203852c37121b 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -4,7 +4,7 @@ #[cfg(not(test))] #[cfg(not(testlib))] -mod allocator; +pub mod allocator; pub mod box_ext; pub mod vec_ext; From 933d9e95aa9d9c78da3ac1c2e403c7ee6431d4cb Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:10 +0200 Subject: [PATCH 03/51] rust: alloc: implement `Allocator` for `Kmalloc` Implement `Allocator` for `Kmalloc`, the kernel's default allocator, typically used for objects smaller than page size. All memory allocations made with `Kmalloc` end up in `krealloc()`. It serves as allocator for the subsequently introduced types `KBox` and `KVec`. Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-7-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/alloc/allocator.rs | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index e5bac90046a43a..b1894c800448bd 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -13,10 +13,16 @@ use core::alloc::{GlobalAlloc, Layout}; use core::ptr; use core::ptr::NonNull; -use crate::alloc::AllocError; +use crate::alloc::{AllocError, Allocator}; use crate::bindings; -struct Kmalloc; +/// The contiguous kernel allocator. +/// +/// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also +/// supports larger allocations up to `bindings::KMALLOC_MAX_SIZE`, which is hardware specific. +/// +/// For more details see [self]. +pub struct Kmalloc; /// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment. fn aligned_size(new_layout: Layout) -> usize { @@ -53,8 +59,10 @@ struct ReallocFunc( unsafe extern "C" fn(*const core::ffi::c_void, usize, u32) -> *mut core::ffi::c_void, ); -#[expect(dead_code)] impl ReallocFunc { + // INVARIANT: `krealloc` satisfies the type invariants. + const KREALLOC: Self = Self(bindings::krealloc); + /// # Safety /// /// This method has the same safety requirements as [`Allocator::realloc`]. @@ -106,6 +114,23 @@ impl ReallocFunc { } } +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for Kmalloc { + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError> { + // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`. + unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags) } + } +} + // SAFETY: TODO. unsafe impl GlobalAlloc for Kmalloc { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { From 2c2ea829e81426080f33a9ee1c02194ec6f86658 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:11 +0200 Subject: [PATCH 04/51] rust: alloc: add module `allocator_test` `Allocator`s, such as `Kmalloc`, will be used by e.g. `Box` and `Vec` in subsequent patches, and hence this dependency propagates throughout the whole kernel. Add the `allocator_test` module that provides an empty implementation for all `Allocator`s in the kernel, such that we don't break the `rusttest` make target in subsequent patches. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-8-dakr@kernel.org [ Added missing `_old_layout` parameter as discussed. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 9 +++++++-- rust/kernel/alloc/allocator_test.rs | 20 ++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 rust/kernel/alloc/allocator_test.rs diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 2203852c37121b..b5605aab182d3d 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -2,12 +2,17 @@ //! Extensions to the [`alloc`] crate. -#[cfg(not(test))] -#[cfg(not(testlib))] +#[cfg(not(any(test, testlib)))] pub mod allocator; pub mod box_ext; pub mod vec_ext; +#[cfg(any(test, testlib))] +pub mod allocator_test; + +#[cfg(any(test, testlib))] +pub use self::allocator_test as allocator; + /// Indicates an allocation error. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct AllocError; diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs new file mode 100644 index 00000000000000..c5d325506f0c7b --- /dev/null +++ b/rust/kernel/alloc/allocator_test.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#![allow(missing_docs)] + +use super::{AllocError, Allocator, Flags}; +use core::alloc::Layout; +use core::ptr::NonNull; + +pub struct Kmalloc; + +unsafe impl Allocator for Kmalloc { + unsafe fn realloc( + _ptr: Option>, + _layout: Layout, + _old_layout: Layout, + _flags: Flags, + ) -> Result, AllocError> { + panic!(); + } +} From d8416d94047a65eb1ea02cebad47d7063f509146 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:12 +0200 Subject: [PATCH 05/51] rust: alloc: implement `Vmalloc` allocator Implement `Allocator` for `Vmalloc`, the kernel's virtually contiguous allocator, typically used for larger objects, (much) larger than page size. All memory allocations made with `Vmalloc` end up in `vrealloc()`. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-9-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/helpers/helpers.c | 1 + rust/helpers/vmalloc.c | 9 +++++++ rust/kernel/alloc/allocator.rs | 37 +++++++++++++++++++++++++++++ rust/kernel/alloc/allocator_test.rs | 1 + 4 files changed, 48 insertions(+) create mode 100644 rust/helpers/vmalloc.c diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 30f40149f3a969..20a0c69d5cc7b8 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -22,5 +22,6 @@ #include "spinlock.c" #include "task.c" #include "uaccess.c" +#include "vmalloc.c" #include "wait.c" #include "workqueue.c" diff --git a/rust/helpers/vmalloc.c b/rust/helpers/vmalloc.c new file mode 100644 index 00000000000000..80d34501bbc010 --- /dev/null +++ b/rust/helpers/vmalloc.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void * __must_check __realloc_size(2) +rust_helper_vrealloc(const void *p, size_t size, gfp_t flags) +{ + return vrealloc(p, size, flags); +} diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index b1894c800448bd..5742ce7d04535c 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -15,6 +15,7 @@ use core::ptr::NonNull; use crate::alloc::{AllocError, Allocator}; use crate::bindings; +use crate::pr_warn; /// The contiguous kernel allocator. /// @@ -24,6 +25,15 @@ use crate::bindings; /// For more details see [self]. pub struct Kmalloc; +/// The virtually contiguous kernel allocator. +/// +/// `Vmalloc` allocates pages from the page level allocator and maps them into the contiguous kernel +/// virtual space. It is typically used for large allocations. The memory allocated with this +/// allocator is not physically contiguous. +/// +/// For more details see [self]. +pub struct Vmalloc; + /// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment. fn aligned_size(new_layout: Layout) -> usize { // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first. @@ -63,6 +73,9 @@ impl ReallocFunc { // INVARIANT: `krealloc` satisfies the type invariants. const KREALLOC: Self = Self(bindings::krealloc); + // INVARIANT: `vrealloc` satisfies the type invariants. + const VREALLOC: Self = Self(bindings::vrealloc); + /// # Safety /// /// This method has the same safety requirements as [`Allocator::realloc`]. @@ -168,6 +181,30 @@ unsafe impl GlobalAlloc for Kmalloc { } } +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for Vmalloc { + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError> { + // TODO: Support alignments larger than PAGE_SIZE. + if layout.align() > bindings::PAGE_SIZE { + pr_warn!("Vmalloc does not support alignments larger than PAGE_SIZE yet.\n"); + return Err(AllocError); + } + + // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously + // allocated with this `Allocator`. + unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags) } + } +} + #[global_allocator] static ALLOCATOR: Kmalloc = Kmalloc; diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs index c5d325506f0c7b..32a1450c4d3920 100644 --- a/rust/kernel/alloc/allocator_test.rs +++ b/rust/kernel/alloc/allocator_test.rs @@ -7,6 +7,7 @@ use core::alloc::Layout; use core::ptr::NonNull; pub struct Kmalloc; +pub type Vmalloc = Kmalloc; unsafe impl Allocator for Kmalloc { unsafe fn realloc( From 2e3b351d0a11e8a7e446f04e5e2113b6c507c582 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:13 +0200 Subject: [PATCH 06/51] rust: alloc: implement `KVmalloc` allocator Implement `Allocator` for `KVmalloc`, an `Allocator` that tries to allocate memory with `kmalloc` first and, on failure, falls back to `vmalloc`. All memory allocations made with `KVmalloc` end up in `kvrealloc_noprof()`; all frees in `kvfree()`. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-10-dakr@kernel.org [ Reworded typo. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/helpers/slab.c | 6 +++++ rust/kernel/alloc/allocator.rs | 36 +++++++++++++++++++++++++++++ rust/kernel/alloc/allocator_test.rs | 1 + 3 files changed, 43 insertions(+) diff --git a/rust/helpers/slab.c b/rust/helpers/slab.c index f043e087f9d666..a842bfbddcba91 100644 --- a/rust/helpers/slab.c +++ b/rust/helpers/slab.c @@ -7,3 +7,9 @@ rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) { return krealloc(objp, new_size, flags); } + +void * __must_check __realloc_size(2) +rust_helper_kvrealloc(const void *p, size_t size, gfp_t flags) +{ + return kvrealloc(p, size, flags); +} diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 5742ce7d04535c..d9043ca42ce50a 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -34,6 +34,15 @@ pub struct Kmalloc; /// For more details see [self]. pub struct Vmalloc; +/// The kvmalloc kernel allocator. +/// +/// `KVmalloc` attempts to allocate memory with `Kmalloc` first, but falls back to `Vmalloc` upon +/// failure. This allocator is typically used when the size for the requested allocation is not +/// known and may exceed the capabilities of `Kmalloc`. +/// +/// For more details see [self]. +pub struct KVmalloc; + /// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment. fn aligned_size(new_layout: Layout) -> usize { // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first. @@ -76,6 +85,9 @@ impl ReallocFunc { // INVARIANT: `vrealloc` satisfies the type invariants. const VREALLOC: Self = Self(bindings::vrealloc); + // INVARIANT: `kvrealloc` satisfies the type invariants. + const KVREALLOC: Self = Self(bindings::kvrealloc); + /// # Safety /// /// This method has the same safety requirements as [`Allocator::realloc`]. @@ -205,6 +217,30 @@ unsafe impl Allocator for Vmalloc { } } +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for KVmalloc { + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError> { + // TODO: Support alignments larger than PAGE_SIZE. + if layout.align() > bindings::PAGE_SIZE { + pr_warn!("KVmalloc does not support alignments larger than PAGE_SIZE yet.\n"); + return Err(AllocError); + } + + // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously + // allocated with this `Allocator`. + unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags) } + } +} + #[global_allocator] static ALLOCATOR: Kmalloc = Kmalloc; diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs index 32a1450c4d3920..bd0cbcd93e5283 100644 --- a/rust/kernel/alloc/allocator_test.rs +++ b/rust/kernel/alloc/allocator_test.rs @@ -8,6 +8,7 @@ use core::ptr::NonNull; pub struct Kmalloc; pub type Vmalloc = Kmalloc; +pub type KVmalloc = Kmalloc; unsafe impl Allocator for Kmalloc { unsafe fn realloc( From 51c3a4deb83738c749f9dbea49b143aaea6542bc Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:14 +0200 Subject: [PATCH 07/51] rust: alloc: add __GFP_NOWARN to `Flags` Some test cases in subsequent patches provoke allocation failures. Add `__GFP_NOWARN` to enable test cases to silence unpleasant warnings. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-11-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/bindings/bindings_helper.h | 1 + rust/kernel/alloc.rs | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index ae82e9c941afa1..a80783fcbe042a 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -31,4 +31,5 @@ const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT; const gfp_t RUST_CONST_HELPER_GFP_NOWAIT = GFP_NOWAIT; const gfp_t RUST_CONST_HELPER___GFP_ZERO = __GFP_ZERO; const gfp_t RUST_CONST_HELPER___GFP_HIGHMEM = ___GFP_HIGHMEM; +const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index b5605aab182d3d..8172106a1423ef 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -91,6 +91,11 @@ pub mod flags { /// use any filesystem callback. It is very likely to fail to allocate memory, even for very /// small allocations. pub const GFP_NOWAIT: Flags = Flags(bindings::GFP_NOWAIT); + + /// Suppresses allocation failure reports. + /// + /// This is normally or'd with other flags. + pub const __GFP_NOWARN: Flags = Flags(bindings::__GFP_NOWARN); } /// The kernel's [`Allocator`] trait. From 89ec226664aa5408adf24354f71f5f7c77923de4 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:15 +0200 Subject: [PATCH 08/51] rust: alloc: implement kernel `Box` `Box` provides the simplest way to allocate memory for a generic type with one of the kernel's allocators, e.g. `Kmalloc`, `Vmalloc` or `KVmalloc`. In contrast to Rust's `Box` type, the kernel `Box` type considers the kernel's GFP flags for all appropriate functions, always reports allocation failures through `Result<_, AllocError>` and remains independent from unstable features. Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-12-dakr@kernel.org [ Added backticks, fixed typos. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 6 + rust/kernel/alloc/kbox.rs | 456 ++++++++++++++++++++++++++++++++++++++ rust/kernel/prelude.rs | 2 +- 3 files changed, 463 insertions(+), 1 deletion(-) create mode 100644 rust/kernel/alloc/kbox.rs diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 8172106a1423ef..3628e16240a69a 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -5,6 +5,7 @@ #[cfg(not(any(test, testlib)))] pub mod allocator; pub mod box_ext; +pub mod kbox; pub mod vec_ext; #[cfg(any(test, testlib))] @@ -13,6 +14,11 @@ pub mod allocator_test; #[cfg(any(test, testlib))] pub use self::allocator_test as allocator; +pub use self::kbox::Box; +pub use self::kbox::KBox; +pub use self::kbox::KVBox; +pub use self::kbox::VBox; + /// Indicates an allocation error. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct AllocError; diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs new file mode 100644 index 00000000000000..d69c32496b86a2 --- /dev/null +++ b/rust/kernel/alloc/kbox.rs @@ -0,0 +1,456 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of [`Box`]. + +#[allow(unused_imports)] // Used in doc comments. +use super::allocator::{KVmalloc, Kmalloc, Vmalloc}; +use super::{AllocError, Allocator, Flags}; +use core::alloc::Layout; +use core::fmt; +use core::marker::PhantomData; +use core::mem::ManuallyDrop; +use core::mem::MaybeUninit; +use core::ops::{Deref, DerefMut}; +use core::pin::Pin; +use core::ptr::NonNull; +use core::result::Result; + +use crate::init::{InPlaceInit, InPlaceWrite, Init, PinInit}; +use crate::types::ForeignOwnable; + +/// The kernel's [`Box`] type -- a heap allocation for a single value of type `T`. +/// +/// This is the kernel's version of the Rust stdlib's `Box`. There are several differences, +/// for example no `noalias` attribute is emitted and partially moving out of a `Box` is not +/// supported. There are also several API differences, e.g. `Box` always requires an [`Allocator`] +/// implementation to be passed as generic, page [`Flags`] when allocating memory and all functions +/// that may allocate memory are fallible. +/// +/// `Box` works with any of the kernel's allocators, e.g. [`Kmalloc`], [`Vmalloc`] or [`KVmalloc`]. +/// There are aliases for `Box` with these allocators ([`KBox`], [`VBox`], [`KVBox`]). +/// +/// When dropping a [`Box`], the value is also dropped and the heap memory is automatically freed. +/// +/// # Examples +/// +/// ``` +/// let b = KBox::::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// ``` +/// # use kernel::bindings; +/// const SIZE: usize = bindings::KMALLOC_MAX_SIZE as usize + 1; +/// struct Huge([u8; SIZE]); +/// +/// assert!(KBox::::new_uninit(GFP_KERNEL | __GFP_NOWARN).is_err()); +/// ``` +/// +/// ``` +/// # use kernel::bindings; +/// const SIZE: usize = bindings::KMALLOC_MAX_SIZE as usize + 1; +/// struct Huge([u8; SIZE]); +/// +/// assert!(KVBox::::new_uninit(GFP_KERNEL).is_ok()); +/// ``` +/// +/// # Invariants +/// +/// `self.0` is always properly aligned and either points to memory allocated with `A` or, for +/// zero-sized types, is a dangling, well aligned pointer. +#[repr(transparent)] +pub struct Box(NonNull, PhantomData); + +/// Type alias for [`Box`] with a [`Kmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = KBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type KBox = Box; + +/// Type alias for [`Box`] with a [`Vmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = VBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type VBox = Box; + +/// Type alias for [`Box`] with a [`KVmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = KVBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type KVBox = Box; + +// SAFETY: `Box` is `Send` if `T` is `Send` because the `Box` owns a `T`. +unsafe impl Send for Box +where + T: Send + ?Sized, + A: Allocator, +{ +} + +// SAFETY: `Box` is `Sync` if `T` is `Sync` because the `Box` owns a `T`. +unsafe impl Sync for Box +where + T: Sync + ?Sized, + A: Allocator, +{ +} + +impl Box +where + T: ?Sized, + A: Allocator, +{ + /// Creates a new `Box` from a raw pointer. + /// + /// # Safety + /// + /// For non-ZSTs, `raw` must point at an allocation allocated with `A` that is sufficiently + /// aligned for and holds a valid `T`. The caller passes ownership of the allocation to the + /// `Box`. + /// + /// For ZSTs, `raw` must be a dangling, well aligned pointer. + #[inline] + pub const unsafe fn from_raw(raw: *mut T) -> Self { + // INVARIANT: Validity of `raw` is guaranteed by the safety preconditions of this function. + // SAFETY: By the safety preconditions of this function, `raw` is not a NULL pointer. + Self(unsafe { NonNull::new_unchecked(raw) }, PhantomData) + } + + /// Consumes the `Box` and returns a raw pointer. + /// + /// This will not run the destructor of `T` and for non-ZSTs the allocation will stay alive + /// indefinitely. Use [`Box::from_raw`] to recover the [`Box`], drop the value and free the + /// allocation, if any. + /// + /// # Examples + /// + /// ``` + /// let x = KBox::new(24, GFP_KERNEL)?; + /// let ptr = KBox::into_raw(x); + /// // SAFETY: `ptr` comes from a previous call to `KBox::into_raw`. + /// let x = unsafe { KBox::from_raw(ptr) }; + /// + /// assert_eq!(*x, 24); + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + pub fn into_raw(b: Self) -> *mut T { + ManuallyDrop::new(b).0.as_ptr() + } + + /// Consumes and leaks the `Box` and returns a mutable reference. + /// + /// See [`Box::into_raw`] for more details. + #[inline] + pub fn leak<'a>(b: Self) -> &'a mut T { + // SAFETY: `Box::into_raw` always returns a properly aligned and dereferenceable pointer + // which points to an initialized instance of `T`. + unsafe { &mut *Box::into_raw(b) } + } +} + +impl Box, A> +where + A: Allocator, +{ + /// Converts a `Box, A>` to a `Box`. + /// + /// It is undefined behavior to call this function while the value inside of `b` is not yet + /// fully initialized. + /// + /// # Safety + /// + /// Callers must ensure that the value inside of `b` is in an initialized state. + pub unsafe fn assume_init(self) -> Box { + let raw = Self::into_raw(self); + + // SAFETY: `raw` comes from a previous call to `Box::into_raw`. By the safety requirements + // of this function, the value inside the `Box` is in an initialized state. Hence, it is + // safe to reconstruct the `Box` as `Box`. + unsafe { Box::from_raw(raw.cast()) } + } + + /// Writes the value and converts to `Box`. + pub fn write(mut self, value: T) -> Box { + (*self).write(value); + + // SAFETY: We've just initialized `b`'s value. + unsafe { self.assume_init() } + } +} + +impl Box +where + A: Allocator, +{ + /// Creates a new `Box` and initializes its contents with `x`. + /// + /// New memory is allocated with `A`. The allocation may fail, in which case an error is + /// returned. For ZSTs no memory is allocated. + pub fn new(x: T, flags: Flags) -> Result { + let b = Self::new_uninit(flags)?; + Ok(Box::write(b, x)) + } + + /// Creates a new `Box` with uninitialized contents. + /// + /// New memory is allocated with `A`. The allocation may fail, in which case an error is + /// returned. For ZSTs no memory is allocated. + /// + /// # Examples + /// + /// ``` + /// let b = KBox::::new_uninit(GFP_KERNEL)?; + /// let b = KBox::write(b, 24); + /// + /// assert_eq!(*b, 24_u64); + /// # Ok::<(), Error>(()) + /// ``` + pub fn new_uninit(flags: Flags) -> Result, A>, AllocError> { + let layout = Layout::new::>(); + let ptr = A::alloc(layout, flags)?; + + // INVARIANT: `ptr` is either a dangling pointer or points to memory allocated with `A`, + // which is sufficient in size and alignment for storing a `T`. + Ok(Box(ptr.cast(), PhantomData)) + } + + /// Constructs a new `Pin>`. If `T` does not implement [`Unpin`], then `x` will be + /// pinned in memory and can't be moved. + #[inline] + pub fn pin(x: T, flags: Flags) -> Result>, AllocError> + where + A: 'static, + { + Ok(Self::new(x, flags)?.into()) + } + + /// Forgets the contents (does not run the destructor), but keeps the allocation. + fn forget_contents(this: Self) -> Box, A> { + let ptr = Self::into_raw(this); + + // SAFETY: `ptr` is valid, because it came from `Box::into_raw`. + unsafe { Box::from_raw(ptr.cast()) } + } + + /// Drops the contents, but keeps the allocation. + /// + /// # Examples + /// + /// ``` + /// let value = KBox::new([0; 32], GFP_KERNEL)?; + /// assert_eq!(*value, [0; 32]); + /// let value = KBox::drop_contents(value); + /// // Now we can re-use `value`: + /// let value = KBox::write(value, [1; 32]); + /// assert_eq!(*value, [1; 32]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn drop_contents(this: Self) -> Box, A> { + let ptr = this.0.as_ptr(); + + // SAFETY: `ptr` is valid, because it came from `this`. After this call we never access the + // value stored in `this` again. + unsafe { core::ptr::drop_in_place(ptr) }; + + Self::forget_contents(this) + } + + /// Moves the `Box`'s value out of the `Box` and consumes the `Box`. + pub fn into_inner(b: Self) -> T { + // SAFETY: By the type invariant `&*b` is valid for `read`. + let value = unsafe { core::ptr::read(&*b) }; + let _ = Self::forget_contents(b); + value + } +} + +impl From> for Pin> +where + T: ?Sized, + A: Allocator, +{ + /// Converts a `Box` into a `Pin>`. If `T` does not implement [`Unpin`], then + /// `*b` will be pinned in memory and can't be moved. + /// + /// This moves `b` into `Pin` without moving `*b` or allocating and copying any memory. + fn from(b: Box) -> Self { + // SAFETY: The value wrapped inside a `Pin>` cannot be moved or replaced as long + // as `T` does not implement `Unpin`. + unsafe { Pin::new_unchecked(b) } + } +} + +impl InPlaceWrite for Box, A> +where + A: Allocator + 'static, +{ + type Initialized = Box; + + fn write_init(mut self, init: impl Init) -> Result { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { Box::assume_init(self) }) + } + + fn write_pin_init(mut self, init: impl PinInit) -> Result, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__pinned_init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { Box::assume_init(self) }.into()) + } +} + +impl InPlaceInit for Box +where + A: Allocator + 'static, +{ + type PinnedSelf = Pin; + + #[inline] + fn try_pin_init(init: impl PinInit, flags: Flags) -> Result, E> + where + E: From, + { + Box::<_, A>::new_uninit(flags)?.write_pin_init(init) + } + + #[inline] + fn try_init(init: impl Init, flags: Flags) -> Result + where + E: From, + { + Box::<_, A>::new_uninit(flags)?.write_init(init) + } +} + +impl ForeignOwnable for Box +where + A: Allocator, +{ + type Borrowed<'a> = &'a T; + + fn into_foreign(self) -> *const core::ffi::c_void { + Box::into_raw(self) as _ + } + + unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + unsafe { Box::from_raw(ptr as _) } + } + + unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> &'a T { + // SAFETY: The safety requirements of this method ensure that the object remains alive and + // immutable for the duration of 'a. + unsafe { &*ptr.cast() } + } +} + +impl ForeignOwnable for Pin> +where + A: Allocator, +{ + type Borrowed<'a> = Pin<&'a T>; + + fn into_foreign(self) -> *const core::ffi::c_void { + // SAFETY: We are still treating the box as pinned. + Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }) as _ + } + + unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + unsafe { Pin::new_unchecked(Box::from_raw(ptr as _)) } + } + + unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> Pin<&'a T> { + // SAFETY: The safety requirements for this function ensure that the object is still alive, + // so it is safe to dereference the raw pointer. + // The safety requirements of `from_foreign` also ensure that the object remains alive for + // the lifetime of the returned value. + let r = unsafe { &*ptr.cast() }; + + // SAFETY: This pointer originates from a `Pin>`. + unsafe { Pin::new_unchecked(r) } + } +} + +impl Deref for Box +where + T: ?Sized, + A: Allocator, +{ + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: `self.0` is always properly aligned, dereferenceable and points to an initialized + // instance of `T`. + unsafe { self.0.as_ref() } + } +} + +impl DerefMut for Box +where + T: ?Sized, + A: Allocator, +{ + fn deref_mut(&mut self) -> &mut T { + // SAFETY: `self.0` is always properly aligned, dereferenceable and points to an initialized + // instance of `T`. + unsafe { self.0.as_mut() } + } +} + +impl fmt::Debug for Box +where + T: ?Sized + fmt::Debug, + A: Allocator, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl Drop for Box +where + T: ?Sized, + A: Allocator, +{ + fn drop(&mut self) { + let layout = Layout::for_value::(self); + + // SAFETY: The pointer in `self.0` is guaranteed to be valid by the type invariant. + unsafe { core::ptr::drop_in_place::(self.deref_mut()) }; + + // SAFETY: + // - `self.0` was previously allocated with `A`. + // - `layout` is equal to the `Layout´ `self.0` was allocated with. + unsafe { A::free(self.0.cast(), layout) }; + } +} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 4571daec0961bb..a9210634a8c30d 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -14,7 +14,7 @@ #[doc(no_inline)] pub use core::pin::Pin; -pub use crate::alloc::{box_ext::BoxExt, flags::*, vec_ext::VecExt}; +pub use crate::alloc::{box_ext::BoxExt, flags::*, vec_ext::VecExt, KBox, KVBox, VBox}; #[doc(no_inline)] pub use alloc::{boxed::Box, vec::Vec}; From 5b36d1f304c0b4fc4fc36eac60ea0196d2129573 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:16 +0200 Subject: [PATCH 09/51] rust: treewide: switch to our kernel `Box` type Now that we got the kernel `Box` type in place, convert all existing `Box` users to make use of it. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-13-dakr@kernel.org Signed-off-by: Miguel Ojeda --- drivers/block/rnull.rs | 4 +-- rust/kernel/init.rs | 51 ++++++++++++++++--------------- rust/kernel/init/__internal.rs | 2 +- rust/kernel/rbtree.rs | 49 ++++++++++++++++------------- rust/kernel/sync/arc.rs | 17 +++++------ rust/kernel/sync/condvar.rs | 4 +-- rust/kernel/sync/lock/mutex.rs | 2 +- rust/kernel/sync/lock/spinlock.rs | 2 +- rust/kernel/workqueue.rs | 20 ++++++------ rust/macros/lib.rs | 6 ++-- 10 files changed, 81 insertions(+), 76 deletions(-) diff --git a/drivers/block/rnull.rs b/drivers/block/rnull.rs index b0227cf9ddd387..5de7223beb4d5b 100644 --- a/drivers/block/rnull.rs +++ b/drivers/block/rnull.rs @@ -32,7 +32,7 @@ module! { } struct NullBlkModule { - _disk: Pin>>>, + _disk: Pin>>>, } impl kernel::Module for NullBlkModule { @@ -47,7 +47,7 @@ impl kernel::Module for NullBlkModule { .rotational(false) .build(format_args!("rnullb{}", 0), tagset)?; - let disk = Box::pin_init(new_mutex!(disk, "nullb:disk"), flags::GFP_KERNEL)?; + let disk = KBox::pin_init(new_mutex!(disk, "nullb:disk"), flags::GFP_KERNEL)?; Ok(Self { _disk: disk }) } diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 25057cbed40b5f..c889f2640b56a8 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -13,7 +13,7 @@ //! To initialize a `struct` with an in-place constructor you will need two things: //! - an in-place constructor, //! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc`], -//! [`UniqueArc`], [`Box`] or any other smart pointer that implements [`InPlaceInit`]). +//! [`UniqueArc`], [`KBox`] or any other smart pointer that implements [`InPlaceInit`]). //! //! To get an in-place constructor there are generally three options: //! - directly creating an in-place constructor using the [`pin_init!`] macro, @@ -68,7 +68,7 @@ //! # a <- new_mutex!(42, "Foo::a"), //! # b: 24, //! # }); -//! let foo: Result>> = Box::pin_init(foo, GFP_KERNEL); +//! let foo: Result>> = KBox::pin_init(foo, GFP_KERNEL); //! ``` //! //! For more information see the [`pin_init!`] macro. @@ -92,14 +92,14 @@ //! struct DriverData { //! #[pin] //! status: Mutex, -//! buffer: Box<[u8; 1_000_000]>, +//! buffer: KBox<[u8; 1_000_000]>, //! } //! //! impl DriverData { //! fn new() -> impl PinInit { //! try_pin_init!(Self { //! status <- new_mutex!(0, "DriverData::status"), -//! buffer: Box::init(kernel::init::zeroed(), GFP_KERNEL)?, +//! buffer: KBox::init(kernel::init::zeroed(), GFP_KERNEL)?, //! }) //! } //! } @@ -211,7 +211,7 @@ //! [`pin_init!`]: crate::pin_init! use crate::{ - alloc::{box_ext::BoxExt, AllocError, Flags}, + alloc::{box_ext::BoxExt, AllocError, Flags, KBox}, error::{self, Error}, sync::Arc, sync::UniqueArc, @@ -298,7 +298,7 @@ macro_rules! stack_pin_init { /// struct Foo { /// #[pin] /// a: Mutex, -/// b: Box, +/// b: KBox, /// } /// /// struct Bar { @@ -307,7 +307,7 @@ macro_rules! stack_pin_init { /// /// stack_try_pin_init!(let foo: Result, AllocError> = pin_init!(Foo { /// a <- new_mutex!(42), -/// b: Box::new(Bar { +/// b: KBox::new(Bar { /// x: 64, /// }, GFP_KERNEL)?, /// })); @@ -324,7 +324,7 @@ macro_rules! stack_pin_init { /// struct Foo { /// #[pin] /// a: Mutex, -/// b: Box, +/// b: KBox, /// } /// /// struct Bar { @@ -333,7 +333,7 @@ macro_rules! stack_pin_init { /// /// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo { /// a <- new_mutex!(42), -/// b: Box::new(Bar { +/// b: KBox::new(Bar { /// x: 64, /// }, GFP_KERNEL)?, /// })); @@ -391,7 +391,7 @@ macro_rules! stack_try_pin_init { /// }, /// }); /// # initializer } -/// # Box::pin_init(demo(), GFP_KERNEL).unwrap(); +/// # KBox::pin_init(demo(), GFP_KERNEL).unwrap(); /// ``` /// /// Arbitrary Rust expressions can be used to set the value of a variable. @@ -460,7 +460,7 @@ macro_rules! stack_try_pin_init { /// # }) /// # } /// # } -/// let foo = Box::pin_init(Foo::new(), GFP_KERNEL); +/// let foo = KBox::pin_init(Foo::new(), GFP_KERNEL); /// ``` /// /// They can also easily embed it into their own `struct`s: @@ -592,7 +592,7 @@ macro_rules! pin_init { /// use kernel::{init::{self, PinInit}, error::Error}; /// #[pin_data] /// struct BigBuf { -/// big: Box<[u8; 1024 * 1024 * 1024]>, +/// big: KBox<[u8; 1024 * 1024 * 1024]>, /// small: [u8; 1024 * 1024], /// ptr: *mut u8, /// } @@ -600,7 +600,7 @@ macro_rules! pin_init { /// impl BigBuf { /// fn new() -> impl PinInit { /// try_pin_init!(Self { -/// big: Box::init(init::zeroed(), GFP_KERNEL)?, +/// big: KBox::init(init::zeroed(), GFP_KERNEL)?, /// small: [0; 1024 * 1024], /// ptr: core::ptr::null_mut(), /// }? Error) @@ -692,16 +692,16 @@ macro_rules! init { /// # Examples /// /// ```rust -/// use kernel::{init::{PinInit, zeroed}, error::Error}; +/// use kernel::{alloc::KBox, init::{PinInit, zeroed}, error::Error}; /// struct BigBuf { -/// big: Box<[u8; 1024 * 1024 * 1024]>, +/// big: KBox<[u8; 1024 * 1024 * 1024]>, /// small: [u8; 1024 * 1024], /// } /// /// impl BigBuf { /// fn new() -> impl Init { /// try_init!(Self { -/// big: Box::init(zeroed(), GFP_KERNEL)?, +/// big: KBox::init(zeroed(), GFP_KERNEL)?, /// small: [0; 1024 * 1024], /// }? Error) /// } @@ -812,8 +812,8 @@ macro_rules! assert_pinned { /// A pin-initializer for the type `T`. /// /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can -/// be [`Box`], [`Arc`], [`UniqueArc`] or even the stack (see [`stack_pin_init!`]). Use the -/// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc`] on this. +/// be [`KBox`], [`Arc`], [`UniqueArc`] or even the stack (see [`stack_pin_init!`]). Use +/// the [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc`] on this. /// /// Also see the [module description](self). /// @@ -893,7 +893,7 @@ pub unsafe trait PinInit: Sized { } /// An initializer returned by [`PinInit::pin_chain`]. -pub struct ChainPinInit(I, F, __internal::Invariant<(E, Box)>); +pub struct ChainPinInit(I, F, __internal::Invariant<(E, KBox)>); // SAFETY: The `__pinned_init` function is implemented such that it // - returns `Ok(())` on successful initialization, @@ -919,8 +919,8 @@ where /// An initializer for `T`. /// /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can -/// be [`Box`], [`Arc`], [`UniqueArc`] or even the stack (see [`stack_pin_init!`]). Use the -/// [`InPlaceInit::init`] function of a smart pointer like [`Arc`] on this. Because +/// be [`KBox`], [`Arc`], [`UniqueArc`] or even the stack (see [`stack_pin_init!`]). Use +/// the [`InPlaceInit::init`] function of a smart pointer like [`Arc`] on this. Because /// [`PinInit`] is a super trait, you can use every function that takes it as well. /// /// Also see the [module description](self). @@ -992,7 +992,7 @@ pub unsafe trait Init: PinInit { } /// An initializer returned by [`Init::chain`]. -pub struct ChainInit(I, F, __internal::Invariant<(E, Box)>); +pub struct ChainInit(I, F, __internal::Invariant<(E, KBox)>); // SAFETY: The `__init` function is implemented such that it // - returns `Ok(())` on successful initialization, @@ -1076,8 +1076,9 @@ pub fn uninit() -> impl Init, E> { /// # Examples /// /// ```rust -/// use kernel::{error::Error, init::init_array_from_fn}; -/// let array: Box<[usize; 1_000]> = Box::init::(init_array_from_fn(|i| i), GFP_KERNEL).unwrap(); +/// use kernel::{alloc::KBox, error::Error, init::init_array_from_fn}; +/// let array: KBox<[usize; 1_000]> = +/// KBox::init::(init_array_from_fn(|i| i), GFP_KERNEL).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` pub fn init_array_from_fn( @@ -1453,7 +1454,7 @@ impl_zeroable! { // // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant. {} Option>, - {} Option>, + {} Option>, // SAFETY: `null` pointer is valid. // diff --git a/rust/kernel/init/__internal.rs b/rust/kernel/init/__internal.rs index 44431fba7aabf7..74329cc3262c05 100644 --- a/rust/kernel/init/__internal.rs +++ b/rust/kernel/init/__internal.rs @@ -105,7 +105,7 @@ pub unsafe trait InitData: Copy { } } -pub struct AllData(PhantomData) -> Box>); +pub struct AllData(PhantomData) -> KBox>); impl Clone for AllData { fn clone(&self) -> Self { diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs index d03e4aa1f4812b..cb4415a1225825 100644 --- a/rust/kernel/rbtree.rs +++ b/rust/kernel/rbtree.rs @@ -7,7 +7,6 @@ //! Reference: use crate::{alloc::Flags, bindings, container_of, error::Result, prelude::*}; -use alloc::boxed::Box; use core::{ cmp::{Ord, Ordering}, marker::PhantomData, @@ -497,7 +496,7 @@ impl Drop for RBTree { // but it is not observable. The loop invariant is still maintained. // SAFETY: `this` is valid per the loop invariant. - unsafe { drop(Box::from_raw(this.cast_mut())) }; + unsafe { drop(KBox::from_raw(this.cast_mut())) }; } } } @@ -764,7 +763,7 @@ impl<'a, K, V> Cursor<'a, K, V> { // point to the links field of `Node` objects. let this = unsafe { container_of!(self.current.as_ptr(), Node, links) }.cast_mut(); // SAFETY: `this` is valid by the type invariants as described above. - let node = unsafe { Box::from_raw(this) }; + let node = unsafe { KBox::from_raw(this) }; let node = RBTreeNode { node }; // SAFETY: The reference to the tree used to create the cursor outlives the cursor, so // the tree cannot change. By the tree invariant, all nodes are valid. @@ -809,7 +808,7 @@ impl<'a, K, V> Cursor<'a, K, V> { // point to the links field of `Node` objects. let this = unsafe { container_of!(neighbor, Node, links) }.cast_mut(); // SAFETY: `this` is valid by the type invariants as described above. - let node = unsafe { Box::from_raw(this) }; + let node = unsafe { KBox::from_raw(this) }; return Some(RBTreeNode { node }); } None @@ -1038,7 +1037,7 @@ impl Iterator for IterRaw { /// It contains the memory needed to hold a node that can be inserted into a red-black tree. One /// can be obtained by directly allocating it ([`RBTreeNodeReservation::new`]). pub struct RBTreeNodeReservation { - node: Box>>, + node: KBox>>, } impl RBTreeNodeReservation { @@ -1046,7 +1045,7 @@ impl RBTreeNodeReservation { /// call to [`RBTree::insert`]. pub fn new(flags: Flags) -> Result> { Ok(RBTreeNodeReservation { - node: as BoxExt<_>>::new_uninit(flags)?, + node: KBox::new_uninit(flags)?, }) } } @@ -1062,14 +1061,15 @@ impl RBTreeNodeReservation { /// Initialises a node reservation. /// /// It then becomes an [`RBTreeNode`] that can be inserted into a tree. - pub fn into_node(mut self, key: K, value: V) -> RBTreeNode { - self.node.write(Node { - key, - value, - links: bindings::rb_node::default(), - }); - // SAFETY: We just wrote to it. - let node = unsafe { self.node.assume_init() }; + pub fn into_node(self, key: K, value: V) -> RBTreeNode { + let node = KBox::write( + self.node, + Node { + key, + value, + links: bindings::rb_node::default(), + }, + ); RBTreeNode { node } } } @@ -1079,7 +1079,7 @@ impl RBTreeNodeReservation { /// The node is fully initialised (with key and value) and can be inserted into a tree without any /// extra allocations or failure paths. pub struct RBTreeNode { - node: Box>, + node: KBox>, } impl RBTreeNode { @@ -1091,7 +1091,9 @@ impl RBTreeNode { /// Get the key and value from inside the node. pub fn to_key_value(self) -> (K, V) { - (self.node.key, self.node.value) + let node = KBox::into_inner(self.node); + + (node.key, node.value) } } @@ -1113,7 +1115,7 @@ impl RBTreeNode { /// may be freed (but only for the key/value; memory for the node itself is kept for reuse). pub fn into_reservation(self) -> RBTreeNodeReservation { RBTreeNodeReservation { - node: Box::drop_contents(self.node), + node: KBox::drop_contents(self.node), } } } @@ -1164,7 +1166,7 @@ impl<'a, K, V> RawVacantEntry<'a, K, V> { /// The `node` must have a key such that inserting it here does not break the ordering of this /// [`RBTree`]. fn insert(self, node: RBTreeNode) -> &'a mut V { - let node = Box::into_raw(node.node); + let node = KBox::into_raw(node.node); // SAFETY: `node` is valid at least until we call `Box::from_raw`, which only happens when // the node is removed or replaced. @@ -1238,21 +1240,24 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> { // SAFETY: The node was a node in the tree, but we removed it, so we can convert it // back into a box. node: unsafe { - Box::from_raw(container_of!(self.node_links, Node, links).cast_mut()) + KBox::from_raw(container_of!(self.node_links, Node, links).cast_mut()) }, } } /// Takes the value of the entry out of the map, and returns it. pub fn remove(self) -> V { - self.remove_node().node.value + let rb_node = self.remove_node(); + let node = KBox::into_inner(rb_node.node); + + node.value } /// Swap the current node for the provided node. /// /// The key of both nodes must be equal. fn replace(self, node: RBTreeNode) -> RBTreeNode { - let node = Box::into_raw(node.node); + let node = KBox::into_raw(node.node); // SAFETY: `node` is valid at least until we call `Box::from_raw`, which only happens when // the node is removed or replaced. @@ -1268,7 +1273,7 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> { // - `self.node_ptr` produces a valid pointer to a node in the tree. // - Now that we removed this entry from the tree, we can convert the node to a box. let old_node = - unsafe { Box::from_raw(container_of!(self.node_links, Node, links).cast_mut()) }; + unsafe { KBox::from_raw(container_of!(self.node_links, Node, links).cast_mut()) }; RBTreeNode { node: old_node } } diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 9325cc5a16a4c4..db9da352d588f6 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -17,13 +17,12 @@ //! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html use crate::{ - alloc::{box_ext::BoxExt, AllocError, Flags}, + alloc::{AllocError, Flags, KBox}, bindings, init::{self, InPlaceInit, Init, PinInit}, try_init, types::{ForeignOwnable, Opaque}, }; -use alloc::boxed::Box; use core::{ alloc::Layout, fmt, @@ -201,11 +200,11 @@ impl Arc { data: contents, }; - let inner = as BoxExt<_>>::new(value, flags)?; + let inner = KBox::new(value, flags)?; // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new // `Arc` object. - Ok(unsafe { Self::from_inner(Box::leak(inner).into()) }) + Ok(unsafe { Self::from_inner(KBox::leak(inner).into()) }) } } @@ -398,8 +397,8 @@ impl Drop for Arc { if is_zero { // The count reached zero, we must free the memory. // - // SAFETY: The pointer was initialised from the result of `Box::leak`. - unsafe { drop(Box::from_raw(self.ptr.as_ptr())) }; + // SAFETY: The pointer was initialised from the result of `KBox::leak`. + unsafe { drop(KBox::from_raw(self.ptr.as_ptr())) }; } } } @@ -641,7 +640,7 @@ impl UniqueArc { /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet. pub fn new_uninit(flags: Flags) -> Result>, AllocError> { // INVARIANT: The refcount is initialised to a non-zero value. - let inner = Box::try_init::( + let inner = KBox::try_init::( try_init!(ArcInner { // SAFETY: There are no safety requirements for this FFI call. refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), @@ -651,8 +650,8 @@ impl UniqueArc { )?; Ok(UniqueArc { // INVARIANT: The newly-created object has a refcount of 1. - // SAFETY: The pointer from the `Box` is valid. - inner: unsafe { Arc::from_inner(Box::leak(inner).into()) }, + // SAFETY: The pointer from the `KBox` is valid. + inner: unsafe { Arc::from_inner(KBox::leak(inner).into()) }, }) } } diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs index dec2e5ffc91939..a1a29c0bdb3aa2 100644 --- a/rust/kernel/sync/condvar.rs +++ b/rust/kernel/sync/condvar.rs @@ -70,8 +70,8 @@ pub use new_condvar; /// } /// /// /// Allocates a new boxed `Example`. -/// fn new_example() -> Result>> { -/// Box::pin_init(pin_init!(Example { +/// fn new_example() -> Result>> { +/// KBox::pin_init(pin_init!(Example { /// value <- new_mutex!(0), /// value_changed <- new_condvar!(), /// }), GFP_KERNEL) diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index c4f3b6cbfe48de..9ce43ccb451583 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -58,7 +58,7 @@ pub use new_mutex; /// } /// /// // Allocate a boxed `Example`. -/// let e = Box::pin_init(Example::new(), GFP_KERNEL)?; +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; /// assert_eq!(e.c, 10); /// assert_eq!(e.d.lock().a, 20); /// assert_eq!(e.d.lock().b, 30); diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index c900ae23db76fe..040dc16975a68a 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -56,7 +56,7 @@ pub use new_spinlock; /// } /// /// // Allocate a boxed `Example`. -/// let e = Box::pin_init(Example::new(), GFP_KERNEL)?; +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; /// assert_eq!(e.c, 10); /// assert_eq!(e.d.lock().a, 20); /// assert_eq!(e.d.lock().b, 30); diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 10d2bc62e2cff7..4d1d2062f6eba5 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -216,7 +216,7 @@ impl Queue { func: Some(func), }); - self.enqueue(Box::pin_init(init, flags).map_err(|_| AllocError)?); + self.enqueue(KBox::pin_init(init, flags).map_err(|_| AllocError)?); Ok(()) } } @@ -239,9 +239,9 @@ impl ClosureWork { } impl WorkItem for ClosureWork { - type Pointer = Pin>; + type Pointer = Pin>; - fn run(mut this: Pin>) { + fn run(mut this: Pin>) { if let Some(func) = this.as_mut().project().take() { (func)() } @@ -297,7 +297,7 @@ pub unsafe trait RawWorkItem { /// Defines the method that should be called directly when a work item is executed. /// -/// This trait is implemented by `Pin>` and [`Arc`], and is mainly intended to be +/// This trait is implemented by `Pin>` and [`Arc`], and is mainly intended to be /// implemented for smart pointer types. For your own structs, you would implement [`WorkItem`] /// instead. The [`run`] method on this trait will usually just perform the appropriate /// `container_of` translation and then call into the [`run`][WorkItem::run] method from the @@ -329,7 +329,7 @@ pub unsafe trait WorkItemPointer: RawWorkItem { /// This trait is used when the `work_struct` field is defined using the [`Work`] helper. pub trait WorkItem { /// The pointer type that this struct is wrapped in. This will typically be `Arc` or - /// `Pin>`. + /// `Pin>`. type Pointer: WorkItemPointer; /// The method that should be called when this work item is executed. @@ -567,7 +567,7 @@ where } // SAFETY: TODO. -unsafe impl WorkItemPointer for Pin> +unsafe impl WorkItemPointer for Pin> where T: WorkItem, T: HasWork, @@ -578,7 +578,7 @@ where // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. let ptr = unsafe { T::work_container_of(ptr) }; // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. - let boxed = unsafe { Box::from_raw(ptr) }; + let boxed = unsafe { KBox::from_raw(ptr) }; // SAFETY: The box was already pinned when it was enqueued. let pinned = unsafe { Pin::new_unchecked(boxed) }; @@ -587,7 +587,7 @@ where } // SAFETY: TODO. -unsafe impl RawWorkItem for Pin> +unsafe impl RawWorkItem for Pin> where T: WorkItem, T: HasWork, @@ -601,9 +601,9 @@ where // SAFETY: We're not going to move `self` or any of its fields, so its okay to temporarily // remove the `Pin` wrapper. let boxed = unsafe { Pin::into_inner_unchecked(self) }; - let ptr = Box::into_raw(boxed); + let ptr = KBox::into_raw(boxed); - // SAFETY: Pointers into a `Box` point at a valid value. + // SAFETY: Pointers into a `KBox` point at a valid value. let work_ptr = unsafe { T::raw_get_work(ptr) }; // SAFETY: `raw_get_work` returns a pointer to a valid value. let work_ptr = unsafe { Work::raw_get(work_ptr) }; diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index a626b1145e5c4f..ab93111a048c90 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -243,7 +243,7 @@ pub fn concat_idents(ts: TokenStream) -> TokenStream { /// struct DriverData { /// #[pin] /// queue: Mutex>, -/// buf: Box<[u8; 1024 * 1024]>, +/// buf: KBox<[u8; 1024 * 1024]>, /// } /// ``` /// @@ -252,7 +252,7 @@ pub fn concat_idents(ts: TokenStream) -> TokenStream { /// struct DriverData { /// #[pin] /// queue: Mutex>, -/// buf: Box<[u8; 1024 * 1024]>, +/// buf: KBox<[u8; 1024 * 1024]>, /// raw_info: *mut Info, /// } /// @@ -282,7 +282,7 @@ pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream { /// struct DriverData { /// #[pin] /// queue: Mutex>, -/// buf: Box<[u8; 1024 * 1024]>, +/// buf: KBox<[u8; 1024 * 1024]>, /// raw_info: *mut Info, /// } /// From 90acde2a1b925e99580116e0bc6fd1fed081d5a2 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:17 +0200 Subject: [PATCH 10/51] rust: alloc: remove extension of std's `Box` Now that all existing `Box` users were moved to the kernel `Box` type, remove the `BoxExt` extension and all other related extensions. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-14-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 1 - rust/kernel/alloc/box_ext.rs | 89 ------------------------------------ rust/kernel/init.rs | 46 +------------------ rust/kernel/lib.rs | 1 - rust/kernel/prelude.rs | 4 +- rust/kernel/types.rs | 50 -------------------- 6 files changed, 3 insertions(+), 188 deletions(-) delete mode 100644 rust/kernel/alloc/box_ext.rs diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 3628e16240a69a..94f4de5e0cdc42 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -4,7 +4,6 @@ #[cfg(not(any(test, testlib)))] pub mod allocator; -pub mod box_ext; pub mod kbox; pub mod vec_ext; diff --git a/rust/kernel/alloc/box_ext.rs b/rust/kernel/alloc/box_ext.rs deleted file mode 100644 index 7009ad78d4e082..00000000000000 --- a/rust/kernel/alloc/box_ext.rs +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! Extensions to [`Box`] for fallible allocations. - -use super::{AllocError, Flags}; -use alloc::boxed::Box; -use core::{mem::MaybeUninit, ptr, result::Result}; - -/// Extensions to [`Box`]. -pub trait BoxExt: Sized { - /// Allocates a new box. - /// - /// The allocation may fail, in which case an error is returned. - fn new(x: T, flags: Flags) -> Result; - - /// Allocates a new uninitialised box. - /// - /// The allocation may fail, in which case an error is returned. - fn new_uninit(flags: Flags) -> Result>, AllocError>; - - /// Drops the contents, but keeps the allocation. - /// - /// # Examples - /// - /// ``` - /// use kernel::alloc::{flags, box_ext::BoxExt}; - /// let value = Box::new([0; 32], flags::GFP_KERNEL)?; - /// assert_eq!(*value, [0; 32]); - /// let mut value = Box::drop_contents(value); - /// // Now we can re-use `value`: - /// value.write([1; 32]); - /// // SAFETY: We just wrote to it. - /// let value = unsafe { value.assume_init() }; - /// assert_eq!(*value, [1; 32]); - /// # Ok::<(), Error>(()) - /// ``` - fn drop_contents(this: Self) -> Box>; -} - -impl BoxExt for Box { - fn new(x: T, flags: Flags) -> Result { - let mut b = >::new_uninit(flags)?; - b.write(x); - // SAFETY: We just wrote to it. - Ok(unsafe { b.assume_init() }) - } - - #[cfg(any(test, testlib))] - fn new_uninit(_flags: Flags) -> Result>, AllocError> { - Ok(Box::new_uninit()) - } - - #[cfg(not(any(test, testlib)))] - fn new_uninit(flags: Flags) -> Result>, AllocError> { - let ptr = if core::mem::size_of::>() == 0 { - core::ptr::NonNull::<_>::dangling().as_ptr() - } else { - let layout = core::alloc::Layout::new::>(); - - // SAFETY: Memory is being allocated (first arg is null). The only other source of - // safety issues is sleeping on atomic context, which is addressed by klint. Lastly, - // the type is not a SZT (checked above). - let ptr = - unsafe { super::allocator::krealloc_aligned(core::ptr::null_mut(), layout, flags) }; - if ptr.is_null() { - return Err(AllocError); - } - - ptr.cast::>() - }; - - // SAFETY: For non-zero-sized types, we allocate above using the global allocator. For - // zero-sized types, we use `NonNull::dangling`. - Ok(unsafe { Box::from_raw(ptr) }) - } - - fn drop_contents(this: Self) -> Box> { - let ptr = Box::into_raw(this); - // SAFETY: `ptr` is valid, because it came from `Box::into_raw`. - unsafe { ptr::drop_in_place(ptr) }; - - // CAST: `MaybeUninit` is a transparent wrapper of `T`. - let ptr = ptr.cast::>(); - - // SAFETY: `ptr` is valid for writes, because it came from `Box::into_raw` and it is valid for - // reads, since the pointer came from `Box::into_raw` and the type is `MaybeUninit`. - unsafe { Box::from_raw(ptr) } - } -} diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index c889f2640b56a8..c9919ba0b68368 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -211,13 +211,12 @@ //! [`pin_init!`]: crate::pin_init! use crate::{ - alloc::{box_ext::BoxExt, AllocError, Flags, KBox}, + alloc::{AllocError, Flags, KBox}, error::{self, Error}, sync::Arc, sync::UniqueArc, types::{Opaque, ScopeGuard}, }; -use alloc::boxed::Box; use core::{ cell::UnsafeCell, convert::Infallible, @@ -588,7 +587,6 @@ macro_rules! pin_init { /// # Examples /// /// ```rust -/// # #![feature(new_uninit)] /// use kernel::{init::{self, PinInit}, error::Error}; /// #[pin_data] /// struct BigBuf { @@ -1245,26 +1243,6 @@ impl InPlaceInit for Arc { } } -impl InPlaceInit for Box { - type PinnedSelf = Pin; - - #[inline] - fn try_pin_init(init: impl PinInit, flags: Flags) -> Result - where - E: From, - { - as BoxExt<_>>::new_uninit(flags)?.write_pin_init(init) - } - - #[inline] - fn try_init(init: impl Init, flags: Flags) -> Result - where - E: From, - { - as BoxExt<_>>::new_uninit(flags)?.write_init(init) - } -} - impl InPlaceInit for UniqueArc { type PinnedSelf = Pin; @@ -1301,28 +1279,6 @@ pub trait InPlaceWrite { fn write_pin_init(self, init: impl PinInit) -> Result, E>; } -impl InPlaceWrite for Box> { - type Initialized = Box; - - fn write_init(mut self, init: impl Init) -> Result { - let slot = self.as_mut_ptr(); - // SAFETY: When init errors/panics, slot will get deallocated but not dropped, - // slot is valid. - unsafe { init.__init(slot)? }; - // SAFETY: All fields have been initialized. - Ok(unsafe { self.assume_init() }) - } - - fn write_pin_init(mut self, init: impl PinInit) -> Result, E> { - let slot = self.as_mut_ptr(); - // SAFETY: When init errors/panics, slot will get deallocated but not dropped, - // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; - // SAFETY: All fields have been initialized. - Ok(unsafe { self.assume_init() }.into()) - } -} - impl InPlaceWrite for UniqueArc> { type Initialized = UniqueArc; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index dc37aef6a0085a..34e9151db58d2d 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -16,7 +16,6 @@ #![feature(coerce_unsized)] #![feature(dispatch_from_dyn)] #![feature(lint_reasons)] -#![feature(new_uninit)] #![feature(unsize)] // Ensure conditional compilation based on the kernel configuration works; diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index a9210634a8c30d..c1f8e5c832e2d7 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -14,10 +14,10 @@ #[doc(no_inline)] pub use core::pin::Pin; -pub use crate::alloc::{box_ext::BoxExt, flags::*, vec_ext::VecExt, KBox, KVBox, VBox}; +pub use crate::alloc::{flags::*, vec_ext::VecExt, KBox, KVBox, VBox}; #[doc(no_inline)] -pub use alloc::{boxed::Box, vec::Vec}; +pub use alloc::vec::Vec; #[doc(no_inline)] pub use macros::{module, pin_data, pinned_drop, vtable, Zeroable}; diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 085e8076f07883..34f1b31753df1c 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -3,13 +3,11 @@ //! Kernel types. use crate::init::{self, PinInit}; -use alloc::boxed::Box; use core::{ cell::UnsafeCell, marker::{PhantomData, PhantomPinned}, mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, - pin::Pin, ptr::NonNull, }; @@ -71,54 +69,6 @@ pub trait ForeignOwnable: Sized { } } -impl ForeignOwnable for Box { - type Borrowed<'a> = &'a T; - - fn into_foreign(self) -> *const core::ffi::c_void { - Box::into_raw(self) as _ - } - - unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> &'a T { - // SAFETY: The safety requirements for this function ensure that the object is still alive, - // so it is safe to dereference the raw pointer. - // The safety requirements of `from_foreign` also ensure that the object remains alive for - // the lifetime of the returned value. - unsafe { &*ptr.cast() } - } - - unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { - // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous - // call to `Self::into_foreign`. - unsafe { Box::from_raw(ptr as _) } - } -} - -impl ForeignOwnable for Pin> { - type Borrowed<'a> = Pin<&'a T>; - - fn into_foreign(self) -> *const core::ffi::c_void { - // SAFETY: We are still treating the box as pinned. - Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }) as _ - } - - unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> Pin<&'a T> { - // SAFETY: The safety requirements for this function ensure that the object is still alive, - // so it is safe to dereference the raw pointer. - // The safety requirements of `from_foreign` also ensure that the object remains alive for - // the lifetime of the returned value. - let r = unsafe { &*ptr.cast() }; - - // SAFETY: This pointer originates from a `Pin>`. - unsafe { Pin::new_unchecked(r) } - } - - unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { - // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous - // call to `Self::into_foreign`. - unsafe { Pin::new_unchecked(Box::from_raw(ptr as _)) } - } -} - impl ForeignOwnable for () { type Borrowed<'a> = (); From 48134b2321f35e5f064f4d830ee899a4aff3b918 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:18 +0200 Subject: [PATCH 11/51] rust: alloc: add `Box` to prelude Now that we removed `BoxExt` and the corresponding includes in prelude.rs, add the new kernel `Box` type instead. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-15-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/prelude.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index c1f8e5c832e2d7..d5f2fe42d0932f 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -14,7 +14,7 @@ #[doc(no_inline)] pub use core::pin::Pin; -pub use crate::alloc::{flags::*, vec_ext::VecExt, KBox, KVBox, VBox}; +pub use crate::alloc::{flags::*, vec_ext::VecExt, Box, KBox, KVBox, VBox}; #[doc(no_inline)] pub use alloc::vec::Vec; From b457a4950bf48dca354b2f99151f797329996222 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Fri, 4 Oct 2024 17:41:19 +0200 Subject: [PATCH 12/51] rust: alloc: introduce `ArrayLayout` When allocating memory for arrays using allocators, the `Layout::array` function is typically used. It returns a result, since the given size might be too big. However, `Vec` and its iterators store their allocated capacity and thus they already did check that the size is not too big. The `ArrayLayout` type provides this exact behavior, as it can be infallibly converted into a `Layout`. Instead of a `usize` capacity, `Vec` and other similar array-storing types can use `ArrayLayout` instead. Reviewed-by: Gary Guo Signed-off-by: Benno Lossin Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-16-dakr@kernel.org [ Formatted a few comments. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 1 + rust/kernel/alloc/layout.rs | 91 +++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 rust/kernel/alloc/layout.rs diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 94f4de5e0cdc42..c679d93b0523ac 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -5,6 +5,7 @@ #[cfg(not(any(test, testlib)))] pub mod allocator; pub mod kbox; +pub mod layout; pub mod vec_ext; #[cfg(any(test, testlib))] diff --git a/rust/kernel/alloc/layout.rs b/rust/kernel/alloc/layout.rs new file mode 100644 index 00000000000000..7e0c2f46157b77 --- /dev/null +++ b/rust/kernel/alloc/layout.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory layout. +//! +//! Custom layout types extending or improving [`Layout`]. + +use core::{alloc::Layout, marker::PhantomData}; + +/// Error when constructing an [`ArrayLayout`]. +pub struct LayoutError; + +/// A layout for an array `[T; n]`. +/// +/// # Invariants +/// +/// - `len * size_of::() <= isize::MAX`. +pub struct ArrayLayout { + len: usize, + _phantom: PhantomData T>, +} + +impl Clone for ArrayLayout { + fn clone(&self) -> Self { + *self + } +} +impl Copy for ArrayLayout {} + +const ISIZE_MAX: usize = isize::MAX as usize; + +impl ArrayLayout { + /// Creates a new layout for `[T; 0]`. + pub const fn empty() -> Self { + // INVARIANT: `0 * size_of::() <= isize::MAX`. + Self { + len: 0, + _phantom: PhantomData, + } + } + + /// Creates a new layout for `[T; len]`. + /// + /// # Errors + /// + /// When `len * size_of::()` overflows or when `len * size_of::() > isize::MAX`. + pub const fn new(len: usize) -> Result { + match len.checked_mul(core::mem::size_of::()) { + Some(len) if len <= ISIZE_MAX => { + // INVARIANT: We checked above that `len * size_of::() <= isize::MAX`. + Ok(Self { + len, + _phantom: PhantomData, + }) + } + _ => Err(LayoutError), + } + } + + /// Creates a new layout for `[T; len]`. + /// + /// # Safety + /// + /// `len` must be a value, for which `len * size_of::() <= isize::MAX` is true. + pub unsafe fn new_unchecked(len: usize) -> Self { + // INVARIANT: By the safety requirements of this function + // `len * size_of::() <= isize::MAX`. + Self { + len, + _phantom: PhantomData, + } + } + + /// Returns the number of array elements represented by this layout. + pub const fn len(&self) -> usize { + self.len + } + + /// Returns `true` when no array elements are represented by this layout. + pub const fn is_empty(&self) -> bool { + self.len == 0 + } +} + +impl From> for Layout { + fn from(value: ArrayLayout) -> Self { + let res = Layout::array::(value.len); + // SAFETY: By the type invariant of `ArrayLayout` we have + // `len * size_of::() <= isize::MAX` and thus the result must be `Ok`. + unsafe { res.unwrap_unchecked() } + } +} From 77b4b2460e31a8f03d5b6285894bec8eaf8f0f12 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:20 +0200 Subject: [PATCH 13/51] rust: alloc: implement kernel `Vec` type `Vec` provides a contiguous growable array type with contents allocated with the kernel's allocators (e.g. `Kmalloc`, `Vmalloc` or `KVmalloc`). In contrast to Rust's stdlib `Vec` type, the kernel `Vec` type considers the kernel's GFP flags for all appropriate functions, always reports allocation failures through `Result<_, AllocError>` and remains independent from unstable features. [ This patch starts using a new unstable feature, `inline_const`, but it was stabilized in Rust 1.79.0, i.e. the next version after the minimum one, thus it will not be an issue. - Miguel ] Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-17-dakr@kernel.org [ Cleaned `rustdoc` unescaped backtick warning, added a couple more backticks elsewhere, fixed typos, sorted `feature`s, rewrapped documentation line. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 6 + rust/kernel/alloc/kvec.rs | 649 ++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + rust/kernel/prelude.rs | 2 +- 4 files changed, 657 insertions(+), 1 deletion(-) create mode 100644 rust/kernel/alloc/kvec.rs diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index c679d93b0523ac..7f654b214ec249 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -5,6 +5,7 @@ #[cfg(not(any(test, testlib)))] pub mod allocator; pub mod kbox; +pub mod kvec; pub mod layout; pub mod vec_ext; @@ -19,6 +20,11 @@ pub use self::kbox::KBox; pub use self::kbox::KVBox; pub use self::kbox::VBox; +pub use self::kvec::KVVec; +pub use self::kvec::KVec; +pub use self::kvec::VVec; +pub use self::kvec::Vec; + /// Indicates an allocation error. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct AllocError; diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs new file mode 100644 index 00000000000000..ad64313a950c01 --- /dev/null +++ b/rust/kernel/alloc/kvec.rs @@ -0,0 +1,649 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of [`Vec`]. + +use super::{ + allocator::{KVmalloc, Kmalloc, Vmalloc}, + layout::ArrayLayout, + AllocError, Allocator, Box, Flags, +}; +use core::{ + fmt, + marker::PhantomData, + mem::{ManuallyDrop, MaybeUninit}, + ops::Deref, + ops::DerefMut, + ops::Index, + ops::IndexMut, + ptr, + ptr::NonNull, + slice, + slice::SliceIndex, +}; + +/// Create a [`KVec`] containing the arguments. +/// +/// New memory is allocated with `GFP_KERNEL`. +/// +/// # Examples +/// +/// ``` +/// let mut v = kernel::kvec![]; +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(v, [1]); +/// +/// let mut v = kernel::kvec![1; 3]?; +/// v.push(4, GFP_KERNEL)?; +/// assert_eq!(v, [1, 1, 1, 4]); +/// +/// let mut v = kernel::kvec![1, 2, 3]?; +/// v.push(4, GFP_KERNEL)?; +/// assert_eq!(v, [1, 2, 3, 4]); +/// +/// # Ok::<(), Error>(()) +/// ``` +#[macro_export] +macro_rules! kvec { + () => ( + $crate::alloc::KVec::new() + ); + ($elem:expr; $n:expr) => ( + $crate::alloc::KVec::from_elem($elem, $n, GFP_KERNEL) + ); + ($($x:expr),+ $(,)?) => ( + match $crate::alloc::KBox::new_uninit(GFP_KERNEL) { + Ok(b) => Ok($crate::alloc::KVec::from($crate::alloc::KBox::write(b, [$($x),+]))), + Err(e) => Err(e), + } + ); +} + +/// The kernel's [`Vec`] type. +/// +/// A contiguous growable array type with contents allocated with the kernel's allocators (e.g. +/// [`Kmalloc`], [`Vmalloc`] or [`KVmalloc`]), written `Vec`. +/// +/// For non-zero-sized values, a [`Vec`] will use the given allocator `A` for its allocation. For +/// the most common allocators the type aliases [`KVec`], [`VVec`] and [`KVVec`] exist. +/// +/// For zero-sized types the [`Vec`]'s pointer must be `dangling_mut::`; no memory is allocated. +/// +/// Generally, [`Vec`] consists of a pointer that represents the vector's backing buffer, the +/// capacity of the vector (the number of elements that currently fit into the vector), its length +/// (the number of elements that are currently stored in the vector) and the `Allocator` type used +/// to allocate (and free) the backing buffer. +/// +/// A [`Vec`] can be deconstructed into and (re-)constructed from its previously named raw parts +/// and manually modified. +/// +/// [`Vec`]'s backing buffer gets, if required, automatically increased (re-allocated) when elements +/// are added to the vector. +/// +/// # Invariants +/// +/// - `self.ptr` is always properly aligned and either points to memory allocated with `A` or, for +/// zero-sized types, is a dangling, well aligned pointer. +/// +/// - `self.len` always represents the exact number of elements stored in the vector. +/// +/// - `self.layout` represents the absolute number of elements that can be stored within the vector +/// without re-allocation. For ZSTs `self.layout`'s capacity is zero. However, it is legal for the +/// backing buffer to be larger than `layout`. +/// +/// - The `Allocator` type `A` of the vector is the exact same `Allocator` type the backing buffer +/// was allocated with (and must be freed with). +pub struct Vec { + ptr: NonNull, + /// Represents the actual buffer size as `cap` times `size_of::` bytes. + /// + /// Note: This isn't quite the same as `Self::capacity`, which in contrast returns the number of + /// elements we can still store without reallocating. + layout: ArrayLayout, + len: usize, + _p: PhantomData, +} + +/// Type alias for [`Vec`] with a [`Kmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = KVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type KVec = Vec; + +/// Type alias for [`Vec`] with a [`Vmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = VVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type VVec = Vec; + +/// Type alias for [`Vec`] with a [`KVmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = KVVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type KVVec = Vec; + +// SAFETY: `Vec` is `Send` if `T` is `Send` because `Vec` owns its elements. +unsafe impl Send for Vec +where + T: Send, + A: Allocator, +{ +} + +// SAFETY: `Vec` is `Sync` if `T` is `Sync` because `Vec` owns its elements. +unsafe impl Sync for Vec +where + T: Sync, + A: Allocator, +{ +} + +impl Vec +where + A: Allocator, +{ + #[inline] + const fn is_zst() -> bool { + core::mem::size_of::() == 0 + } + + /// Returns the number of elements that can be stored within the vector without allocating + /// additional memory. + pub fn capacity(&self) -> usize { + if const { Self::is_zst() } { + usize::MAX + } else { + self.layout.len() + } + } + + /// Returns the number of elements stored within the vector. + #[inline] + pub fn len(&self) -> usize { + self.len + } + + /// Forcefully sets `self.len` to `new_len`. + /// + /// # Safety + /// + /// - `new_len` must be less than or equal to [`Self::capacity`]. + /// - If `new_len` is greater than `self.len`, all elements within the interval + /// [`self.len`,`new_len`) must be initialized. + #[inline] + pub unsafe fn set_len(&mut self, new_len: usize) { + debug_assert!(new_len <= self.capacity()); + self.len = new_len; + } + + /// Returns a slice of the entire vector. + #[inline] + pub fn as_slice(&self) -> &[T] { + self + } + + /// Returns a mutable slice of the entire vector. + #[inline] + pub fn as_mut_slice(&mut self) -> &mut [T] { + self + } + + /// Returns a mutable raw pointer to the vector's backing buffer, or, if `T` is a ZST, a + /// dangling raw pointer. + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut T { + self.ptr.as_ptr() + } + + /// Returns a raw pointer to the vector's backing buffer, or, if `T` is a ZST, a dangling raw + /// pointer. + #[inline] + pub fn as_ptr(&self) -> *const T { + self.ptr.as_ptr() + } + + /// Returns `true` if the vector contains no elements, `false` otherwise. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// assert!(v.is_empty()); + /// + /// v.push(1, GFP_KERNEL); + /// assert!(!v.is_empty()); + /// ``` + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Creates a new, empty `Vec`. + /// + /// This method does not allocate by itself. + #[inline] + pub const fn new() -> Self { + // INVARIANT: Since this is a new, empty `Vec` with no backing memory yet, + // - `ptr` is a properly aligned dangling pointer for type `T`, + // - `layout` is an empty `ArrayLayout` (zero capacity) + // - `len` is zero, since no elements can be or have been stored, + // - `A` is always valid. + Self { + ptr: NonNull::dangling(), + layout: ArrayLayout::empty(), + len: 0, + _p: PhantomData::, + } + } + + /// Returns a slice of `MaybeUninit` for the remaining spare capacity of the vector. + pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit] { + // SAFETY: + // - `self.len` is smaller than `self.capacity` and hence, the resulting pointer is + // guaranteed to be part of the same allocated object. + // - `self.len` can not overflow `isize`. + let ptr = unsafe { self.as_mut_ptr().add(self.len) } as *mut MaybeUninit; + + // SAFETY: The memory between `self.len` and `self.capacity` is guaranteed to be allocated + // and valid, but uninitialized. + unsafe { slice::from_raw_parts_mut(ptr, self.capacity() - self.len) } + } + + /// Appends an element to the back of the [`Vec`] instance. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// assert_eq!(&v, &[1]); + /// + /// v.push(2, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 2]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError> { + self.reserve(1, flags)?; + + // SAFETY: + // - `self.len` is smaller than `self.capacity` and hence, the resulting pointer is + // guaranteed to be part of the same allocated object. + // - `self.len` can not overflow `isize`. + let ptr = unsafe { self.as_mut_ptr().add(self.len) }; + + // SAFETY: + // - `ptr` is properly aligned and valid for writes. + unsafe { core::ptr::write(ptr, v) }; + + // SAFETY: We just initialised the first spare entry, so it is safe to increase the length + // by 1. We also know that the new length is <= capacity because of the previous call to + // `reserve` above. + unsafe { self.set_len(self.len() + 1) }; + Ok(()) + } + + /// Creates a new [`Vec`] instance with at least the given capacity. + /// + /// # Examples + /// + /// ``` + /// let v = KVec::::with_capacity(20, GFP_KERNEL)?; + /// + /// assert!(v.capacity() >= 20); + /// # Ok::<(), Error>(()) + /// ``` + pub fn with_capacity(capacity: usize, flags: Flags) -> Result { + let mut v = Vec::new(); + + v.reserve(capacity, flags)?; + + Ok(v) + } + + /// Creates a `Vec` from a pointer, a length and a capacity using the allocator `A`. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3]?; + /// v.reserve(1, GFP_KERNEL)?; + /// + /// let (mut ptr, mut len, cap) = v.into_raw_parts(); + /// + /// // SAFETY: We've just reserved memory for another element. + /// unsafe { ptr.add(len).write(4) }; + /// len += 1; + /// + /// // SAFETY: We only wrote an additional element at the end of the `KVec`'s buffer and + /// // correspondingly increased the length of the `KVec` by one. Otherwise, we construct it + /// // from the exact same raw parts. + /// let v = unsafe { KVec::from_raw_parts(ptr, len, cap) }; + /// + /// assert_eq!(v, [1, 2, 3, 4]); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// # Safety + /// + /// If `T` is a ZST: + /// + /// - `ptr` must be a dangling, well aligned pointer. + /// + /// Otherwise: + /// + /// - `ptr` must have been allocated with the allocator `A`. + /// - `ptr` must satisfy or exceed the alignment requirements of `T`. + /// - `ptr` must point to memory with a size of at least `size_of::() * capacity`. + /// bytes. + /// - The allocated size in bytes must not be larger than `isize::MAX`. + /// - `length` must be less than or equal to `capacity`. + /// - The first `length` elements must be initialized values of type `T`. + /// + /// It is also valid to create an empty `Vec` passing a dangling pointer for `ptr` and zero for + /// `cap` and `len`. + pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> Self { + let layout = if Self::is_zst() { + ArrayLayout::empty() + } else { + // SAFETY: By the safety requirements of this function, `capacity * size_of::()` is + // smaller than `isize::MAX`. + unsafe { ArrayLayout::new_unchecked(capacity) } + }; + + // INVARIANT: For ZSTs, we store an empty `ArrayLayout`, all other type invariants are + // covered by the safety requirements of this function. + Self { + // SAFETY: By the safety requirements, `ptr` is either dangling or pointing to a valid + // memory allocation, allocated with `A`. + ptr: unsafe { NonNull::new_unchecked(ptr) }, + layout, + len: length, + _p: PhantomData::, + } + } + + /// Consumes the `Vec` and returns its raw components `pointer`, `length` and `capacity`. + /// + /// This will not run the destructor of the contained elements and for non-ZSTs the allocation + /// will stay alive indefinitely. Use [`Vec::from_raw_parts`] to recover the [`Vec`], drop the + /// elements and free the allocation, if any. + pub fn into_raw_parts(self) -> (*mut T, usize, usize) { + let mut me = ManuallyDrop::new(self); + let len = me.len(); + let capacity = me.capacity(); + let ptr = me.as_mut_ptr(); + (ptr, len, capacity) + } + + /// Ensures that the capacity exceeds the length by at least `additional` elements. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.reserve(10, GFP_KERNEL)?; + /// let cap = v.capacity(); + /// assert!(cap >= 10); + /// + /// v.reserve(10, GFP_KERNEL)?; + /// let new_cap = v.capacity(); + /// assert_eq!(new_cap, cap); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError> { + let len = self.len(); + let cap = self.capacity(); + + if cap - len >= additional { + return Ok(()); + } + + if Self::is_zst() { + // The capacity is already `usize::MAX` for ZSTs, we can't go higher. + return Err(AllocError); + } + + // We know that `cap <= isize::MAX` because of the type invariants of `Self`. So the + // multiplication by two won't overflow. + let new_cap = core::cmp::max(cap * 2, len.checked_add(additional).ok_or(AllocError)?); + let layout = ArrayLayout::new(new_cap).map_err(|_| AllocError)?; + + // SAFETY: + // - `ptr` is valid because it's either `None` or comes from a previous call to + // `A::realloc`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + let ptr = unsafe { + A::realloc( + Some(self.ptr.cast()), + layout.into(), + self.layout.into(), + flags, + )? + }; + + // INVARIANT: + // - `layout` is some `ArrayLayout::`, + // - `ptr` has been created by `A::realloc` from `layout`. + self.ptr = ptr.cast(); + self.layout = layout; + + Ok(()) + } +} + +impl Vec { + /// Extend the vector by `n` clones of `value`. + pub fn extend_with(&mut self, n: usize, value: T, flags: Flags) -> Result<(), AllocError> { + if n == 0 { + return Ok(()); + } + + self.reserve(n, flags)?; + + let spare = self.spare_capacity_mut(); + + for item in spare.iter_mut().take(n - 1) { + item.write(value.clone()); + } + + // We can write the last element directly without cloning needlessly. + spare[n - 1].write(value); + + // SAFETY: + // - `self.len() + n < self.capacity()` due to the call to reserve above, + // - the loop and the line above initialized the next `n` elements. + unsafe { self.set_len(self.len() + n) }; + + Ok(()) + } + + /// Pushes clones of the elements of slice into the [`Vec`] instance. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.extend_from_slice(&[20, 30, 40], GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 20, 30, 40]); + /// + /// v.extend_from_slice(&[50, 60], GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 20, 30, 40, 50, 60]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> { + self.reserve(other.len(), flags)?; + for (slot, item) in core::iter::zip(self.spare_capacity_mut(), other) { + slot.write(item.clone()); + } + + // SAFETY: + // - `other.len()` spare entries have just been initialized, so it is safe to increase + // the length by the same number. + // - `self.len() + other.len() <= self.capacity()` is guaranteed by the preceding `reserve` + // call. + unsafe { self.set_len(self.len() + other.len()) }; + Ok(()) + } + + /// Create a new `Vec` and extend it by `n` clones of `value`. + pub fn from_elem(value: T, n: usize, flags: Flags) -> Result { + let mut v = Self::with_capacity(n, flags)?; + + v.extend_with(n, value, flags)?; + + Ok(v) + } +} + +impl Drop for Vec +where + A: Allocator, +{ + fn drop(&mut self) { + // SAFETY: `self.as_mut_ptr` is guaranteed to be valid by the type invariant. + unsafe { + ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( + self.as_mut_ptr(), + self.len, + )) + }; + + // SAFETY: + // - `self.ptr` was previously allocated with `A`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { A::free(self.ptr.cast(), self.layout.into()) }; + } +} + +impl From> for Vec +where + A: Allocator, +{ + fn from(b: Box<[T; N], A>) -> Vec { + let len = b.len(); + let ptr = Box::into_raw(b); + + // SAFETY: + // - `b` has been allocated with `A`, + // - `ptr` fulfills the alignment requirements for `T`, + // - `ptr` points to memory with at least a size of `size_of::() * len`, + // - all elements within `b` are initialized values of `T`, + // - `len` does not exceed `isize::MAX`. + unsafe { Vec::from_raw_parts(ptr as _, len, len) } + } +} + +impl Default for KVec { + #[inline] + fn default() -> Self { + Self::new() + } +} + +impl fmt::Debug for Vec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl Deref for Vec +where + A: Allocator, +{ + type Target = [T]; + + #[inline] + fn deref(&self) -> &[T] { + // SAFETY: The memory behind `self.as_ptr()` is guaranteed to contain `self.len` + // initialized elements of type `T`. + unsafe { slice::from_raw_parts(self.as_ptr(), self.len) } + } +} + +impl DerefMut for Vec +where + A: Allocator, +{ + #[inline] + fn deref_mut(&mut self) -> &mut [T] { + // SAFETY: The memory behind `self.as_ptr()` is guaranteed to contain `self.len` + // initialized elements of type `T`. + unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.len) } + } +} + +impl Eq for Vec where A: Allocator {} + +impl, A> Index for Vec +where + A: Allocator, +{ + type Output = I::Output; + + #[inline] + fn index(&self, index: I) -> &Self::Output { + Index::index(&**self, index) + } +} + +impl, A> IndexMut for Vec +where + A: Allocator, +{ + #[inline] + fn index_mut(&mut self, index: I) -> &mut Self::Output { + IndexMut::index_mut(&mut **self, index) + } +} + +macro_rules! impl_slice_eq { + ($([$($vars:tt)*] $lhs:ty, $rhs:ty,)*) => { + $( + impl PartialEq<$rhs> for $lhs + where + T: PartialEq, + { + #[inline] + fn eq(&self, other: &$rhs) -> bool { self[..] == other[..] } + } + )* + } +} + +impl_slice_eq! { + [A1: Allocator, A2: Allocator] Vec, Vec, + [A: Allocator] Vec, &[U], + [A: Allocator] Vec, &mut [U], + [A: Allocator] &[T], Vec, + [A: Allocator] &mut [T], Vec, + [A: Allocator] Vec, [U], + [A: Allocator] [T], Vec, + [A: Allocator, const N: usize] Vec, [U; N], + [A: Allocator, const N: usize] Vec, &[U; N], +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 34e9151db58d2d..620de74d128f9f 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -15,6 +15,7 @@ #![feature(arbitrary_self_types)] #![feature(coerce_unsized)] #![feature(dispatch_from_dyn)] +#![feature(inline_const)] #![feature(lint_reasons)] #![feature(unsize)] diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index d5f2fe42d0932f..80223cdaa4850b 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -14,7 +14,7 @@ #[doc(no_inline)] pub use core::pin::Pin; -pub use crate::alloc::{flags::*, vec_ext::VecExt, Box, KBox, KVBox, VBox}; +pub use crate::alloc::{flags::*, vec_ext::VecExt, Box, KBox, KVBox, KVVec, KVec, VBox, VVec}; #[doc(no_inline)] pub use alloc::vec::Vec; From 467851869cd489823b5bf6915b076596b21447cd Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:21 +0200 Subject: [PATCH 14/51] rust: alloc: implement `IntoIterator` for `Vec` Implement `IntoIterator` for `Vec`, `Vec`'s `IntoIter` type, as well as `Iterator` for `IntoIter`. `Vec::into_iter` disassembles the `Vec` into its raw parts; additionally, `IntoIter` keeps track of a separate pointer, which is incremented correspondingsly as the iterator advances, while the length, or the count of elements, is decremented. This also means that `IntoIter` takes the ownership of the backing buffer and is responsible to drop the remaining elements and free the backing buffer, if it's dropped. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-18-dakr@kernel.org [ Fixed typo. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 1 + rust/kernel/alloc/kvec.rs | 170 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 7f654b214ec249..86d2077a8e64f8 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -20,6 +20,7 @@ pub use self::kbox::KBox; pub use self::kbox::KVBox; pub use self::kbox::VBox; +pub use self::kvec::IntoIter; pub use self::kvec::KVVec; pub use self::kvec::KVec; pub use self::kvec::VVec; diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index ad64313a950c01..9723a7fd3a31db 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -647,3 +647,173 @@ impl_slice_eq! { [A: Allocator, const N: usize] Vec, [U; N], [A: Allocator, const N: usize] Vec, &[U; N], } + +impl<'a, T, A> IntoIterator for &'a Vec +where + A: Allocator, +{ + type Item = &'a T; + type IntoIter = slice::Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a, T, A: Allocator> IntoIterator for &'a mut Vec +where + A: Allocator, +{ + type Item = &'a mut T; + type IntoIter = slice::IterMut<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + +/// An [`Iterator`] implementation for [`Vec`] that moves elements out of a vector. +/// +/// This structure is created by the [`Vec::into_iter`] method on [`Vec`] (provided by the +/// [`IntoIterator`] trait). +/// +/// # Examples +/// +/// ``` +/// let v = kernel::kvec![0, 1, 2]?; +/// let iter = v.into_iter(); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub struct IntoIter { + ptr: *mut T, + buf: NonNull, + len: usize, + layout: ArrayLayout, + _p: PhantomData, +} + +impl Iterator for IntoIter +where + A: Allocator, +{ + type Item = T; + + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2, 3]?; + /// let mut it = v.into_iter(); + /// + /// assert_eq!(it.next(), Some(1)); + /// assert_eq!(it.next(), Some(2)); + /// assert_eq!(it.next(), Some(3)); + /// assert_eq!(it.next(), None); + /// + /// # Ok::<(), Error>(()) + /// ``` + fn next(&mut self) -> Option { + if self.len == 0 { + return None; + } + + let current = self.ptr; + + // SAFETY: We can't overflow; decreasing `self.len` by one every time we advance `self.ptr` + // by one guarantees that. + unsafe { self.ptr = self.ptr.add(1) }; + + self.len -= 1; + + // SAFETY: `current` is guaranteed to point at a valid element within the buffer. + Some(unsafe { current.read() }) + } + + /// # Examples + /// + /// ``` + /// let v: KVec = kernel::kvec![1, 2, 3]?; + /// let mut iter = v.into_iter(); + /// let size = iter.size_hint().0; + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 1); + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 2); + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 3); + /// + /// # Ok::<(), Error>(()) + /// ``` + fn size_hint(&self) -> (usize, Option) { + (self.len, Some(self.len)) + } +} + +impl Drop for IntoIter +where + A: Allocator, +{ + fn drop(&mut self) { + // SAFETY: `self.ptr` is guaranteed to be valid by the type invariant. + unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.ptr, self.len)) }; + + // SAFETY: + // - `self.buf` was previously allocated with `A`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { A::free(self.buf.cast(), self.layout.into()) }; + } +} + +impl IntoIterator for Vec +where + A: Allocator, +{ + type Item = T; + type IntoIter = IntoIter; + + /// Consumes the `Vec` and creates an `Iterator`, which moves each value out of the + /// vector (from start to end). + /// + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2]?; + /// let mut v_iter = v.into_iter(); + /// + /// let first_element: Option = v_iter.next(); + /// + /// assert_eq!(first_element, Some(1)); + /// assert_eq!(v_iter.next(), Some(2)); + /// assert_eq!(v_iter.next(), None); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// ``` + /// let v = kernel::kvec![]; + /// let mut v_iter = v.into_iter(); + /// + /// let first_element: Option = v_iter.next(); + /// + /// assert_eq!(first_element, None); + /// + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + fn into_iter(self) -> Self::IntoIter { + let buf = self.ptr; + let layout = self.layout; + let (ptr, len, _) = self.into_raw_parts(); + + IntoIter { + ptr, + buf, + len, + layout, + _p: PhantomData::, + } + } +} From e4fcfb3606e562ab8eb47981860d255bdf4b9ab9 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:22 +0200 Subject: [PATCH 15/51] rust: alloc: implement `collect` for `IntoIter` Currently, we can't implement `FromIterator`. There are a couple of issues with this trait in the kernel, namely: - Rust's specialization feature is unstable. This prevents us to optimize for the special case where `I::IntoIter` equals `Vec`'s `IntoIter` type. - We also can't use `I::IntoIter`'s type ID either to work around this, since `FromIterator` doesn't require this type to be `'static`. - `FromIterator::from_iter` does return `Self` instead of `Result`, hence we can't properly handle allocation failures. - Neither `Iterator::collect` nor `FromIterator::from_iter` can handle additional allocation flags. Instead, provide `IntoIter::collect`, such that we can at least convert `IntoIter` into a `Vec` again. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-19-dakr@kernel.org [ Added newline in documentation, change case of section to be consistent with an existing one, fixed typo. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/alloc/kvec.rs | 95 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index 9723a7fd3a31db..fb1b2bb4cb360b 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -693,6 +693,101 @@ pub struct IntoIter { _p: PhantomData, } +impl IntoIter +where + A: Allocator, +{ + fn into_raw_parts(self) -> (*mut T, NonNull, usize, usize) { + let me = ManuallyDrop::new(self); + let ptr = me.ptr; + let buf = me.buf; + let len = me.len; + let cap = me.layout.len(); + (ptr, buf, len, cap) + } + + /// Same as `Iterator::collect` but specialized for `Vec`'s `IntoIter`. + /// + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2, 3]?; + /// let mut it = v.into_iter(); + /// + /// assert_eq!(it.next(), Some(1)); + /// + /// let v = it.collect(GFP_KERNEL); + /// assert_eq!(v, [2, 3]); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// # Implementation details + /// + /// Currently, we can't implement `FromIterator`. There are a couple of issues with this trait + /// in the kernel, namely: + /// + /// - Rust's specialization feature is unstable. This prevents us to optimize for the special + /// case where `I::IntoIter` equals `Vec`'s `IntoIter` type. + /// - We also can't use `I::IntoIter`'s type ID either to work around this, since `FromIterator` + /// doesn't require this type to be `'static`. + /// - `FromIterator::from_iter` does return `Self` instead of `Result`, hence + /// we can't properly handle allocation failures. + /// - Neither `Iterator::collect` nor `FromIterator::from_iter` can handle additional allocation + /// flags. + /// + /// Instead, provide `IntoIter::collect`, such that we can at least convert a `IntoIter` into a + /// `Vec` again. + /// + /// Note that `IntoIter::collect` doesn't require `Flags`, since it re-uses the existing backing + /// buffer. However, this backing buffer may be shrunk to the actual count of elements. + pub fn collect(self, flags: Flags) -> Vec { + let old_layout = self.layout; + let (mut ptr, buf, len, mut cap) = self.into_raw_parts(); + let has_advanced = ptr != buf.as_ptr(); + + if has_advanced { + // Copy the contents we have advanced to at the beginning of the buffer. + // + // SAFETY: + // - `ptr` is valid for reads of `len * size_of::()` bytes, + // - `buf.as_ptr()` is valid for writes of `len * size_of::()` bytes, + // - `ptr` and `buf.as_ptr()` are not be subject to aliasing restrictions relative to + // each other, + // - both `ptr` and `buf.ptr()` are properly aligned. + unsafe { ptr::copy(ptr, buf.as_ptr(), len) }; + ptr = buf.as_ptr(); + + // SAFETY: `len` is guaranteed to be smaller than `self.layout.len()`. + let layout = unsafe { ArrayLayout::::new_unchecked(len) }; + + // SAFETY: `buf` points to the start of the backing buffer and `len` is guaranteed to be + // smaller than `cap`. Depending on `alloc` this operation may shrink the buffer or leaves + // it as it is. + ptr = match unsafe { + A::realloc(Some(buf.cast()), layout.into(), old_layout.into(), flags) + } { + // If we fail to shrink, which likely can't even happen, continue with the existing + // buffer. + Err(_) => ptr, + Ok(ptr) => { + cap = len; + ptr.as_ptr().cast() + } + }; + } + + // SAFETY: If the iterator has been advanced, the advanced elements have been copied to + // the beginning of the buffer and `len` has been adjusted accordingly. + // + // - `ptr` is guaranteed to point to the start of the backing buffer. + // - `cap` is either the original capacity or, after shrinking the buffer, equal to `len`. + // - `alloc` is guaranteed to be unchanged since `into_iter` has been called on the original + // `Vec`. + unsafe { Vec::from_raw_parts(ptr, len, cap) } + } +} + impl Iterator for IntoIter where A: Allocator, From f1af10d6ab16db25e348bc3dbe95254c74d9e227 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:23 +0200 Subject: [PATCH 16/51] rust: treewide: switch to the kernel `Vec` type Now that we got the kernel `Vec` in place, convert all existing `Vec` users to make use of it. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-20-dakr@kernel.org [ Converted `kasan_test_rust.rs` too, as discussed. - Miguel ] Signed-off-by: Miguel Ojeda --- mm/kasan/kasan_test_rust.rs | 2 +- rust/kernel/str.rs | 12 +++++------- rust/kernel/sync/locked_by.rs | 2 +- rust/kernel/types.rs | 2 +- rust/kernel/uaccess.rs | 17 +++++++---------- rust/macros/lib.rs | 6 +++--- samples/rust/rust_minimal.rs | 4 ++-- 7 files changed, 20 insertions(+), 25 deletions(-) diff --git a/mm/kasan/kasan_test_rust.rs b/mm/kasan/kasan_test_rust.rs index 47bcf033dd4f06..5b34edf30e7244 100644 --- a/mm/kasan/kasan_test_rust.rs +++ b/mm/kasan/kasan_test_rust.rs @@ -11,7 +11,7 @@ use kernel::prelude::*; /// drop the vector, and touch it. #[no_mangle] pub extern "C" fn kasan_test_rust_uaf() -> u8 { - let mut v: Vec = Vec::new(); + let mut v: KVec = KVec::new(); for _ in 0..4096 { v.push(0x42, GFP_KERNEL).unwrap(); } diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index 66d4527f6c6f0b..6053bc7a98d191 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -2,8 +2,7 @@ //! String representations. -use crate::alloc::{flags::*, vec_ext::VecExt, AllocError}; -use alloc::vec::Vec; +use crate::alloc::{flags::*, AllocError, KVec}; use core::fmt::{self, Write}; use core::ops::{self, Deref, DerefMut, Index}; @@ -791,7 +790,7 @@ impl fmt::Write for Formatter { /// assert_eq!(s.is_ok(), false); /// ``` pub struct CString { - buf: Vec, + buf: KVec, } impl CString { @@ -804,7 +803,7 @@ impl CString { let size = f.bytes_written(); // Allocate a vector with the required number of bytes, and write to it. - let mut buf = as VecExt<_>>::with_capacity(size, GFP_KERNEL)?; + let mut buf = KVec::with_capacity(size, GFP_KERNEL)?; // SAFETY: The buffer stored in `buf` is at least of size `size` and is valid for writes. let mut f = unsafe { Formatter::from_buffer(buf.as_mut_ptr(), size) }; f.write_fmt(args)?; @@ -851,10 +850,9 @@ impl<'a> TryFrom<&'a CStr> for CString { type Error = AllocError; fn try_from(cstr: &'a CStr) -> Result { - let mut buf = Vec::new(); + let mut buf = KVec::new(); - as VecExt<_>>::extend_from_slice(&mut buf, cstr.as_bytes_with_nul(), GFP_KERNEL) - .map_err(|_| AllocError)?; + buf.extend_from_slice(cstr.as_bytes_with_nul(), GFP_KERNEL)?; // INVARIANT: The `CStr` and `CString` types have the same invariants for // the string data, and we copied it over without changes. diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs index ce2ee8d8786587..a7b244675c2b96 100644 --- a/rust/kernel/sync/locked_by.rs +++ b/rust/kernel/sync/locked_by.rs @@ -43,7 +43,7 @@ use core::{cell::UnsafeCell, mem::size_of, ptr}; /// struct InnerDirectory { /// /// The sum of the bytes used by all files. /// bytes_used: u64, -/// _files: Vec, +/// _files: KVec, /// } /// /// struct Directory { diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 34f1b31753df1c..ced143600eb1b4 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -135,7 +135,7 @@ impl ForeignOwnable for () { /// # use kernel::types::ScopeGuard; /// fn example3(arg: bool) -> Result { /// let mut vec = -/// ScopeGuard::new_with_data(Vec::new(), |v| pr_info!("vec had {} elements\n", v.len())); +/// ScopeGuard::new_with_data(KVec::new(), |v| pr_info!("vec had {} elements\n", v.len())); /// /// vec.push(10u8, GFP_KERNEL)?; /// if arg { diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index 0af243482eac9a..2c953ba53c771c 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -11,7 +11,6 @@ use crate::{ prelude::*, transmute::{AsBytes, FromBytes}, }; -use alloc::vec::Vec; use core::ffi::{c_ulong, c_void}; use core::mem::{size_of, MaybeUninit}; @@ -46,7 +45,6 @@ pub type UserPtr = usize; /// every byte in the region. /// /// ```no_run -/// use alloc::vec::Vec; /// use core::ffi::c_void; /// use kernel::error::Result; /// use kernel::uaccess::{UserPtr, UserSlice}; @@ -54,7 +52,7 @@ pub type UserPtr = usize; /// fn bytes_add_one(uptr: UserPtr, len: usize) -> Result<()> { /// let (read, mut write) = UserSlice::new(uptr, len).reader_writer(); /// -/// let mut buf = Vec::new(); +/// let mut buf = KVec::new(); /// read.read_all(&mut buf, GFP_KERNEL)?; /// /// for b in &mut buf { @@ -69,7 +67,6 @@ pub type UserPtr = usize; /// Example illustrating a TOCTOU (time-of-check to time-of-use) bug. /// /// ```no_run -/// use alloc::vec::Vec; /// use core::ffi::c_void; /// use kernel::error::{code::EINVAL, Result}; /// use kernel::uaccess::{UserPtr, UserSlice}; @@ -78,21 +75,21 @@ pub type UserPtr = usize; /// fn is_valid(uptr: UserPtr, len: usize) -> Result { /// let read = UserSlice::new(uptr, len).reader(); /// -/// let mut buf = Vec::new(); +/// let mut buf = KVec::new(); /// read.read_all(&mut buf, GFP_KERNEL)?; /// /// todo!() /// } /// /// /// Returns the bytes behind this user pointer if they are valid. -/// fn get_bytes_if_valid(uptr: UserPtr, len: usize) -> Result> { +/// fn get_bytes_if_valid(uptr: UserPtr, len: usize) -> Result> { /// if !is_valid(uptr, len)? { /// return Err(EINVAL); /// } /// /// let read = UserSlice::new(uptr, len).reader(); /// -/// let mut buf = Vec::new(); +/// let mut buf = KVec::new(); /// read.read_all(&mut buf, GFP_KERNEL)?; /// /// // THIS IS A BUG! The bytes could have changed since we checked them. @@ -130,7 +127,7 @@ impl UserSlice { /// Reads the entirety of the user slice, appending it to the end of the provided buffer. /// /// Fails with [`EFAULT`] if the read happens on a bad address. - pub fn read_all(self, buf: &mut Vec, flags: Flags) -> Result { + pub fn read_all(self, buf: &mut KVec, flags: Flags) -> Result { self.reader().read_all(buf, flags) } @@ -291,9 +288,9 @@ impl UserSliceReader { /// Reads the entirety of the user slice, appending it to the end of the provided buffer. /// /// Fails with [`EFAULT`] if the read happens on a bad address. - pub fn read_all(mut self, buf: &mut Vec, flags: Flags) -> Result { + pub fn read_all(mut self, buf: &mut KVec, flags: Flags) -> Result { let len = self.length; - VecExt::::reserve(buf, len, flags)?; + buf.reserve(len, flags)?; // The call to `try_reserve` was successful, so the spare capacity is at least `len` bytes // long. diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index ab93111a048c90..8d4ac914b48bc8 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -242,7 +242,7 @@ pub fn concat_idents(ts: TokenStream) -> TokenStream { /// #[pin_data] /// struct DriverData { /// #[pin] -/// queue: Mutex>, +/// queue: Mutex>, /// buf: KBox<[u8; 1024 * 1024]>, /// } /// ``` @@ -251,7 +251,7 @@ pub fn concat_idents(ts: TokenStream) -> TokenStream { /// #[pin_data(PinnedDrop)] /// struct DriverData { /// #[pin] -/// queue: Mutex>, +/// queue: Mutex>, /// buf: KBox<[u8; 1024 * 1024]>, /// raw_info: *mut Info, /// } @@ -281,7 +281,7 @@ pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream { /// #[pin_data(PinnedDrop)] /// struct DriverData { /// #[pin] -/// queue: Mutex>, +/// queue: Mutex>, /// buf: KBox<[u8; 1024 * 1024]>, /// raw_info: *mut Info, /// } diff --git a/samples/rust/rust_minimal.rs b/samples/rust/rust_minimal.rs index 2a9eaab62d1ca7..4aaf117bf8e3c0 100644 --- a/samples/rust/rust_minimal.rs +++ b/samples/rust/rust_minimal.rs @@ -13,7 +13,7 @@ module! { } struct RustMinimal { - numbers: Vec, + numbers: KVec, } impl kernel::Module for RustMinimal { @@ -21,7 +21,7 @@ impl kernel::Module for RustMinimal { pr_info!("Rust minimal sample (init)\n"); pr_info!("Am I built-in? {}\n", !cfg!(MODULE)); - let mut numbers = Vec::new(); + let mut numbers = KVec::new(); numbers.push(72, GFP_KERNEL)?; numbers.push(108, GFP_KERNEL)?; numbers.push(200, GFP_KERNEL)?; From 0a001a1fdd34a786b4feacc42a5053aa02975613 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:24 +0200 Subject: [PATCH 17/51] rust: alloc: remove `VecExt` extension Now that all existing `Vec` users were moved to the kernel `Vec` type, remove the `VecExt` extension. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-21-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 1 - rust/kernel/alloc/vec_ext.rs | 185 ----------------------------------- rust/kernel/prelude.rs | 5 +- 3 files changed, 1 insertion(+), 190 deletions(-) delete mode 100644 rust/kernel/alloc/vec_ext.rs diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 86d2077a8e64f8..7fc2e404e594dc 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -7,7 +7,6 @@ pub mod allocator; pub mod kbox; pub mod kvec; pub mod layout; -pub mod vec_ext; #[cfg(any(test, testlib))] pub mod allocator_test; diff --git a/rust/kernel/alloc/vec_ext.rs b/rust/kernel/alloc/vec_ext.rs deleted file mode 100644 index 1297a4be32e8c4..00000000000000 --- a/rust/kernel/alloc/vec_ext.rs +++ /dev/null @@ -1,185 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! Extensions to [`Vec`] for fallible allocations. - -use super::{AllocError, Flags}; -use alloc::vec::Vec; - -/// Extensions to [`Vec`]. -pub trait VecExt: Sized { - /// Creates a new [`Vec`] instance with at least the given capacity. - /// - /// # Examples - /// - /// ``` - /// let v = Vec::::with_capacity(20, GFP_KERNEL)?; - /// - /// assert!(v.capacity() >= 20); - /// # Ok::<(), Error>(()) - /// ``` - fn with_capacity(capacity: usize, flags: Flags) -> Result; - - /// Appends an element to the back of the [`Vec`] instance. - /// - /// # Examples - /// - /// ``` - /// let mut v = Vec::new(); - /// v.push(1, GFP_KERNEL)?; - /// assert_eq!(&v, &[1]); - /// - /// v.push(2, GFP_KERNEL)?; - /// assert_eq!(&v, &[1, 2]); - /// # Ok::<(), Error>(()) - /// ``` - fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError>; - - /// Pushes clones of the elements of slice into the [`Vec`] instance. - /// - /// # Examples - /// - /// ``` - /// let mut v = Vec::new(); - /// v.push(1, GFP_KERNEL)?; - /// - /// v.extend_from_slice(&[20, 30, 40], GFP_KERNEL)?; - /// assert_eq!(&v, &[1, 20, 30, 40]); - /// - /// v.extend_from_slice(&[50, 60], GFP_KERNEL)?; - /// assert_eq!(&v, &[1, 20, 30, 40, 50, 60]); - /// # Ok::<(), Error>(()) - /// ``` - fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> - where - T: Clone; - - /// Ensures that the capacity exceeds the length by at least `additional` elements. - /// - /// # Examples - /// - /// ``` - /// let mut v = Vec::new(); - /// v.push(1, GFP_KERNEL)?; - /// - /// v.reserve(10, GFP_KERNEL)?; - /// let cap = v.capacity(); - /// assert!(cap >= 10); - /// - /// v.reserve(10, GFP_KERNEL)?; - /// let new_cap = v.capacity(); - /// assert_eq!(new_cap, cap); - /// - /// # Ok::<(), Error>(()) - /// ``` - fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError>; -} - -impl VecExt for Vec { - fn with_capacity(capacity: usize, flags: Flags) -> Result { - let mut v = Vec::new(); - >::reserve(&mut v, capacity, flags)?; - Ok(v) - } - - fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError> { - >::reserve(self, 1, flags)?; - let s = self.spare_capacity_mut(); - s[0].write(v); - - // SAFETY: We just initialised the first spare entry, so it is safe to increase the length - // by 1. We also know that the new length is <= capacity because of the previous call to - // `reserve` above. - unsafe { self.set_len(self.len() + 1) }; - Ok(()) - } - - fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> - where - T: Clone, - { - >::reserve(self, other.len(), flags)?; - for (slot, item) in core::iter::zip(self.spare_capacity_mut(), other) { - slot.write(item.clone()); - } - - // SAFETY: We just initialised the `other.len()` spare entries, so it is safe to increase - // the length by the same amount. We also know that the new length is <= capacity because - // of the previous call to `reserve` above. - unsafe { self.set_len(self.len() + other.len()) }; - Ok(()) - } - - #[cfg(any(test, testlib))] - fn reserve(&mut self, additional: usize, _flags: Flags) -> Result<(), AllocError> { - Vec::reserve(self, additional); - Ok(()) - } - - #[cfg(not(any(test, testlib)))] - fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError> { - let len = self.len(); - let cap = self.capacity(); - - if cap - len >= additional { - return Ok(()); - } - - if core::mem::size_of::() == 0 { - // The capacity is already `usize::MAX` for SZTs, we can't go higher. - return Err(AllocError); - } - - // We know cap is <= `isize::MAX` because `Layout::array` fails if the resulting byte size - // is greater than `isize::MAX`. So the multiplication by two won't overflow. - let new_cap = core::cmp::max(cap * 2, len.checked_add(additional).ok_or(AllocError)?); - let layout = core::alloc::Layout::array::(new_cap).map_err(|_| AllocError)?; - - let (old_ptr, len, cap) = destructure(self); - - // We need to make sure that `ptr` is either NULL or comes from a previous call to - // `krealloc_aligned`. A `Vec`'s `ptr` value is not guaranteed to be NULL and might be - // dangling after being created with `Vec::new`. Instead, we can rely on `Vec`'s capacity - // to be zero if no memory has been allocated yet. - let ptr = if cap == 0 { - core::ptr::null_mut() - } else { - old_ptr - }; - - // SAFETY: `ptr` is valid because it's either NULL or comes from a previous call to - // `krealloc_aligned`. We also verified that the type is not a ZST. - let new_ptr = unsafe { super::allocator::krealloc_aligned(ptr.cast(), layout, flags) }; - if new_ptr.is_null() { - // SAFETY: We are just rebuilding the existing `Vec` with no changes. - unsafe { rebuild(self, old_ptr, len, cap) }; - Err(AllocError) - } else { - // SAFETY: `ptr` has been reallocated with the layout for `new_cap` elements. New cap - // is greater than `cap`, so it continues to be >= `len`. - unsafe { rebuild(self, new_ptr.cast::(), len, new_cap) }; - Ok(()) - } - } -} - -#[cfg(not(any(test, testlib)))] -fn destructure(v: &mut Vec) -> (*mut T, usize, usize) { - let mut tmp = Vec::new(); - core::mem::swap(&mut tmp, v); - let mut tmp = core::mem::ManuallyDrop::new(tmp); - let len = tmp.len(); - let cap = tmp.capacity(); - (tmp.as_mut_ptr(), len, cap) -} - -/// Rebuilds a `Vec` from a pointer, length, and capacity. -/// -/// # Safety -/// -/// The same as [`Vec::from_raw_parts`]. -#[cfg(not(any(test, testlib)))] -unsafe fn rebuild(v: &mut Vec, ptr: *mut T, len: usize, cap: usize) { - // SAFETY: The safety requirements from this function satisfy those of `from_raw_parts`. - let mut tmp = unsafe { Vec::from_raw_parts(ptr, len, cap) }; - core::mem::swap(&mut tmp, v); -} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 80223cdaa4850b..07daccf6ca8e7b 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -14,10 +14,7 @@ #[doc(no_inline)] pub use core::pin::Pin; -pub use crate::alloc::{flags::*, vec_ext::VecExt, Box, KBox, KVBox, KVVec, KVec, VBox, VVec}; - -#[doc(no_inline)] -pub use alloc::vec::Vec; +pub use crate::alloc::{flags::*, Box, KBox, KVBox, KVVec, KVec, VBox, VVec}; #[doc(no_inline)] pub use macros::{module, pin_data, pinned_drop, vtable, Zeroable}; From 13639edf4659d22adff13c599e273822d861da61 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:25 +0200 Subject: [PATCH 18/51] rust: alloc: add `Vec` to prelude Now that we removed `VecExt` and the corresponding includes in prelude.rs, add the new kernel `Vec` type instead. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-22-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/prelude.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 07daccf6ca8e7b..8bdab9aa0d16bf 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -14,7 +14,7 @@ #[doc(no_inline)] pub use core::pin::Pin; -pub use crate::alloc::{flags::*, Box, KBox, KVBox, KVVec, KVec, VBox, VVec}; +pub use crate::alloc::{flags::*, Box, KBox, KVBox, KVVec, KVec, VBox, VVec, Vec}; #[doc(no_inline)] pub use macros::{module, pin_data, pinned_drop, vtable, Zeroable}; From b9fce3688b45a48267e2f4396784694c73fb246c Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:26 +0200 Subject: [PATCH 19/51] rust: error: use `core::alloc::LayoutError` Use `core::alloc::LayoutError` instead of `alloc::alloc::LayoutError` in preparation to get rid of Rust's alloc crate. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-23-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index be6509d5f4a4a7..aced2fe68b86b6 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -6,7 +6,7 @@ use crate::{alloc::AllocError, str::CStr}; -use alloc::alloc::LayoutError; +use core::alloc::LayoutError; use core::fmt; use core::num::NonZeroI32; From ee94f805bccf4bbcaea7416eff3475e61f560b80 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:27 +0200 Subject: [PATCH 20/51] rust: error: check for config `test` in `Error::name` Additional to `testlib` also check for `test` in `Error::name`. This is required by a subsequent patch that (indirectly) uses `Error` in test cases. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-24-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/error.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index aced2fe68b86b6..7cd3bbab52f208 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -161,7 +161,7 @@ impl Error { } /// Returns a string representing the error, if one exists. - #[cfg(not(testlib))] + #[cfg(not(any(test, testlib)))] pub fn name(&self) -> Option<&'static CStr> { // SAFETY: Just an FFI call, there are no extra safety requirements. let ptr = unsafe { bindings::errname(-self.0.get()) }; @@ -178,7 +178,7 @@ impl Error { /// When `testlib` is configured, this always returns `None` to avoid the dependency on a /// kernel function so that tests that use this (e.g., by calling [`Result::unwrap`]) can still /// run in userspace. - #[cfg(testlib)] + #[cfg(any(test, testlib))] pub fn name(&self) -> Option<&'static CStr> { None } From 061ef9e5b3c494f303cfac6896a3f5403fba6e84 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:28 +0200 Subject: [PATCH 21/51] rust: alloc: implement `contains` for `Flags` Provide a simple helper function to check whether given flags do contain one or multiple other flags. This is used by a subsequent patch implementing the Cmalloc `Allocator` to check for __GFP_ZERO. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-25-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 7fc2e404e594dc..049fca7a514d1a 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -35,7 +35,7 @@ use core::{alloc::Layout, ptr::NonNull}; /// They can be combined with the operators `|`, `&`, and `!`. /// /// Values can be used from the [`flags`] module. -#[derive(Clone, Copy)] +#[derive(Clone, Copy, PartialEq)] pub struct Flags(u32); impl Flags { @@ -43,6 +43,11 @@ impl Flags { pub(crate) fn as_raw(self) -> u32 { self.0 } + + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Flags) -> bool { + (self & flags) == flags + } } impl core::ops::BitOr for Flags { From ea53ed0ccbd54eb44fcb0e3ef50a4461e93805a6 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:29 +0200 Subject: [PATCH 22/51] rust: alloc: implement `Cmalloc` in module allocator_test So far the kernel's `Box` and `Vec` types can't be used by userspace test cases, since all users of those types (e.g. `CString`) use kernel allocators for instantiation. In order to allow userspace test cases to make use of such types as well, implement the `Cmalloc` allocator within the allocator_test module and type alias all kernel allocators to `Cmalloc`. The `Cmalloc` allocator uses libc's realloc() function as allocator backend. Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-26-dakr@kernel.org [ Removed the temporary `allow(dead_code)` as discussed in the list and fixed typo. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 1 - rust/kernel/alloc/allocator_test.rs | 89 ++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 049fca7a514d1a..c6024afa373973 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -215,7 +215,6 @@ pub unsafe trait Allocator { } } -#[allow(dead_code)] /// Returns a properly aligned dangling pointer from the given `layout`. pub(crate) fn dangling_from_layout(layout: Layout) -> NonNull { let ptr = layout.align() as *mut u8; diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs index bd0cbcd93e5283..20b7dcb20376a9 100644 --- a/rust/kernel/alloc/allocator_test.rs +++ b/rust/kernel/alloc/allocator_test.rs @@ -1,22 +1,95 @@ // SPDX-License-Identifier: GPL-2.0 +//! So far the kernel's `Box` and `Vec` types can't be used by userspace test cases, since all users +//! of those types (e.g. `CString`) use kernel allocators for instantiation. +//! +//! In order to allow userspace test cases to make use of such types as well, implement the +//! `Cmalloc` allocator within the allocator_test module and type alias all kernel allocators to +//! `Cmalloc`. The `Cmalloc` allocator uses libc's realloc() function as allocator backend. + #![allow(missing_docs)] -use super::{AllocError, Allocator, Flags}; +use super::{flags::*, AllocError, Allocator, Flags}; use core::alloc::Layout; +use core::cmp; +use core::ptr; use core::ptr::NonNull; -pub struct Kmalloc; +/// The userspace allocator based on libc. +pub struct Cmalloc; + +pub type Kmalloc = Cmalloc; pub type Vmalloc = Kmalloc; pub type KVmalloc = Kmalloc; -unsafe impl Allocator for Kmalloc { +extern "C" { + #[link_name = "aligned_alloc"] + fn libc_aligned_alloc(align: usize, size: usize) -> *mut core::ffi::c_void; + + #[link_name = "free"] + fn libc_free(ptr: *mut core::ffi::c_void); +} + +// SAFETY: +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation created by this `Allocator` is always OK, +// - `realloc` provides the guarantees as provided in the `# Guarantees` section. +unsafe impl Allocator for Cmalloc { unsafe fn realloc( - _ptr: Option>, - _layout: Layout, - _old_layout: Layout, - _flags: Flags, + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, ) -> Result, AllocError> { - panic!(); + let src = match ptr { + Some(src) => { + if old_layout.size() == 0 { + ptr::null_mut() + } else { + src.as_ptr() + } + } + None => ptr::null_mut(), + }; + + if layout.size() == 0 { + // SAFETY: `src` is either NULL or was previously allocated with this `Allocator` + unsafe { libc_free(src.cast()) }; + + return Ok(NonNull::slice_from_raw_parts( + crate::alloc::dangling_from_layout(layout), + 0, + )); + } + + // SAFETY: Returns either NULL or a pointer to a memory allocation that satisfies or + // exceeds the given size and alignment requirements. + let dst = unsafe { libc_aligned_alloc(layout.align(), layout.size()) } as *mut u8; + let dst = NonNull::new(dst).ok_or(AllocError)?; + + if flags.contains(__GFP_ZERO) { + // SAFETY: The preceding calls to `libc_aligned_alloc` and `NonNull::new` + // guarantee that `dst` points to memory of at least `layout.size()` bytes. + unsafe { dst.as_ptr().write_bytes(0, layout.size()) }; + } + + if !src.is_null() { + // SAFETY: + // - `src` has previously been allocated with this `Allocator`; `dst` has just been + // newly allocated, hence the memory regions do not overlap. + // - both` src` and `dst` are properly aligned and valid for reads and writes + unsafe { + ptr::copy_nonoverlapping( + src, + dst.as_ptr(), + cmp::min(layout.size(), old_layout.size()), + ) + }; + } + + // SAFETY: `src` is either NULL or was previously allocated with this `Allocator` + unsafe { libc_free(src.cast()) }; + + Ok(NonNull::slice_from_raw_parts(dst, layout.size())) } } From 6923f8307a0d8eff5866e62ac8a6be224ca85f0c Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:30 +0200 Subject: [PATCH 23/51] rust: str: test: replace `alloc::format` The current implementation of tests in str.rs use `format!` to format strings for comparison, which, internally, creates a new `String`. In order to prepare for getting rid of Rust's alloc crate, we have to cut this dependency. Instead, implement `format!` for `CString`. Note that for userspace tests, `Kmalloc`, which is backing `CString`'s memory, is just a type alias to `Cmalloc`. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-27-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/str.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index 6053bc7a98d191..aff6baa521d4a0 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -524,7 +524,28 @@ macro_rules! c_str { #[cfg(test)] mod tests { use super::*; - use alloc::format; + + struct String(CString); + + impl String { + fn from_fmt(args: fmt::Arguments<'_>) -> Self { + String(CString::try_from_fmt(args).unwrap()) + } + } + + impl Deref for String { + type Target = str; + + fn deref(&self) -> &str { + self.0.to_str().unwrap() + } + } + + macro_rules! format { + ($($f:tt)*) => ({ + &*String::from_fmt(kernel::fmt!($($f)*)) + }) + } const ALL_ASCII_CHARS: &'static str = "\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\x09\\x0a\\x0b\\x0c\\x0d\\x0e\\x0f\ From f9fa50834e5a68c16424a8fe430db2ffeb54e8b5 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:31 +0200 Subject: [PATCH 24/51] rust: alloc: update module comment of alloc.rs Before we remove Rust's alloc crate, rewrite the module comment in alloc.rs to avoid a rustdoc warning. Besides that, the module comment in alloc.rs isn't correct anymore, we're no longer extending Rust's alloc crate. Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Gary Guo Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-28-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/alloc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index c6024afa373973..f2f7f3a53d298c 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -//! Extensions to the [`alloc`] crate. +//! Implementation of the kernel's memory allocation infrastructure. #[cfg(not(any(test, testlib)))] pub mod allocator; From eed0405dace501a4b3e0f9475acdceb88b7866ac Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:32 +0200 Subject: [PATCH 25/51] kbuild: rust: remove the `alloc` crate and `GlobalAlloc` Now that we have our own `Allocator`, `Box` and `Vec` types we can remove Rust's `alloc` crate and the `new_uninit` unstable feature. Also remove `Kmalloc`'s `GlobalAlloc` implementation -- we can't remove this in a separate patch, since the `alloc` crate requires a `#[global_allocator]` to set, that implements `GlobalAlloc`. Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-29-dakr@kernel.org Signed-off-by: Miguel Ojeda --- rust/Makefile | 43 +++++--------------- rust/exports.c | 1 - rust/kernel/alloc/allocator.rs | 65 +------------------------------ scripts/Makefile.build | 4 +- scripts/generate_rust_analyzer.py | 11 +----- 5 files changed, 16 insertions(+), 108 deletions(-) diff --git a/rust/Makefile b/rust/Makefile index 0856fd6bc61057..3678e79317f12d 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -15,8 +15,8 @@ always-$(CONFIG_RUST) += libmacros.so no-clean-files += libmacros.so always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs -obj-$(CONFIG_RUST) += alloc.o bindings.o kernel.o -always-$(CONFIG_RUST) += exports_alloc_generated.h exports_helpers_generated.h \ +obj-$(CONFIG_RUST) += bindings.o kernel.o +always-$(CONFIG_RUST) += exports_helpers_generated.h \ exports_bindings_generated.h exports_kernel_generated.h always-$(CONFIG_RUST) += uapi/uapi_generated.rs @@ -53,11 +53,6 @@ endif core-cfgs = \ --cfg no_fp_fmt_parse -alloc-cfgs = \ - --cfg no_global_oom_handling \ - --cfg no_rc \ - --cfg no_sync - quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< cmd_rustdoc = \ OBJTREE=$(abspath $(objtree)) \ @@ -81,7 +76,7 @@ quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< # command-like flags to solve the issue. Meanwhile, we use the non-custom case # and then retouch the generated files. rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \ - rustdoc-alloc rustdoc-kernel + rustdoc-kernel $(Q)cp $(srctree)/Documentation/images/logo.svg $(rustdoc_output)/static.files/ $(Q)cp $(srctree)/Documentation/images/COPYING-logo $(rustdoc_output)/static.files/ $(Q)find $(rustdoc_output) -name '*.html' -type f -print0 | xargs -0 sed -Ei \ @@ -108,20 +103,11 @@ rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs FORCE rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE +$(call if_changed,rustdoc) -# We need to allow `rustdoc::broken_intra_doc_links` because some -# `no_global_oom_handling` functions refer to non-`no_global_oom_handling` -# functions. Ideally `rustdoc` would have a way to distinguish broken links -# due to things that are "configured out" vs. entirely non-existing ones. -rustdoc-alloc: private rustc_target_flags = $(alloc-cfgs) \ - -Arustdoc::broken_intra_doc_links -rustdoc-alloc: $(RUST_LIB_SRC)/alloc/src/lib.rs rustdoc-core rustdoc-compiler_builtins FORCE - +$(call if_changed,rustdoc) - -rustdoc-kernel: private rustc_target_flags = --extern alloc \ +rustdoc-kernel: private rustc_target_flags = \ --extern build_error --extern macros=$(objtree)/$(obj)/libmacros.so \ --extern bindings --extern uapi rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-macros \ - rustdoc-compiler_builtins rustdoc-alloc $(obj)/libmacros.so \ + rustdoc-compiler_builtins $(obj)/libmacros.so \ $(obj)/bindings.o FORCE +$(call if_changed,rustdoc) @@ -165,7 +151,7 @@ quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $< mkdir -p $(objtree)/$(obj)/test/doctests/kernel; \ OBJTREE=$(abspath $(objtree)) \ $(RUSTDOC) --test $(rust_flags) \ - -L$(objtree)/$(obj) --extern alloc --extern kernel \ + -L$(objtree)/$(obj) --extern kernel \ --extern build_error --extern macros \ --extern bindings --extern uapi \ --no-run --crate-name kernel -Zunstable-options \ @@ -201,7 +187,7 @@ rusttest-macros: $(src)/macros/lib.rs FORCE +$(call if_changed,rustc_test) +$(call if_changed,rustdoc_test) -rusttest-kernel: private rustc_target_flags = --extern alloc \ +rusttest-kernel: private rustc_target_flags = \ --extern build_error --extern macros --extern bindings --extern uapi rusttest-kernel: $(src)/kernel/lib.rs \ rusttestlib-build_error rusttestlib-macros rusttestlib-bindings \ @@ -314,9 +300,6 @@ quiet_cmd_exports = EXPORTS $@ $(obj)/exports_core_generated.h: $(obj)/core.o FORCE $(call if_changed,exports) -$(obj)/exports_alloc_generated.h: $(obj)/alloc.o FORCE - $(call if_changed,exports) - # Even though Rust kernel modules should never use the bindings directly, # symbols from the `bindings` crate and the C helpers need to be exported # because Rust generics and inlined functions may not get their code generated @@ -363,7 +346,7 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L rust-analyzer: $(Q)$(srctree)/scripts/generate_rust_analyzer.py \ - --cfgs='core=$(core-cfgs)' --cfgs='alloc=$(alloc-cfgs)' \ + --cfgs='core=$(core-cfgs)' \ $(realpath $(srctree)) $(realpath $(objtree)) \ $(rustc_sysroot) $(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \ $(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json @@ -401,12 +384,6 @@ $(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' $(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE +$(call if_changed_rule,rustc_library) -$(obj)/alloc.o: private skip_clippy = 1 -$(obj)/alloc.o: private skip_flags = -Wunreachable_pub -$(obj)/alloc.o: private rustc_target_flags = $(alloc-cfgs) -$(obj)/alloc.o: $(RUST_LIB_SRC)/alloc/src/lib.rs $(obj)/compiler_builtins.o FORCE - +$(call if_changed_rule,rustc_library) - $(obj)/build_error.o: $(src)/build_error.rs $(obj)/compiler_builtins.o FORCE +$(call if_changed_rule,rustc_library) @@ -421,9 +398,9 @@ $(obj)/uapi.o: $(src)/uapi/lib.rs \ $(obj)/uapi/uapi_generated.rs FORCE +$(call if_changed_rule,rustc_library) -$(obj)/kernel.o: private rustc_target_flags = --extern alloc \ +$(obj)/kernel.o: private rustc_target_flags = \ --extern build_error --extern macros --extern bindings --extern uapi -$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/alloc.o $(obj)/build_error.o \ +$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/build_error.o \ $(obj)/libmacros.so $(obj)/bindings.o $(obj)/uapi.o FORCE +$(call if_changed_rule,rustc_library) diff --git a/rust/exports.c b/rust/exports.c index e5695f3b45b7aa..82a037381798d7 100644 --- a/rust/exports.c +++ b/rust/exports.c @@ -16,7 +16,6 @@ #define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym) #include "exports_core_generated.h" -#include "exports_alloc_generated.h" #include "exports_helpers_generated.h" #include "exports_bindings_generated.h" #include "exports_kernel_generated.h" diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index d9043ca42ce50a..067719f11d131c 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -8,8 +8,8 @@ //! //! Reference: -use super::{flags::*, Flags}; -use core::alloc::{GlobalAlloc, Layout}; +use super::Flags; +use core::alloc::Layout; use core::ptr; use core::ptr::NonNull; @@ -54,23 +54,6 @@ fn aligned_size(new_layout: Layout) -> usize { layout.size() } -/// Calls `krealloc` with a proper size to alloc a new object aligned to `new_layout`'s alignment. -/// -/// # Safety -/// -/// - `ptr` can be either null or a pointer which has been allocated by this allocator. -/// - `new_layout` must have a non-zero size. -pub(crate) unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: Flags) -> *mut u8 { - let size = aligned_size(new_layout); - - // SAFETY: - // - `ptr` is either null or a pointer returned from a previous `k{re}alloc()` by the - // function safety requirement. - // - `size` is greater than 0 since it's from `layout.size()` (which cannot be zero according - // to the function safety requirement) - unsafe { bindings::krealloc(ptr as *const core::ffi::c_void, size, flags.0) as *mut u8 } -} - /// # Invariants /// /// One of the following `krealloc`, `vrealloc`, `kvrealloc`. @@ -156,43 +139,6 @@ unsafe impl Allocator for Kmalloc { } } -// SAFETY: TODO. -unsafe impl GlobalAlloc for Kmalloc { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety - // requirement. - unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL) } - } - - unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { - // SAFETY: TODO. - unsafe { - bindings::kfree(ptr as *const core::ffi::c_void); - } - } - - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - // SAFETY: - // - `new_size`, when rounded up to the nearest multiple of `layout.align()`, will not - // overflow `isize` by the function safety requirement. - // - `layout.align()` is a proper alignment (i.e. not zero and must be a power of two). - let layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; - - // SAFETY: - // - `ptr` is either null or a pointer allocated by this allocator by the function safety - // requirement. - // - the size of `layout` is not zero because `new_size` is not zero by the function safety - // requirement. - unsafe { krealloc_aligned(ptr, layout, GFP_KERNEL) } - } - - unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety - // requirement. - unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL | __GFP_ZERO) } - } -} - // SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that // - memory remains valid until it is explicitly freed, // - passing a pointer to a valid memory allocation is OK, @@ -240,10 +186,3 @@ unsafe impl Allocator for KVmalloc { unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags) } } } - -#[global_allocator] -static ALLOCATOR: Kmalloc = Kmalloc; - -// See . -#[no_mangle] -static __rust_no_alloc_shim_is_unstable: u8 = 0; diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 64518c2f3d9c3a..2bba59e790b8a4 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -248,7 +248,7 @@ $(obj)/%.lst: $(obj)/%.c FORCE # Compile Rust sources (.rs) # --------------------------------------------------------------------------- -rust_allowed_features := arbitrary_self_types,lint_reasons,new_uninit +rust_allowed_features := arbitrary_self_types,lint_reasons # `--out-dir` is required to avoid temporaries being created by `rustc` in the # current working directory, which may be not accessible in the out-of-tree @@ -258,7 +258,7 @@ rust_common_cmd = \ -Zallow-features=$(rust_allowed_features) \ -Zcrate-attr=no_std \ -Zcrate-attr='feature($(rust_allowed_features))' \ - -Zunstable-options --extern force:alloc --extern kernel \ + -Zunstable-options --extern kernel \ --crate-type rlib -L $(objtree)/rust/ \ --crate-name $(basename $(notdir $@)) \ --sysroot=/dev/null \ diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index d2bc63cde8c6a3..09e1d166d8d236 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -64,13 +64,6 @@ def append_crate(display_name, root_module, deps, cfg=[], is_workspace_member=Tr [], ) - append_crate( - "alloc", - sysroot_src / "alloc" / "src" / "lib.rs", - ["core", "compiler_builtins"], - cfg=crates_cfgs.get("alloc", []), - ) - append_crate( "macros", srctree / "rust" / "macros" / "lib.rs", @@ -96,7 +89,7 @@ def append_crate(display_name, root_module, deps, cfg=[], is_workspace_member=Tr append_crate( "kernel", srctree / "rust" / "kernel" / "lib.rs", - ["core", "alloc", "macros", "build_error", "bindings"], + ["core", "macros", "build_error", "bindings"], cfg=cfg, ) crates[-1]["source"] = { @@ -133,7 +126,7 @@ def is_root_crate(build_file, target): append_crate( name, path, - ["core", "alloc", "kernel"], + ["core", "kernel"], cfg=cfg, ) From 8d8f785ceb21b9a0de11e05b811cc52d6fa79318 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 4 Oct 2024 17:41:33 +0200 Subject: [PATCH 26/51] MAINTAINERS: add entry for the Rust `alloc` module Add maintainers entry for the Rust `alloc` module. Currently, this includes the `Allocator` API itself, `Allocator` implementations, such as `Kmalloc` or `Vmalloc`, as well as the kernel's implementation of the primary memory allocation data structures, `Box` and `Vec`. Signed-off-by: Danilo Krummrich Link: https://lore.kernel.org/r/20241004154149.93856-30-dakr@kernel.org Signed-off-by: Miguel Ojeda --- MAINTAINERS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 3380b87dcbb983..b77f4495dcf427 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -20230,6 +20230,13 @@ F: scripts/*rust* F: tools/testing/selftests/rust/ K: \b(?i:rust)\b +RUST [ALLOC] +M: Danilo Krummrich +L: rust-for-linux@vger.kernel.org +S: Maintained +F: rust/kernel/alloc.rs +F: rust/kernel/alloc/ + RXRPC SOCKETS (AF_RXRPC) M: David Howells M: Marc Dionne From da4a5265eb07efb4dc62e1bf7232820a45205ed6 Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Fri, 29 Sep 2023 17:58:09 -0300 Subject: [PATCH 27/51] rust: init: introduce `Opaque::try_ffi_init` We'll need it, for example, when calling `register_filesystem` to initialise a file system registration. Signed-off-by: Wedson Almeida Filho Signed-off-by: Danilo Krummrich --- rust/kernel/types.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index ced143600eb1b4..236e8de5844b51 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -239,14 +239,22 @@ impl Opaque { /// uninitialized. Additionally, access to the inner `T` requires `unsafe`, so the caller needs /// to verify at that point that the inner value is valid. pub fn ffi_init(init_func: impl FnOnce(*mut T)) -> impl PinInit { + Self::try_ffi_init(move |slot| { + init_func(slot); + Ok(()) + }) + } + + /// Similar to [`Self::ffi_init`], except that the closure can fail. + /// + /// To avoid leaks on failure, the closure must drop any fields it has initialised before the + /// failure. + pub fn try_ffi_init( + init_func: impl FnOnce(*mut T) -> Result<(), E>, + ) -> impl PinInit { // SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully // initialize the `T`. - unsafe { - init::pin_init_from_closure::<_, ::core::convert::Infallible>(move |slot| { - init_func(Self::raw_get(slot)); - Ok(()) - }) - } + unsafe { init::pin_init_from_closure(|slot| init_func(Self::raw_get(slot))) } } /// Returns a raw pointer to the opaque data. From 19f84d2fc16d1ec3cc517e09e050a854b4b1fd1f Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Fri, 29 Sep 2023 17:58:08 -0300 Subject: [PATCH 28/51] rust: introduce `InPlaceModule` This allows modules to be initialised in-place in pinned memory, which enables the usage of pinned types (e.g., mutexes, spinlocks, driver registrations, etc.) in modules without any extra allocations. Drivers that don't need this may continue to implement `Module` without any changes. Signed-off-by: Wedson Almeida Filho Signed-off-by: Danilo Krummrich --- rust/kernel/lib.rs | 23 +++++++++++++++++++++++ rust/macros/module.rs | 26 ++++++++++---------------- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 620de74d128f9f..435896e160e26d 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -83,6 +83,29 @@ pub trait Module: Sized + Sync + Send { fn init(module: &'static ThisModule) -> error::Result; } +/// A module that is pinned and initialised in-place. +pub trait InPlaceModule: Sync + Send { + /// Creates an initialiser for the module. + /// + /// It is called when the module is loaded. + fn init(module: &'static ThisModule) -> impl init::PinInit; +} + +impl InPlaceModule for T { + fn init(module: &'static ThisModule) -> impl init::PinInit { + let initer = move |slot: *mut Self| { + let m = ::init(module)?; + + // SAFETY: `slot` is valid for write per the contract with `pin_init_from_closure`. + unsafe { slot.write(m) }; + Ok(()) + }; + + // SAFETY: On success, `initer` always fully initialises an instance of `Self`. + unsafe { init::pin_init_from_closure(initer) } + } +} + /// Equivalent to `THIS_MODULE` in the C API. /// /// C header: [`include/linux/export.h`](srctree/include/linux/export.h) diff --git a/rust/macros/module.rs b/rust/macros/module.rs index aef3b132f32b33..ba2d6d36e0389d 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -232,6 +232,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { mod __module_init {{ mod __module_init {{ use super::super::{type_}; + use kernel::init::PinInit; /// The \"Rust loadable module\" mark. // @@ -242,7 +243,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { #[used] static __IS_RUST_MODULE: () = (); - static mut __MOD: Option<{type_}> = None; + static mut __MOD: core::mem::MaybeUninit<{type_}> = core::mem::MaybeUninit::uninit(); // Loadable modules need to export the `{{init,cleanup}}_module` identifiers. /// # Safety @@ -331,20 +332,13 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { /// /// This function must only be called once. unsafe fn __init() -> core::ffi::c_int {{ - match <{type_} as kernel::Module>::init(&super::super::THIS_MODULE) {{ - Ok(m) => {{ - // SAFETY: No data race, since `__MOD` can only be accessed by this - // module and there only `__init` and `__exit` access it. These - // functions are only called once and `__exit` cannot be called - // before or during `__init`. - unsafe {{ - __MOD = Some(m); - }} - return 0; - }} - Err(e) => {{ - return e.to_errno(); - }} + let initer = <{type_} as kernel::InPlaceModule>::init(&super::super::THIS_MODULE); + // SAFETY: No data race, since `__MOD` can only be accessed by this module + // and there only `__init` and `__exit` access it. These functions are only + // called once and `__exit` cannot be called before or during `__init`. + match unsafe {{ initer.__pinned_init(__MOD.as_mut_ptr()) }} {{ + Ok(m) => 0, + Err(e) => e.to_errno(), }} }} @@ -359,7 +353,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { // called once and `__init` was already called. unsafe {{ // Invokes `drop()` on `__MOD`, which should be used for cleanup. - __MOD = None; + __MOD.assume_init_drop(); }} }} From d88dd907494978b8cf811dcd876fbb9eefad766b Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 15 Oct 2024 16:04:37 +0200 Subject: [PATCH 29/51] rust: pass module name to `Module::init` In a subsequent patch we introduce the `Registration` abstraction used to register driver structures. Some subsystems require the module name on driver registration (e.g. PCI in __pci_register_driver()), hence pass the module name to `Module::init`. Signed-off-by: Danilo Krummrich --- drivers/block/rnull.rs | 2 +- rust/kernel/lib.rs | 14 ++++++++++---- rust/kernel/net/phy.rs | 2 +- rust/macros/module.rs | 3 ++- samples/rust/rust_minimal.rs | 2 +- samples/rust/rust_print.rs | 2 +- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/drivers/block/rnull.rs b/drivers/block/rnull.rs index 5de7223beb4d5b..0e0e9ed7851e96 100644 --- a/drivers/block/rnull.rs +++ b/drivers/block/rnull.rs @@ -36,7 +36,7 @@ struct NullBlkModule { } impl kernel::Module for NullBlkModule { - fn init(_module: &'static ThisModule) -> Result { + fn init(_name: &'static CStr, _module: &'static ThisModule) -> Result { pr_info!("Rust null_blk loaded\n"); let tagset = Arc::pin_init(TagSet::new(1, 256, 1), flags::GFP_KERNEL)?; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 435896e160e26d..b12bad737e1696 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -80,7 +80,7 @@ pub trait Module: Sized + Sync + Send { /// should do. /// /// Equivalent to the `module_init` macro in the C API. - fn init(module: &'static ThisModule) -> error::Result; + fn init(name: &'static str::CStr, module: &'static ThisModule) -> error::Result; } /// A module that is pinned and initialised in-place. @@ -88,13 +88,19 @@ pub trait InPlaceModule: Sync + Send { /// Creates an initialiser for the module. /// /// It is called when the module is loaded. - fn init(module: &'static ThisModule) -> impl init::PinInit; + fn init( + name: &'static str::CStr, + module: &'static ThisModule, + ) -> impl init::PinInit; } impl InPlaceModule for T { - fn init(module: &'static ThisModule) -> impl init::PinInit { + fn init( + name: &'static str::CStr, + module: &'static ThisModule, + ) -> impl init::PinInit { let initer = move |slot: *mut Self| { - let m = ::init(module)?; + let m = ::init(name, module)?; // SAFETY: `slot` is valid for write per the contract with `pin_init_from_closure`. unsafe { slot.write(m) }; diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs index 910ce867480a8a..26631b0e127ac0 100644 --- a/rust/kernel/net/phy.rs +++ b/rust/kernel/net/phy.rs @@ -899,7 +899,7 @@ macro_rules! module_phy_driver { [$($crate::net::phy::create_phy_driver::<$driver>()),+]; impl $crate::Module for Module { - fn init(module: &'static ThisModule) -> Result { + fn init(_name: &'static CStr, module: &'static ThisModule) -> Result { // SAFETY: The anonymous constant guarantees that nobody else can access // the `DRIVERS` static. The array is used only in the C side. let drivers = unsafe { &mut DRIVERS }; diff --git a/rust/macros/module.rs b/rust/macros/module.rs index ba2d6d36e0389d..471b427bb8b6cc 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -332,7 +332,8 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { /// /// This function must only be called once. unsafe fn __init() -> core::ffi::c_int {{ - let initer = <{type_} as kernel::InPlaceModule>::init(&super::super::THIS_MODULE); + let initer = <{type_} as kernel::InPlaceModule>::init(kernel::c_str!(\"{name}\"), + &super::super::THIS_MODULE); // SAFETY: No data race, since `__MOD` can only be accessed by this module // and there only `__init` and `__exit` access it. These functions are only // called once and `__exit` cannot be called before or during `__init`. diff --git a/samples/rust/rust_minimal.rs b/samples/rust/rust_minimal.rs index 4aaf117bf8e3c0..1577dc34e563d4 100644 --- a/samples/rust/rust_minimal.rs +++ b/samples/rust/rust_minimal.rs @@ -17,7 +17,7 @@ struct RustMinimal { } impl kernel::Module for RustMinimal { - fn init(_module: &'static ThisModule) -> Result { + fn init(_name: &'static CStr, _module: &'static ThisModule) -> Result { pr_info!("Rust minimal sample (init)\n"); pr_info!("Am I built-in? {}\n", !cfg!(MODULE)); diff --git a/samples/rust/rust_print.rs b/samples/rust/rust_print.rs index ba1606bdbd7543..73763ea2dc09d2 100644 --- a/samples/rust/rust_print.rs +++ b/samples/rust/rust_print.rs @@ -41,7 +41,7 @@ fn arc_print() -> Result { } impl kernel::Module for RustPrint { - fn init(_module: &'static ThisModule) -> Result { + fn init(_name: &'static CStr, _module: &'static ThisModule) -> Result { pr_info!("Rust printing macros sample (init)\n"); pr_emerg!("Emergency message (level 0) without args\n"); From 28f06ce2ef6a56e74a35974871838545e34ffe76 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 15 Oct 2024 16:05:18 +0200 Subject: [PATCH 30/51] rust: implement generic driver registration Implement the generic `Registration` type and the `DriverOps` trait. The `Registration` structure is the common type that represents a driver registration and is typically bound to the lifetime of a module. However, it doesn't implement actual calls to the kernel's driver core to register drivers itself. Instead the `DriverOps` trait is provided to subsystems, which have to implement `DriverOps::register` and `DrvierOps::unregister`. Subsystems have to provide an implementation for both of those methods where the subsystem specific variants to register / unregister a driver have to implemented. For instance, the PCI subsystem would call __pci_register_driver() from `DriverOps::register` and pci_unregister_driver() from `DrvierOps::unregister`. Co-developed-by: Wedson Almeida Filho Signed-off-by: Wedson Almeida Filho Signed-off-by: Danilo Krummrich --- rust/kernel/driver.rs | 120 ++++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 2 files changed, 121 insertions(+) create mode 100644 rust/kernel/driver.rs diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs new file mode 100644 index 00000000000000..f9766038b14e54 --- /dev/null +++ b/rust/kernel/driver.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic support for drivers of different buses (e.g., PCI, Platform, Amba, etc.). +//! +//! Each bus / subsystem is expected to implement [`RegistrationOps`], which allows drivers to +//! register using the [`Registration`] class. + +use crate::error::{Error, Result}; +use crate::{init::PinInit, str::CStr, try_pin_init, types::Opaque, ThisModule}; +use core::pin::Pin; +use macros::{pin_data, pinned_drop}; + +/// The [`RegistrationOps`] trait serves as generic interface for subsystems (e.g., PCI, Platform, +/// Amba, etc.) to privide the corresponding subsystem specific implementation to register / +/// unregister a driver of the particular type (`RegType`). +/// +/// For instance, the PCI subsystem would set `RegType` to `bindings::pci_driver` and call +/// `bindings::__pci_register_driver` from `RegistrationOps::register` and +/// `bindings::pci_unregister_driver` from `RegistrationOps::unregister`. +pub trait RegistrationOps { + /// The type that holds information about the registration. This is typically a struct defined + /// by the C portion of the kernel. + type RegType: Default; + + /// Registers a driver. + /// + /// On success, `reg` must remain pinned and valid until the matching call to + /// [`RegistrationOps::unregister`]. + fn register( + reg: &mut Self::RegType, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result; + + /// Unregisters a driver previously registered with [`RegistrationOps::register`]. + fn unregister(reg: &mut Self::RegType); +} + +/// A [`Registration`] is a generic type that represents the registration of some driver type (e.g. +/// `bindings::pci_driver`). Therefore a [`Registration`] must be initialized with a type that +/// implements the [`RegistrationOps`] trait, such that the generic `T::register` and `T::unregister` +/// calls result in the subsystem specific registration calls. +/// +///Once the `Registration` structure is dropped, the driver is unregistered. +#[pin_data(PinnedDrop)] +pub struct Registration { + #[pin] + reg: Opaque, +} + +// SAFETY: `Registration` has no fields or methods accessible via `&Registration`, so it is safe to +// share references to it with multiple threads as nothing can be done. +unsafe impl Sync for Registration {} + +// SAFETY: Both registration and unregistration are implemented in C and safe to be performed from +// any thread, so `Registration` is `Send`. +unsafe impl Send for Registration {} + +impl Registration { + /// Creates a new instance of the registration object. + pub fn new(name: &'static CStr, module: &'static ThisModule) -> impl PinInit { + try_pin_init!(Self { + reg <- Opaque::try_ffi_init(|ptr: *mut T::RegType| { + // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write. + unsafe { ptr.write(T::RegType::default()) }; + + // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write, and it has + // just been initialised above, so it's also valid for read. + let drv = unsafe { &mut *ptr }; + + T::register(drv, name, module) + }), + }) + } +} + +#[pinned_drop] +impl PinnedDrop for Registration { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The existance of the `Registration` guarantees that `self.reg.get()` is properly + // aligned and points to a valid value. + let drv = unsafe { &mut *self.reg.get() }; + + T::unregister(drv); + } +} + +/// A kernel module that only registers the given driver on init. +/// +/// This is a helper struct to make it easier to define single-functionality modules, in this case, +/// modules that offer a single driver. +#[pin_data] +pub struct Module { + #[pin] + _driver: Registration, +} + +impl crate::InPlaceModule for Module { + fn init(name: &'static CStr, module: &'static ThisModule) -> impl PinInit { + try_pin_init!(Self { + _driver <- Registration::::new(name, module), + }) + } +} + +/// Declares a kernel module that exposes a single driver. +/// +/// It is meant to be used as a helper by other subsystems so they can more easily expose their own +/// macros. +#[macro_export] +macro_rules! module_driver { + (<$gen_type:ident>, $driver_ops:ty, { type: $type:ty, $($f:tt)* }) => { + type Ops<$gen_type> = $driver_ops; + type ModuleType = $crate::driver::Module>; + $crate::prelude::module! { + type: ModuleType, + $($f)* + } + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index b12bad737e1696..63b0cee31eb955 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -32,6 +32,7 @@ pub mod alloc; pub mod block; mod build_assert; pub mod device; +pub mod driver; pub mod error; #[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)] pub mod firmware; From 10ccf327add8b045c4c1e836d278839181b44b19 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 15 Oct 2024 16:05:39 +0200 Subject: [PATCH 31/51] rust: implement `IdArray`, `IdTable` and `RawDeviceId` Most subsystems use some kind of ID to match devices and drivers. Hence, we have to provide Rust drivers an abstraction to register an ID table for the driver to match. Generally, those IDs are subsystem specific and hence need to be implemented by the corresponding subsystem. However, the `IdArray`, `IdTable` and `RawDeviceId` types provide a generalized implementation that makes the life of subsystems easier to do so. Co-developed-by: Wedson Almeida Filho Signed-off-by: Wedson Almeida Filho Co-developed-by: Gary Guo Signed-off-by: Gary Guo Signed-off-by: Danilo Krummrich --- rust/kernel/device_id.rs | 126 +++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 7 +++ 2 files changed, 133 insertions(+) create mode 100644 rust/kernel/device_id.rs diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs new file mode 100644 index 00000000000000..0c856181b1c529 --- /dev/null +++ b/rust/kernel/device_id.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic implementation of device IDs. +//! +//! Each bus / subsystem that matches device and driver through a bus / subsystem specific ID is +//! expected to implement [`RawDeviceId`]. + +use core::mem::MaybeUninit; + +/// Marker trait to indicate a Rust device ID type represents a corresponding C device ID type. +/// +/// This is meant to be implemented by buses/subsystems so that they can use [`IdTable`] to +/// guarantee (at compile-time) zero-termination of device id tables provided by drivers. +/// +/// # Safety +/// +/// Implementers must ensure that: +/// - `Self` is layout-compatible with [`RawDeviceId::RawType`]; i.e. it's safe to transmute to +/// `RawDeviceId`. +/// +/// This requirement is needed so `IdArray::new` can convert `Self` to `RawType` when building +/// the ID table. +/// +/// Ideally, this should be achieved using a const function that does conversion instead of +/// transmute; however, const trait functions relies on `const_trait_impl` unstable feature, +/// which is broken/gone in Rust 1.73. +/// +/// - `DRIVER_DATA_OFFSET` is the offset of context/data field of the device ID (usually named +/// `driver_data`) of the device ID, the field is suitable sized to write a `usize` value. +/// +/// Similar to the previous requirement, the data should ideally be added during `Self` to +/// `RawType` conversion, but there's currently no way to do it when using traits in const. +pub unsafe trait RawDeviceId { + /// The raw type that holds the device id. + /// + /// Id tables created from [`Self`] are going to hold this type in its zero-terminated array. + type RawType: Copy; + + /// The offest to the context/data field. + const DRIVER_DATA_OFFSET: usize; +} + +/// A zero-terminated device id array, followed by context data. +#[repr(C)] +pub struct IdArray { + ids: [T::RawType; N], + sentinel: MaybeUninit, + id_infos: [U; N], +} + +impl IdArray { + /// Creates a new instance of the array. + /// + /// The contents are derived from the given identifiers and context information. + pub const fn new(ids: [(T, U); N]) -> Self { + let mut raw_ids = [const { MaybeUninit::::uninit() }; N]; + let mut infos = [const { MaybeUninit::uninit() }; N]; + + let mut i = 0usize; + while i < N { + // SAFETY: by the safety requirement of `RawDeviceId`, we're guaranteed that `T` is + // layout-wise compatible with `RawType`. + raw_ids[i] = unsafe { core::mem::transmute_copy(&ids[i].0) }; + // SAFETY: by the safety requirement of `RawDeviceId`, this would be effectively + // `raw_ids[i].driver_data = i;`. + unsafe { + raw_ids[i] + .as_mut_ptr() + .byte_offset(T::DRIVER_DATA_OFFSET as _) + .cast::() + .write(i); + } + + // SAFETY: this is effectively a move: `infos[i] = ids[i].1`. We make a copy here but + // later forget `ids`. + infos[i] = MaybeUninit::new(unsafe { core::ptr::read(&ids[i].1) }); + i += 1; + } + + core::mem::forget(ids); + + Self { + // SAFETY: this is effectively `array_assume_init`, which is unstable, so we use + // `transmute_copy` instead. We have initialized all elements of `raw_ids` so this + // `array_assume_init` is safe. + ids: unsafe { core::mem::transmute_copy(&raw_ids) }, + sentinel: MaybeUninit::zeroed(), + // SAFETY: We have initialized all elements of `infos` so this `array_assume_init` is + // safe. + id_infos: unsafe { core::mem::transmute_copy(&infos) }, + } + } +} + +/// A device id table. +/// +/// This trait is only implemented by `IdArray`. +/// +/// The purpose of this trait is to allow `&'static dyn IdArray` to be in context when `N` in +/// `IdArray` doesn't matter. +pub trait IdTable { + /// Obtain the pointer to the ID table. + fn as_ptr(&self) -> *const T::RawType; + + /// Obtain the pointer to the bus specific device ID from an index. + fn id(&self, index: usize) -> &T::RawType; + + /// Obtain the pointer to the driver-specific information from an index. + fn info(&self, index: usize) -> &U; +} + +impl IdTable for IdArray { + fn as_ptr(&self) -> *const T::RawType { + // This cannot be `self.ids.as_ptr()`, as the return pointer must have correct provenance + // to access the sentinel. + (self as *const Self).cast() + } + + fn id(&self, index: usize) -> &T::RawType { + &self.ids[index] + } + + fn info(&self, index: usize) -> &U { + &self.id_infos[index] + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 63b0cee31eb955..b9f84e0bddbad1 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -14,10 +14,16 @@ #![no_std] #![feature(arbitrary_self_types)] #![feature(coerce_unsized)] +#![feature(const_refs_to_cell)] #![feature(dispatch_from_dyn)] #![feature(inline_const)] #![feature(lint_reasons)] #![feature(unsize)] +#![allow(stable_features)] +// Stable in Rust 1.83 +#![feature(const_mut_refs)] +#![feature(const_ptr_write)] +#![feature(const_maybe_uninit_as_mut_ptr)] // Ensure conditional compilation based on the kernel configuration works; // otherwise we may silently break things like initcall handling. @@ -32,6 +38,7 @@ pub mod alloc; pub mod block; mod build_assert; pub mod device; +pub mod device_id; pub mod driver; pub mod error; #[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)] From ec2e215b7ff7d372b1aa42d85f6c4d16b8f00d43 Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Mon, 4 Sep 2023 13:21:40 +0200 Subject: [PATCH 32/51] rust: add rcu abstraction Add a simple abstraction to guard critical code sections with an rcu read lock. Signed-off-by: Wedson Almeida Filho Signed-off-by: Danilo Krummrich --- rust/helpers/helpers.c | 1 + rust/helpers/rcu.c | 13 +++++++++++ rust/kernel/sync.rs | 1 + rust/kernel/sync/rcu.rs | 52 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+) create mode 100644 rust/helpers/rcu.c create mode 100644 rust/kernel/sync/rcu.rs diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 20a0c69d5cc7b8..0720debccdd483 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -16,6 +16,7 @@ #include "mutex.c" #include "page.c" #include "rbtree.c" +#include "rcu.c" #include "refcount.c" #include "signal.c" #include "slab.c" diff --git a/rust/helpers/rcu.c b/rust/helpers/rcu.c new file mode 100644 index 00000000000000..f1cec658351349 --- /dev/null +++ b/rust/helpers/rcu.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void rust_helper_rcu_read_lock(void) +{ + rcu_read_lock(); +} + +void rust_helper_rcu_read_unlock(void) +{ + rcu_read_unlock(); +} diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 0ab20975a3b5db..1806767359fec2 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -11,6 +11,7 @@ mod arc; mod condvar; pub mod lock; mod locked_by; +pub mod rcu; pub use arc::{Arc, ArcBorrow, UniqueArc}; pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs new file mode 100644 index 00000000000000..5a35495f69a46b --- /dev/null +++ b/rust/kernel/sync/rcu.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! RCU support. +//! +//! C header: [`include/linux/rcupdate.h`](srctree/include/linux/rcupdate.h) + +use crate::bindings; +use core::marker::PhantomData; + +/// Evidence that the RCU read side lock is held on the current thread/CPU. +/// +/// The type is explicitly not `Send` because this property is per-thread/CPU. +/// +/// # Invariants +/// +/// The RCU read side lock is actually held while instances of this guard exist. +pub struct Guard { + _not_send: PhantomData<*mut ()>, +} + +impl Guard { + /// Acquires the RCU read side lock and returns a guard. + pub fn new() -> Self { + // SAFETY: An FFI call with no additional requirements. + unsafe { bindings::rcu_read_lock() }; + // INVARIANT: The RCU read side lock was just acquired above. + Self { + _not_send: PhantomData, + } + } + + /// Explicitly releases the RCU read side lock. + pub fn unlock(self) {} +} + +impl Default for Guard { + fn default() -> Self { + Self::new() + } +} + +impl Drop for Guard { + fn drop(&mut self) { + // SAFETY: By the type invariants, the rcu read side is locked, so it is ok to unlock it. + unsafe { bindings::rcu_read_unlock() }; + } +} + +/// Acquires the RCU read side lock. +pub fn read_lock() -> Guard { + Guard::new() +} From 9730145e4a242d471dd4bdda56fc9af0726926c2 Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Mon, 4 Sep 2023 13:25:41 +0200 Subject: [PATCH 33/51] rust: add `Revocable` type Revocable allows access to objects to be safely revoked at run time. This is useful, for example, for resources allocated during device probe; when the device is removed, the driver should stop accessing the device resources even if another state is kept in memory due to existing references (i.e., device context data is ref-counted and has a non-zero refcount after removal of the device). Signed-off-by: Wedson Almeida Filho Signed-off-by: Danilo Krummrich --- rust/kernel/lib.rs | 1 + rust/kernel/revocable.rs | 211 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 rust/kernel/revocable.rs diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index b9f84e0bddbad1..32895ae451b851 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -54,6 +54,7 @@ pub mod page; pub mod prelude; pub mod print; pub mod rbtree; +pub mod revocable; pub mod sizes; mod static_assert; #[doc(hidden)] diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs new file mode 100644 index 00000000000000..83455558d7957f --- /dev/null +++ b/rust/kernel/revocable.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Revocable objects. +//! +//! The [`Revocable`] type wraps other types and allows access to them to be revoked. The existence +//! of a [`RevocableGuard`] ensures that objects remain valid. + +use crate::{ + bindings, + init::{self}, + prelude::*, + sync::rcu, +}; +use core::{ + cell::UnsafeCell, + marker::PhantomData, + mem::MaybeUninit, + ops::Deref, + ptr::drop_in_place, + sync::atomic::{AtomicBool, Ordering}, +}; + +/// An object that can become inaccessible at runtime. +/// +/// Once access is revoked and all concurrent users complete (i.e., all existing instances of +/// [`RevocableGuard`] are dropped), the wrapped object is also dropped. +/// +/// # Examples +/// +/// ``` +/// # use kernel::revocable::Revocable; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn add_two(v: &Revocable) -> Option { +/// let guard = v.try_access()?; +/// Some(guard.a + guard.b) +/// } +/// +/// let v = KBox::pin_init(Revocable::new(Example { a: 10, b: 20 }), GFP_KERNEL).unwrap(); +/// assert_eq!(add_two(&v), Some(30)); +/// v.revoke(); +/// assert_eq!(add_two(&v), None); +/// ``` +/// +/// Sample example as above, but explicitly using the rcu read side lock. +/// +/// ``` +/// # use kernel::revocable::Revocable; +/// use kernel::sync::rcu; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn add_two(v: &Revocable) -> Option { +/// let guard = rcu::read_lock(); +/// let e = v.try_access_with_guard(&guard)?; +/// Some(e.a + e.b) +/// } +/// +/// let v = KBox::pin_init(Revocable::new(Example { a: 10, b: 20 }), GFP_KERNEL).unwrap(); +/// assert_eq!(add_two(&v), Some(30)); +/// v.revoke(); +/// assert_eq!(add_two(&v), None); +/// ``` +#[pin_data(PinnedDrop)] +pub struct Revocable { + is_available: AtomicBool, + #[pin] + data: MaybeUninit>, +} + +// SAFETY: `Revocable` is `Send` if the wrapped object is also `Send`. This is because while the +// functionality exposed by `Revocable` can be accessed from any thread/CPU, it is possible that +// this isn't supported by the wrapped object. +unsafe impl Send for Revocable {} + +// SAFETY: `Revocable` is `Sync` if the wrapped object is both `Send` and `Sync`. We require `Send` +// from the wrapped object as well because of `Revocable::revoke`, which can trigger the `Drop` +// implementation of the wrapped object from an arbitrary thread. +unsafe impl Sync for Revocable {} + +impl Revocable { + /// Creates a new revocable instance of the given data. + pub fn new(data: impl PinInit) -> impl PinInit { + pin_init!(Self { + is_available: AtomicBool::new(true), + // SAFETY: The closure only returns `Ok(())` if `slot` is fully initialized; on error + // `slot` is not partially initialized and does not need to be dropped. + data <- unsafe { + init::pin_init_from_closure(move |slot: *mut MaybeUninit>| { + init::PinInit::::__pinned_init(data, + slot as *mut T)?; + Ok::<(), core::convert::Infallible>(()) + }) + }, + }) + } + + /// Tries to access the \[revocable\] wrapped object. + /// + /// Returns `None` if the object has been revoked and is therefore no longer accessible. + /// + /// Returns a guard that gives access to the object otherwise; the object is guaranteed to + /// remain accessible while the guard is alive. In such cases, callers are not allowed to sleep + /// because another CPU may be waiting to complete the revocation of this object. + pub fn try_access(&self) -> Option> { + let guard = rcu::read_lock(); + if self.is_available.load(Ordering::Relaxed) { + // SAFETY: Since `self.is_available` is true, data is initialised and has to remain + // valid because the RCU read side lock prevents it from being dropped. + Some(unsafe { RevocableGuard::new(self.data.assume_init_ref().get(), guard) }) + } else { + None + } + } + + /// Tries to access the \[revocable\] wrapped object. + /// + /// Returns `None` if the object has been revoked and is therefore no longer accessible. + /// + /// Returns a shared reference to the object otherwise; the object is guaranteed to + /// remain accessible while the rcu read side guard is alive. In such cases, callers are not + /// allowed to sleep because another CPU may be waiting to complete the revocation of this + /// object. + pub fn try_access_with_guard<'a>(&'a self, _guard: &'a rcu::Guard) -> Option<&'a T> { + if self.is_available.load(Ordering::Relaxed) { + // SAFETY: Since `self.is_available` is true, data is initialised and has to remain + // valid because the RCU read side lock prevents it from being dropped. + Some(unsafe { &*self.data.assume_init_ref().get() }) + } else { + None + } + } + + /// Revokes access to and drops the wrapped object. + /// + /// Access to the object is revoked immediately to new callers of [`Revocable::try_access`]. If + /// there are concurrent users of the object (i.e., ones that called [`Revocable::try_access`] + /// beforehand and still haven't dropped the returned guard), this function waits for the + /// concurrent access to complete before dropping the wrapped object. + pub fn revoke(&self) { + if self + .is_available + .compare_exchange(true, false, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + // SAFETY: Just an FFI call, there are no further requirements. + unsafe { bindings::synchronize_rcu() }; + + // SAFETY: We know `self.data` is valid because only one CPU can succeed the + // `compare_exchange` above that takes `is_available` from `true` to `false`. + unsafe { drop_in_place(self.data.assume_init_ref().get()) }; + } + } +} + +#[pinned_drop] +impl PinnedDrop for Revocable { + fn drop(self: Pin<&mut Self>) { + // Drop only if the data hasn't been revoked yet (in which case it has already been + // dropped). + // SAFETY: We are not moving out of `p`, only dropping in place + let p = unsafe { self.get_unchecked_mut() }; + if *p.is_available.get_mut() { + // SAFETY: We know `self.data` is valid because no other CPU has changed + // `is_available` to `false` yet, and no other CPU can do it anymore because this CPU + // holds the only reference (mutable) to `self` now. + unsafe { drop_in_place(p.data.assume_init_ref().get()) }; + } + } +} + +/// A guard that allows access to a revocable object and keeps it alive. +/// +/// CPUs may not sleep while holding on to [`RevocableGuard`] because it's in atomic context +/// holding the RCU read-side lock. +/// +/// # Invariants +/// +/// The RCU read-side lock is held while the guard is alive. +pub struct RevocableGuard<'a, T> { + data_ref: *const T, + _rcu_guard: rcu::Guard, + _p: PhantomData<&'a ()>, +} + +impl RevocableGuard<'_, T> { + fn new(data_ref: *const T, rcu_guard: rcu::Guard) -> Self { + Self { + data_ref, + _rcu_guard: rcu_guard, + _p: PhantomData, + } + } +} + +impl Deref for RevocableGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: By the type invariants, we hold the rcu read-side lock, so the object is + // guaranteed to remain valid. + unsafe { &*self.data_ref } + } +} From f444613596fa24030f95ba0d30a200b16972af51 Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Thu, 16 Feb 2023 18:05:48 +0900 Subject: [PATCH 34/51] rust: add `dev_*` print macros. Implement `dev_*` print macros for `device::Device`. They behave like the macros with the same names in C, i.e., they print messages to the kernel ring buffer with the given level, prefixing the messages with corresponding device information. Signed-off-by: Wedson Almeida Filho Signed-off-by: Danilo Krummrich --- rust/kernel/device.rs | 319 ++++++++++++++++++++++++++++++++++++++++- rust/kernel/prelude.rs | 2 + 2 files changed, 320 insertions(+), 1 deletion(-) diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 851018eef885e7..36d5e11e8e5ba7 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -8,7 +8,10 @@ use crate::{ bindings, types::{ARef, Opaque}, }; -use core::ptr; +use core::{fmt, ptr}; + +#[cfg(CONFIG_PRINTK)] +use crate::c_str; /// A reference-counted device. /// @@ -82,6 +85,110 @@ impl Device { // SAFETY: Guaranteed by the safety requirements of the function. unsafe { &*ptr.cast() } } + + /// Prints an emergency-level message (level 0) prefixed with device information. + /// + /// More details are available from [`dev_emerg`]. + /// + /// [`dev_emerg`]: crate::dev_emerg + pub fn pr_emerg(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_EMERG, args) }; + } + + /// Prints an alert-level message (level 1) prefixed with device information. + /// + /// More details are available from [`dev_alert`]. + /// + /// [`dev_alert`]: crate::dev_alert + pub fn pr_alert(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_ALERT, args) }; + } + + /// Prints a critical-level message (level 2) prefixed with device information. + /// + /// More details are available from [`dev_crit`]. + /// + /// [`dev_crit`]: crate::dev_crit + pub fn pr_crit(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_CRIT, args) }; + } + + /// Prints an error-level message (level 3) prefixed with device information. + /// + /// More details are available from [`dev_err`]. + /// + /// [`dev_err`]: crate::dev_err + pub fn pr_err(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_ERR, args) }; + } + + /// Prints a warning-level message (level 4) prefixed with device information. + /// + /// More details are available from [`dev_warn`]. + /// + /// [`dev_warn`]: crate::dev_warn + pub fn pr_warn(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_WARNING, args) }; + } + + /// Prints a notice-level message (level 5) prefixed with device information. + /// + /// More details are available from [`dev_notice`]. + /// + /// [`dev_notice`]: crate::dev_notice + pub fn pr_notice(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_NOTICE, args) }; + } + + /// Prints an info-level message (level 6) prefixed with device information. + /// + /// More details are available from [`dev_info`]. + /// + /// [`dev_info`]: crate::dev_info + pub fn pr_info(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_INFO, args) }; + } + + /// Prints a debug-level message (level 7) prefixed with device information. + /// + /// More details are available from [`dev_dbg`]. + /// + /// [`dev_dbg`]: crate::dev_dbg + pub fn pr_dbg(&self, args: fmt::Arguments<'_>) { + if cfg!(debug_assertions) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_DEBUG, args) }; + } + } + + /// Prints the provided message to the console. + /// + /// # Safety + /// + /// Callers must ensure that `klevel` is null-terminated; in particular, one of the + /// `KERN_*`constants, for example, `KERN_CRIT`, `KERN_ALERT`, etc. + #[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] + unsafe fn printk(&self, klevel: &[u8], msg: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated and one of the kernel constants. `self.as_raw` + // is valid because `self` is valid. The "%pA" format string expects a pointer to + // `fmt::Arguments`, which is what we're passing as the last argument. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_dev_printk( + klevel as *const _ as *const core::ffi::c_char, + self.as_raw(), + c_str!("%pA").as_char_ptr(), + &msg as *const _ as *const core::ffi::c_void, + ) + }; + } } // SAFETY: Instances of `Device` are always reference-counted. @@ -103,3 +210,213 @@ unsafe impl Send for Device {} // SAFETY: `Device` can be shared among threads because all immutable methods are protected by the // synchronization in `struct device`. unsafe impl Sync for Device {} + +#[doc(hidden)] +#[macro_export] +macro_rules! dev_printk { + ($method:ident, $dev:expr, $($f:tt)*) => { + { + ($dev).$method(core::format_args!($($f)*)); + } + } +} + +/// Prints an emergency-level message (level 0) prefixed with device information. +/// +/// This level should be used if the system is unusable. +/// +/// Equivalent to the kernel's `dev_emerg` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_emerg!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_emerg { + ($($f:tt)*) => { $crate::dev_printk!(pr_emerg, $($f)*); } +} + +/// Prints an alert-level message (level 1) prefixed with device information. +/// +/// This level should be used if action must be taken immediately. +/// +/// Equivalent to the kernel's `dev_alert` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_alert!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_alert { + ($($f:tt)*) => { $crate::dev_printk!(pr_alert, $($f)*); } +} + +/// Prints a critical-level message (level 2) prefixed with device information. +/// +/// This level should be used in critical conditions. +/// +/// Equivalent to the kernel's `dev_crit` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_crit!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_crit { + ($($f:tt)*) => { $crate::dev_printk!(pr_crit, $($f)*); } +} + +/// Prints an error-level message (level 3) prefixed with device information. +/// +/// This level should be used in error conditions. +/// +/// Equivalent to the kernel's `dev_err` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_err!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_err { + ($($f:tt)*) => { $crate::dev_printk!(pr_err, $($f)*); } +} + +/// Prints a warning-level message (level 4) prefixed with device information. +/// +/// This level should be used in warning conditions. +/// +/// Equivalent to the kernel's `dev_warn` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_warn!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_warn { + ($($f:tt)*) => { $crate::dev_printk!(pr_warn, $($f)*); } +} + +/// Prints a notice-level message (level 5) prefixed with device information. +/// +/// This level should be used in normal but significant conditions. +/// +/// Equivalent to the kernel's `dev_notice` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_notice!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_notice { + ($($f:tt)*) => { $crate::dev_printk!(pr_notice, $($f)*); } +} + +/// Prints an info-level message (level 6) prefixed with device information. +/// +/// This level should be used for informational messages. +/// +/// Equivalent to the kernel's `dev_info` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_info!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_info { + ($($f:tt)*) => { $crate::dev_printk!(pr_info, $($f)*); } +} + +/// Prints a debug-level message (level 7) prefixed with device information. +/// +/// This level should be used for debug messages. +/// +/// Equivalent to the kernel's `dev_dbg` macro, except that it doesn't support dynamic debug yet. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_dbg!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_dbg { + ($($f:tt)*) => { $crate::dev_printk!(pr_dbg, $($f)*); } +} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 8bdab9aa0d16bf..9ab4e0b6cbc912 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -24,6 +24,8 @@ pub use super::build_assert; // `super::std_vendor` is hidden, which makes the macro inline for some reason. #[doc(no_inline)] pub use super::dbg; +pub use super::fmt; +pub use super::{dev_alert, dev_crit, dev_dbg, dev_emerg, dev_err, dev_info, dev_notice, dev_warn}; pub use super::{pr_alert, pr_crit, pr_debug, pr_emerg, pr_err, pr_info, pr_notice, pr_warn}; pub use super::{init, pin_init, try_init, try_pin_init}; From b1c72ac2eeb2102aeb3c6d9b143e1fd404ed3b8f Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 15 Oct 2024 16:11:21 +0200 Subject: [PATCH 35/51] rust: add `io::Io` base type I/O memory is typically either mapped through direct calls to ioremap() or subsystem / bus specific ones such as pci_iomap(). Even though subsystem / bus specific functions to map I/O memory are based on ioremap() / iounmap() it is not desirable to re-implement them in Rust. Instead, implement a base type for I/O mapped memory, which generically provides the corresponding accessors, such as `Io::readb` or `Io:try_readb`. `Io` supports an optional const generic, such that a driver can indicate the minimal expected and required size of the mapping at compile time. Correspondingly, calls to the 'non-try' accessors, support compile time checks of the I/O memory offset to read / write, while the 'try' accessors, provide boundary checks on runtime. `Io` is meant to be embedded into a structure (e.g. pci::Bar or io::IoMem) which creates the actual I/O memory mapping and initializes `Io` accordingly. To ensure that I/O mapped memory can't out-live the device it may be bound to, subsystems should embedd the corresponding I/O memory type (e.g. pci::Bar) into a `Devres` container, such that it gets revoked once the device is unbound. Co-developed-by: Philipp Stanner Signed-off-by: Philipp Stanner Signed-off-by: Danilo Krummrich --- rust/helpers/helpers.c | 1 + rust/helpers/io.c | 91 ++++++++++++++++ rust/kernel/io.rs | 234 +++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 4 files changed, 327 insertions(+) create mode 100644 rust/helpers/io.c create mode 100644 rust/kernel/io.rs diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 0720debccdd483..e2f6b21970618f 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -12,6 +12,7 @@ #include "build_assert.c" #include "build_bug.c" #include "err.c" +#include "io.c" #include "kunit.c" #include "mutex.c" #include "page.c" diff --git a/rust/helpers/io.c b/rust/helpers/io.c new file mode 100644 index 00000000000000..f9bb1bbf1fd5da --- /dev/null +++ b/rust/helpers/io.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +u8 rust_helper_readb(const volatile void __iomem *addr) +{ + return readb(addr); +} + +u16 rust_helper_readw(const volatile void __iomem *addr) +{ + return readw(addr); +} + +u32 rust_helper_readl(const volatile void __iomem *addr) +{ + return readl(addr); +} + +#ifdef CONFIG_64BIT +u64 rust_helper_readq(const volatile void __iomem *addr) +{ + return readq(addr); +} +#endif + +void rust_helper_writeb(u8 value, volatile void __iomem *addr) +{ + writeb(value, addr); +} + +void rust_helper_writew(u16 value, volatile void __iomem *addr) +{ + writew(value, addr); +} + +void rust_helper_writel(u32 value, volatile void __iomem *addr) +{ + writel(value, addr); +} + +#ifdef CONFIG_64BIT +void rust_helper_writeq(u64 value, volatile void __iomem *addr) +{ + writeq(value, addr); +} +#endif + +u8 rust_helper_readb_relaxed(const volatile void __iomem *addr) +{ + return readb_relaxed(addr); +} + +u16 rust_helper_readw_relaxed(const volatile void __iomem *addr) +{ + return readw_relaxed(addr); +} + +u32 rust_helper_readl_relaxed(const volatile void __iomem *addr) +{ + return readl_relaxed(addr); +} + +#ifdef CONFIG_64BIT +u64 rust_helper_readq_relaxed(const volatile void __iomem *addr) +{ + return readq_relaxed(addr); +} +#endif + +void rust_helper_writeb_relaxed(u8 value, volatile void __iomem *addr) +{ + writeb_relaxed(value, addr); +} + +void rust_helper_writew_relaxed(u16 value, volatile void __iomem *addr) +{ + writew_relaxed(value, addr); +} + +void rust_helper_writel_relaxed(u32 value, volatile void __iomem *addr) +{ + writel_relaxed(value, addr); +} + +#ifdef CONFIG_64BIT +void rust_helper_writeq_relaxed(u64 value, volatile void __iomem *addr) +{ + writeq_relaxed(value, addr); +} +#endif diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs new file mode 100644 index 00000000000000..750af938f83e10 --- /dev/null +++ b/rust/kernel/io.rs @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory-mapped IO. +//! +//! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h) + +use crate::error::{code::EINVAL, Result}; +use crate::{bindings, build_assert}; + +/// IO-mapped memory, starting at the base address @addr and spanning @maxlen bytes. +/// +/// The creator (usually a subsystem / bus such as PCI) is responsible for creating the +/// mapping, performing an additional region request etc. +/// +/// # Invariant +/// +/// `addr` is the start and `maxsize` the length of valid I/O mapped memory region of size +/// `maxsize`. +/// +/// # Examples +/// +/// ```no_run +/// # use kernel::{bindings, io::Io}; +/// # use core::ops::Deref; +/// +/// // See also [`pci::Bar`] for a real example. +/// struct IoMem(Io); +/// +/// impl IoMem { +/// /// # Safety +/// /// +/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs +/// /// virtual address space. +/// unsafe fn new(paddr: usize) -> Result{ +/// +/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is +/// // valid for `ioremap`. +/// let addr = unsafe { bindings::ioremap(paddr as _, SIZE.try_into().unwrap()) }; +/// if addr.is_null() { +/// return Err(ENOMEM); +/// } +/// +/// // SAFETY: `addr` is guaranteed to be the start of a valid I/O mapped memory region of +/// // size `SIZE`. +/// let io = unsafe { Io::new(addr as _, SIZE)? }; +/// +/// Ok(IoMem(io)) +/// } +/// } +/// +/// impl Drop for IoMem { +/// fn drop(&mut self) { +/// // SAFETY: Safe as by the invariant of `Io`. +/// unsafe { bindings::iounmap(self.0.base_addr() as _); }; +/// } +/// } +/// +/// impl Deref for IoMem { +/// type Target = Io; +/// +/// fn deref(&self) -> &Self::Target { +/// &self.0 +/// } +/// } +/// +///# fn no_run() -> Result<(), Error> { +/// // SAFETY: Invalid usage for example purposes. +/// let iomem = unsafe { IoMem::<{ core::mem::size_of::() }>::new(0xBAAAAAAD)? }; +/// iomem.writel(0x42, 0x0); +/// assert!(iomem.try_writel(0x42, 0x0).is_ok()); +/// assert!(iomem.try_writel(0x42, 0x4).is_err()); +/// # Ok(()) +/// # } +/// ``` +pub struct Io { + addr: usize, + maxsize: usize, +} + +macro_rules! define_read { + ($(#[$attr:meta])* $name:ident, $try_name:ident, $type_name:ty) => { + /// Read IO data from a given offset known at compile time. + /// + /// Bound checks are performed on compile time, hence if the offset is not known at compile + /// time, the build will fail. + $(#[$attr])* + #[inline] + pub fn $name(&self, offset: usize) -> $type_name { + let addr = self.io_addr_assert::<$type_name>(offset); + + // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. + unsafe { bindings::$name(addr as _) } + } + + /// Read IO data from a given offset. + /// + /// Bound checks are performed on runtime, it fails if the offset (plus the type size) is + /// out of bounds. + $(#[$attr])* + pub fn $try_name(&self, offset: usize) -> Result<$type_name> { + let addr = self.io_addr::<$type_name>(offset)?; + + // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. + Ok(unsafe { bindings::$name(addr as _) }) + } + }; +} + +macro_rules! define_write { + ($(#[$attr:meta])* $name:ident, $try_name:ident, $type_name:ty) => { + /// Write IO data from a given offset known at compile time. + /// + /// Bound checks are performed on compile time, hence if the offset is not known at compile + /// time, the build will fail. + $(#[$attr])* + #[inline] + pub fn $name(&self, value: $type_name, offset: usize) { + let addr = self.io_addr_assert::<$type_name>(offset); + + // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. + unsafe { bindings::$name(value, addr as _, ) } + } + + /// Write IO data from a given offset. + /// + /// Bound checks are performed on runtime, it fails if the offset (plus the type size) is + /// out of bounds. + $(#[$attr])* + pub fn $try_name(&self, value: $type_name, offset: usize) -> Result { + let addr = self.io_addr::<$type_name>(offset)?; + + // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. + unsafe { bindings::$name(value, addr as _) } + Ok(()) + } + }; +} + +impl Io { + /// + /// + /// # Safety + /// + /// Callers must ensure that `addr` is the start of a valid I/O mapped memory region of size + /// `maxsize`. + pub unsafe fn new(addr: usize, maxsize: usize) -> Result { + if maxsize < SIZE { + return Err(EINVAL); + } + + // INVARIANT: Covered by the safety requirements of this function. + Ok(Self { addr, maxsize }) + } + + /// Returns the base address of this mapping. + #[inline] + pub fn base_addr(&self) -> usize { + self.addr + } + + /// Returns the size of this mapping. + #[inline] + pub fn maxsize(&self) -> usize { + self.maxsize + } + + #[inline] + const fn offset_valid(offset: usize, size: usize) -> bool { + let type_size = core::mem::size_of::(); + if let Some(end) = offset.checked_add(type_size) { + end <= size && offset % type_size == 0 + } else { + false + } + } + + #[inline] + fn io_addr(&self, offset: usize) -> Result { + if !Self::offset_valid::(offset, self.maxsize()) { + return Err(EINVAL); + } + + // Probably no need to check, since the safety requirements of `Self::new` guarantee that + // this can't overflow. + self.base_addr().checked_add(offset).ok_or(EINVAL) + } + + #[inline] + fn io_addr_assert(&self, offset: usize) -> usize { + build_assert!(Self::offset_valid::(offset, SIZE)); + + self.base_addr() + offset + } + + define_read!(readb, try_readb, u8); + define_read!(readw, try_readw, u16); + define_read!(readl, try_readl, u32); + define_read!( + #[cfg(CONFIG_64BIT)] + readq, + try_readq, + u64 + ); + + define_read!(readb_relaxed, try_readb_relaxed, u8); + define_read!(readw_relaxed, try_readw_relaxed, u16); + define_read!(readl_relaxed, try_readl_relaxed, u32); + define_read!( + #[cfg(CONFIG_64BIT)] + readq_relaxed, + try_readq_relaxed, + u64 + ); + + define_write!(writeb, try_writeb, u8); + define_write!(writew, try_writew, u16); + define_write!(writel, try_writel, u32); + define_write!( + #[cfg(CONFIG_64BIT)] + writeq, + try_writeq, + u64 + ); + + define_write!(writeb_relaxed, try_writeb_relaxed, u8); + define_write!(writew_relaxed, try_writew_relaxed, u16); + define_write!(writel_relaxed, try_writel_relaxed, u32); + define_write!( + #[cfg(CONFIG_64BIT)] + writeq_relaxed, + try_writeq_relaxed, + u64 + ); +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 32895ae451b851..fb6204dc22258b 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -70,6 +70,7 @@ pub mod workqueue; #[doc(hidden)] pub use bindings; +pub mod io; pub use macros; pub use uapi; From 5abe57e3a6fcd7282c94d00ae7ce2a8489e6a92c Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 15 Oct 2024 16:54:21 +0200 Subject: [PATCH 36/51] rust: add devres abstraction Add a Rust abstraction for the kernel's devres (device resource management) implementation. The Devres type acts as a container to manage the lifetime and accessibility of device bound resources. Therefore it registers a devres callback and revokes access to the resource on invocation. Users of the Devres abstraction can simply free the corresponding resources in their Drop implementation, which is invoked when either the Devres instance goes out of scope or the devres callback leads to the resource being revoked, which implies a call to drop_in_place(). Signed-off-by: Danilo Krummrich --- rust/helpers/device.c | 10 +++ rust/helpers/helpers.c | 1 + rust/kernel/devres.rs | 180 +++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 4 files changed, 192 insertions(+) create mode 100644 rust/helpers/device.c create mode 100644 rust/kernel/devres.rs diff --git a/rust/helpers/device.c b/rust/helpers/device.c new file mode 100644 index 00000000000000..b2135c6686b027 --- /dev/null +++ b/rust/helpers/device.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +int rust_helper_devm_add_action(struct device *dev, + void (*action)(void *), + void *data) +{ + return devm_add_action(dev, action, data); +} diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index e2f6b21970618f..3acb2b9e52ec2a 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -11,6 +11,7 @@ #include "bug.c" #include "build_assert.c" #include "build_bug.c" +#include "device.c" #include "err.c" #include "io.c" #include "kunit.c" diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs new file mode 100644 index 00000000000000..3511ff8ecef5a4 --- /dev/null +++ b/rust/kernel/devres.rs @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Devres abstraction +//! +//! [`Devres`] represents an abstraction for the kernel devres (device resource management) +//! implementation. + +use crate::{ + alloc::Flags, + bindings, + device::Device, + error::{Error, Result}, + prelude::*, + revocable::Revocable, + sync::Arc, +}; + +use core::ffi::c_void; +use core::ops::Deref; + +#[pin_data] +struct DevresInner { + #[pin] + data: Revocable, +} + +/// This abstraction is meant to be used by subsystems to containerize [`Device`] bound resources to +/// manage their lifetime. +/// +/// [`Device`] bound resources should be freed when either the resource goes out of scope or the +/// [`Device`] is unbound respectively, depending on what happens first. +/// +/// To achieve that [`Devres`] registers a devres callback on creation, which is called once the +/// [`Device`] is unbound, revoking access to the encapsulated resource (see also [`Revocable`]). +/// +/// After the [`Devres`] has been unbound it is not possible to access the encapsulated resource +/// anymore. +/// +/// [`Devres`] users should make sure to simply free the corresponding backing resource in `T`'s +/// [`Drop`] implementation. +/// +/// # Example +/// +/// ```no_run +/// # use kernel::{bindings, c_str, device::Device, devres::Devres, io::Io}; +/// # use core::ops::Deref; +/// +/// // See also [`pci::Bar`] for a real example. +/// struct IoMem(Io); +/// +/// impl IoMem { +/// /// # Safety +/// /// +/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs +/// /// virtual address space. +/// unsafe fn new(paddr: usize) -> Result{ +/// +/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is +/// // valid for `ioremap`. +/// let addr = unsafe { bindings::ioremap(paddr as _, SIZE.try_into().unwrap()) }; +/// if addr.is_null() { +/// return Err(ENOMEM); +/// } +/// +/// // SAFETY: `addr` is guaranteed to be the start of a valid I/O mapped memory region of +/// // size `SIZE`. +/// let io = unsafe { Io::new(addr as _, SIZE)? }; +/// +/// Ok(IoMem(io)) +/// } +/// } +/// +/// impl Drop for IoMem { +/// fn drop(&mut self) { +/// // SAFETY: Safe as by the invariant of `Io`. +/// unsafe { bindings::iounmap(self.0.base_addr() as _); }; +/// } +/// } +/// +/// impl Deref for IoMem { +/// type Target = Io; +/// +/// fn deref(&self) -> &Self::Target { +/// &self.0 +/// } +/// } +/// +/// # fn no_run() -> Result<(), Error> { +/// # // SAFETY: Invalid usage; just for the example to get an `ARef` instance. +/// # let dev = unsafe { Device::from_raw(core::ptr::null_mut()) }; +/// +/// // SAFETY: Invalid usage for example purposes. +/// let iomem = unsafe { IoMem::<{ core::mem::size_of::() }>::new(0xBAAAAAAD)? }; +/// let devres = Devres::new(&dev, iomem, GFP_KERNEL)?; +/// +/// let res = devres.try_access().ok_or(ENXIO)?; +/// res.writel(0x42, 0x0); +/// # Ok(()) +/// # } +/// ``` +pub struct Devres(Arc>); + +impl DevresInner { + fn new(dev: &Device, data: T, flags: Flags) -> Result>> { + let inner = Arc::pin_init( + pin_init!( DevresInner { + data <- Revocable::new(data), + }), + flags, + )?; + + // Convert `Arc` into a raw pointer and make devres own this reference until + // `Self::devres_callback` is called. + let data = inner.clone().into_raw(); + + // SAFETY: `devm_add_action` guarantees to call `Self::devres_callback` once `dev` is + // detached. + let ret = unsafe { + bindings::devm_add_action(dev.as_raw(), Some(Self::devres_callback), data as _) + }; + + if ret != 0 { + // SAFETY: We just created another reference to `inner` in order to pass it to + // `bindings::devm_add_action`. If `bindings::devm_add_action` fails, we have to drop + // this reference accordingly. + let _ = unsafe { Arc::from_raw(data) }; + return Err(Error::from_errno(ret)); + } + + Ok(inner) + } + + #[allow(clippy::missing_safety_doc)] + unsafe extern "C" fn devres_callback(ptr: *mut c_void) { + let ptr = ptr as *mut DevresInner; + // Devres owned this memory; now that we received the callback, drop the `Arc` and hence the + // reference. + // SAFETY: Safe, since we leaked an `Arc` reference to devm_add_action() in + // `DevresInner::new`. + let inner = unsafe { Arc::from_raw(ptr) }; + + inner.data.revoke(); + } +} + +impl Devres { + /// Creates a new [`Devres`] instance of the given `data`. The `data` encapsulated within the + /// returned `Devres` instance' `data` will be revoked once the device is detached. + pub fn new(dev: &Device, data: T, flags: Flags) -> Result { + let inner = DevresInner::new(dev, data, flags)?; + + Ok(Devres(inner)) + } + + /// Same as [Devres::new`], but does not return a `Devres` instance. Instead the given `data` + /// is owned by devres and will be revoked / dropped, once the device is detached. + pub fn new_foreign_owned(dev: &Device, data: T, flags: Flags) -> Result { + let _ = DevresInner::new(dev, data, flags)?; + + Ok(()) + } +} + +impl Deref for Devres { + type Target = Revocable; + + fn deref(&self) -> &Self::Target { + &self.0.data + } +} + +impl Drop for Devres { + fn drop(&mut self) { + // Revoke the data, such that it gets dropped already and the actual resource is freed. + // `DevresInner` has to stay alive until the devres callback has been called. This is + // necessary since we don't know when `Devres` is dropped and calling + // `devm_remove_action()` instead could race with `devres_release_all()`. + self.revoke(); + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index fb6204dc22258b..83fe4232b832c9 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -39,6 +39,7 @@ pub mod block; mod build_assert; pub mod device; pub mod device_id; +pub mod devres; pub mod driver; pub mod error; #[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)] From 265db154247d7edb7d01144120ac792425217f35 Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Sat, 19 Oct 2024 21:54:22 -0700 Subject: [PATCH 37/51] fixup! rust: add `dev_*` print macros. --- rust/kernel/device.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 36d5e11e8e5ba7..e20f1c7eb272d2 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -79,7 +79,7 @@ impl Device { /// /// Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count, /// i.e. it must be ensured that the reference count of the C `struct device` `ptr` points to - /// can't drop to zero, for the duration of this function call and the entire duration when the + /// can't drop to zero, for the duration of this function callhe entire duration when the /// returned reference exists. pub unsafe fn as_ref<'a>(ptr: *mut bindings::device) -> &'a Self { // SAFETY: Guaranteed by the safety requirements of the function. @@ -176,7 +176,7 @@ impl Device { /// `KERN_*`constants, for example, `KERN_CRIT`, `KERN_ALERT`, etc. #[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] unsafe fn printk(&self, klevel: &[u8], msg: fmt::Arguments<'_>) { - // SAFETY: `klevel` is null-terminated and one of the kernel constants. `self.as_raw` + // SAFETY: `klevel` is null-terminatedne of the kernel constants. `self.as_raw` // is valid because `self` is valid. The "%pA" format string expects a pointer to // `fmt::Arguments`, which is what we're passing as the last argument. #[cfg(CONFIG_PRINTK)] @@ -228,7 +228,7 @@ macro_rules! dev_printk { /// Equivalent to the kernel's `dev_emerg` macro. /// /// Mimics the interface of [`std::print!`]. More information about the syntax is available from -/// [`core::fmt`] and [`alloc::format!`]. +/// [`core::fmt`]. /// /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// @@ -253,7 +253,7 @@ macro_rules! dev_emerg { /// Equivalent to the kernel's `dev_alert` macro. /// /// Mimics the interface of [`std::print!`]. More information about the syntax is available from -/// [`core::fmt`] and [`alloc::format!`]. +/// [`core::fmt`]. /// /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// @@ -278,7 +278,7 @@ macro_rules! dev_alert { /// Equivalent to the kernel's `dev_crit` macro. /// /// Mimics the interface of [`std::print!`]. More information about the syntax is available from -/// [`core::fmt`] and [`alloc::format!`]. +/// [`core::fmt`]. /// /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// @@ -303,7 +303,7 @@ macro_rules! dev_crit { /// Equivalent to the kernel's `dev_err` macro. /// /// Mimics the interface of [`std::print!`]. More information about the syntax is available from -/// [`core::fmt`] and [`alloc::format!`]. +/// [`core::fmt`]. /// /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// @@ -328,7 +328,7 @@ macro_rules! dev_err { /// Equivalent to the kernel's `dev_warn` macro. /// /// Mimics the interface of [`std::print!`]. More information about the syntax is available from -/// [`core::fmt`] and [`alloc::format!`]. +/// [`core::fmt`]. /// /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// @@ -353,7 +353,7 @@ macro_rules! dev_warn { /// Equivalent to the kernel's `dev_notice` macro. /// /// Mimics the interface of [`std::print!`]. More information about the syntax is available from -/// [`core::fmt`] and [`alloc::format!`]. +/// [`core::fmt`]. /// /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// @@ -378,7 +378,7 @@ macro_rules! dev_notice { /// Equivalent to the kernel's `dev_info` macro. /// /// Mimics the interface of [`std::print!`]. More information about the syntax is available from -/// [`core::fmt`] and [`alloc::format!`]. +/// [`core::fmt`]. /// /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// @@ -403,7 +403,7 @@ macro_rules! dev_info { /// Equivalent to the kernel's `dev_dbg` macro, except that it doesn't support dynamic debug yet. /// /// Mimics the interface of [`std::print!`]. More information about the syntax is available from -/// [`core::fmt`] and [`alloc::format!`]. +/// [`core::fmt`]. /// /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// From 632bcffc18dc5103a938edf804840cf19e0bc0f7 Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Sat, 19 Oct 2024 21:55:02 -0700 Subject: [PATCH 38/51] fixup! rust: add devres abstraction --- rust/kernel/devres.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 3511ff8ecef5a4..b23559f55214ec 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -152,7 +152,7 @@ impl Devres { Ok(Devres(inner)) } - /// Same as [Devres::new`], but does not return a `Devres` instance. Instead the given `data` + /// Same as [`Devres::new`], but does not return a `Devres` instance. Instead the given `data` /// is owned by devres and will be revoked / dropped, once the device is detached. pub fn new_foreign_owned(dev: &Device, data: T, flags: Flags) -> Result { let _ = DevresInner::new(dev, data, flags)?; From eae8e6138e30d96934da442caef38a9a75827be2 Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Tue, 5 Mar 2024 19:39:54 -0800 Subject: [PATCH 39/51] rust: str: allow dereferencing BStr in a const ctx impl Deref doesn't work in const context. Add a function that is similar to implementing `deref` but that can be used in `const` context. Signed-off-by: Fabien Parent --- rust/kernel/str.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index aff6baa521d4a0..6ae301f64cedc8 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -31,6 +31,12 @@ impl BStr { // SAFETY: `BStr` is transparent to `[u8]`. unsafe { &*(bytes as *const [u8] as *const BStr) } } + + /// Returns a reference to the inner [u8]. + #[inline] + pub const fn deref_const(&self) -> &[u8] { + &self.0 + } } impl fmt::Display for BStr { @@ -102,7 +108,7 @@ impl Deref for BStr { #[inline] fn deref(&self) -> &Self::Target { - &self.0 + self.deref_const() } } From 2282cbacf40e593c9c9db459f1a411c90a8f4360 Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Sat, 19 Oct 2024 18:35:35 -0700 Subject: [PATCH 40/51] rust: device_id: add macro for alias generation For modpost to work, we need the device id table to be named as follow: __mod_{type}__{name}_device_table In C, this is done by calling MODULE_DEVICE_TABLE(type, name), which declares a new symbol that aliases to `name`. As far as I know it is not possible to create such aliases in Rust, so here we instead create a new static variable which contains a copy of the device_id tables, but without the extra `id_infos` used on the Rust side. Signed-off-by: Fabien Parent --- rust/kernel/device_id.rs | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs index 0c856181b1c529..c84e06f774e778 100644 --- a/rust/kernel/device_id.rs +++ b/rust/kernel/device_id.rs @@ -40,11 +40,25 @@ pub unsafe trait RawDeviceId { const DRIVER_DATA_OFFSET: usize; } -/// A zero-terminated device id array, followed by context data. +/// A zero-terminated device id array. #[repr(C)] -pub struct IdArray { +pub struct RawIdArray { ids: [T::RawType; N], sentinel: MaybeUninit, +} + +impl RawIdArray { + #[doc(hidden)] + pub const fn len(&self) -> usize { + (N + 1) * core::mem::size_of::() + } +} + +/// A zero-terminated device id array, followed by context data. +#[repr(C)] +pub struct IdArray { + #[doc(hidden)] + pub raw_ids: RawIdArray, id_infos: [U; N], } @@ -83,8 +97,10 @@ impl IdArray { // SAFETY: this is effectively `array_assume_init`, which is unstable, so we use // `transmute_copy` instead. We have initialized all elements of `raw_ids` so this // `array_assume_init` is safe. - ids: unsafe { core::mem::transmute_copy(&raw_ids) }, - sentinel: MaybeUninit::zeroed(), + raw_ids: RawIdArray { + ids: unsafe { core::mem::transmute_copy(&raw_ids) }, + sentinel: MaybeUninit::zeroed(), + }, // SAFETY: We have initialized all elements of `infos` so this `array_assume_init` is // safe. id_infos: unsafe { core::mem::transmute_copy(&infos) }, @@ -117,10 +133,20 @@ impl IdTable for IdArray { } fn id(&self, index: usize) -> &T::RawType { - &self.ids[index] + &self.raw_ids.ids[index] } fn info(&self, index: usize) -> &U { &self.id_infos[index] } } + +/// Create device table alias for modpost. +#[macro_export] +macro_rules! module_device_table { + ($table_type: literal, $module_table_name:ident, $table_name:ident) => { + #[export_name = concat!("__mod_", $table_type, "__", stringify!($table_name), "_device_table")] + static $module_table_name: [u8; $table_name.raw_ids.len()] = + unsafe { core::mem::transmute_copy(&$table_name.raw_ids) }; + }; +} From cf0351f9e641bf2b0233d4aa80469fdd3030dfb7 Mon Sep 17 00:00:00 2001 From: Sven Van Asbroeck Date: Fri, 10 Nov 2023 11:15:15 -0800 Subject: [PATCH 41/51] rust: add basic OF abstraction Co-developed-by: Wedson Almeida Filho Signed-off-by: Wedson Almeida Filho Signed-off-by: Sven Van Asbroeck Signed-off-by: Fabien Parent --- rust/kernel/lib.rs | 1 + rust/kernel/of.rs | 57 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 rust/kernel/of.rs diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 83fe4232b832c9..b43fc66ef37d3a 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -51,6 +51,7 @@ pub mod kunit; pub mod list; #[cfg(CONFIG_NET)] pub mod net; +pub mod of; pub mod page; pub mod prelude; pub mod print; diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs new file mode 100644 index 00000000000000..85b5ccde408160 --- /dev/null +++ b/rust/kernel/of.rs @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Devicetree and Open Firmware abstractions. +//! +//! C header: [`include/linux/of_*.h`](../../../../include/linux/of_*.h) + +use crate::{ + bindings, device_id, + str::{BStr, CStr}, +}; + +/// An open firmware device id. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::of_device_id); + +impl DeviceId { + /// Create a OF `DeviceId` from a compatible string. + pub const fn with_compatible(compatible: &CStr) -> Self { + let device_id = core::mem::MaybeUninit::::zeroed(); + let mut device_id = unsafe { device_id.assume_init() }; + + let compatible = BStr::from_bytes(compatible.as_bytes_with_nul()); + assert!(compatible.len() <= device_id.compatible.len()); + + let mut i = 0; + while i < compatible.len() { + device_id.compatible[i] = compatible.deref_const()[i] as _; + i += 1; + } + + Self(device_id) + } +} + +// SAFETY: `ZERO` is all zeroed-out and `to_rawid` stores `offset` in `of_device_id::data`. +unsafe impl device_id::RawDeviceId for DeviceId { + type RawType = bindings::of_device_id; + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::of_device_id, data); +} + +/// Alias for `device_id::IdTable` containing OF's `DeviceId` +pub type IdTable = &'static dyn device_id::IdTable; + +/// Create an OF `IdTable` with its alias for modpost. +#[macro_export] +macro_rules! of_device_table { + ($module_table_name:ident, $table_name:ident, $id_info_type: ty, $table_data: expr) => { + const $table_name: $crate::device_id::IdArray< + $crate::of::DeviceId, + $id_info_type, + { $table_data.len() }, + > = $crate::device_id::IdArray::new($table_data); + + $crate::module_device_table!("of", $module_table_name, $table_name); + }; +} From faf3c48854ca01505a8fc6289e4c2ad951c0dab2 Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Fri, 20 Sep 2024 16:15:17 -0700 Subject: [PATCH 42/51] rust: enable associated_type_defaults unstable feature Signed-off-by: Fabien Parent --- rust/kernel/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index b43fc66ef37d3a..a078d36f1eff9e 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -13,6 +13,7 @@ #![no_std] #![feature(arbitrary_self_types)] +#![feature(associated_type_defaults)] #![feature(coerce_unsized)] #![feature(const_refs_to_cell)] #![feature(dispatch_from_dyn)] From b087b606a57b3047ef0c8c8d79f3fdffd2fff6e2 Mon Sep 17 00:00:00 2001 From: Fiona Behrens Date: Sat, 17 Dec 2022 11:29:56 +0100 Subject: [PATCH 43/51] rust: add I2C client abstraction Add a new Rust abstraction that allows writing drivers for I2C based devices. Signed-off-by: Fiona Behrens Co-developed-by: Fabien Parent Signed-off-by: Fabien Parent --- rust/bindings/bindings_helper.h | 1 + rust/helpers/helpers.c | 1 + rust/helpers/i2c.c | 13 ++ rust/kernel/i2c.rs | 228 ++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 2 + 5 files changed, 245 insertions(+) create mode 100644 rust/helpers/i2c.c create mode 100644 rust/kernel/i2c.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index a80783fcbe042a..3165e96062a14c 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 3acb2b9e52ec2a..999bae8ab2599b 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -13,6 +13,7 @@ #include "build_bug.c" #include "device.c" #include "err.c" +#include "i2c.c" #include "io.c" #include "kunit.c" #include "mutex.c" diff --git a/rust/helpers/i2c.c b/rust/helpers/i2c.c new file mode 100644 index 00000000000000..8ffdc454e7597c --- /dev/null +++ b/rust/helpers/i2c.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void *rust_helper_i2c_get_clientdata(const struct i2c_client *client) +{ + return i2c_get_clientdata(client); +} + +void rust_helper_i2c_set_clientdata(struct i2c_client *client, void *data) +{ + i2c_set_clientdata(client, data); +} diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs new file mode 100644 index 00000000000000..4e3a6281b7774c --- /dev/null +++ b/rust/kernel/i2c.rs @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! I2C devices and drivers. +//! +//! C header: [`include/linux/i2c.h`](../../../../include/linux/i2c.h) + +use crate::{ + bindings, + device::Device, + device_id::{self, RawDeviceId}, + driver, + error::{from_result, to_result, Result}, + of, + str::{BStr, CStr}, + types::ForeignOwnable, + ThisModule, +}; + +/// An I2C device id. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::i2c_device_id); + +impl DeviceId { + /// Create a new I2C DeviceId + pub const fn new(name: &CStr) -> Self { + let device_id = core::mem::MaybeUninit::::zeroed(); + let mut device_id = unsafe { device_id.assume_init() }; + + let name = BStr::from_bytes(name.as_bytes_with_nul()); + assert!(name.len() <= device_id.name.len()); + + let mut i = 0; + while i < name.len() { + device_id.name[i] = name.deref_const()[i] as _; + i += 1; + } + + Self(device_id) + } +} + +// SAFETY: `ZERO` is all zeroed-out and `to_rawid` stores `offset` in `i2c_device_id::driver_data`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::i2c_device_id; + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::i2c_device_id, driver_data); +} + +/// Alias for `device_id::IdTable` containing I2C's `DeviceId` +pub type IdTable = &'static dyn device_id::IdTable; + +/// An adapter for the registration of i2c drivers. +pub struct Adapter(T); + +impl driver::RegistrationOps for Adapter { + type RegType = bindings::i2c_driver; + + fn register( + i2cdrv: &mut Self::RegType, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + i2cdrv.driver.name = name.as_char_ptr(); + i2cdrv.probe = Some(Self::probe_callback); + i2cdrv.remove = Some(Self::remove_callback); + if let Some(t) = T::I2C_DEVICE_ID_TABLE { + i2cdrv.id_table = t.as_ptr(); + } + if let Some(t) = T::OF_DEVICE_ID_TABLE { + i2cdrv.driver.of_match_table = t.as_ptr(); + } + + // SAFETY: + // - `pdrv` lives at least until the call to `platform_driver_unregister()` returns. + // - `name` pointer has static lifetime. + // - `module.0` lives at least as long as the module. + // - `probe()` and `remove()` are static functions. + // - `of_match_table` is either a raw pointer with static lifetime, + // as guaranteed by the [`device_id::IdTable`] type, or null. + to_result(unsafe { bindings::i2c_register_driver(module.0, i2cdrv) }) + } + + fn unregister(i2cdrv: &mut Self::RegType) { + // SAFETY: By the safety requirements of this function (defined in the trait definition), + // `reg` was passed (and updated) by a previous successful call to + // `i2c_register_driver`. + unsafe { bindings::i2c_del_driver(i2cdrv) }; + } +} + +impl Adapter { + extern "C" fn probe_callback(i2c: *mut bindings::i2c_client) -> core::ffi::c_int { + from_result(|| { + let mut client = unsafe { Client::from_ptr(i2c) }; + let data = T::probe(&mut client)?; + + // SAFETY: `i2c` is guaranteed to be a valid, non-null pointer. + unsafe { bindings::i2c_set_clientdata(i2c, data.into_foreign() as _) }; + Ok(0) + }) + } + + extern "C" fn remove_callback(i2c: *mut bindings::i2c_client) { + // SAFETY: `i2c` is guaranteed to be a valid, non-null pointer + let ptr = unsafe { bindings::i2c_get_clientdata(i2c) }; + // SAFETY: + // - we allocated this pointer using `T::Data::into_pointer`, + // so it is safe to turn back into a `T::Data`. + // - the allocation happened in `probe`, no-one freed the memory, + // `remove` is the canonical kernel location to free driver data. so OK + // to convert the pointer back to a Rust structure here. + let data = unsafe { T::Data::from_foreign(ptr) }; + T::remove(&data); + } +} + +/// A I2C driver. +pub trait Driver { + /// Data stored on device by driver. + /// + /// Corresponds to the data set or retrieved via the kernel's + /// `i2c_{set,get}_clientdata()` functions. + /// + /// Require that `Data` implements `ForeignOwnable`. We guarantee to + /// never move the underlying wrapped data structure. This allows + type Data: ForeignOwnable = (); + + /// The type holding information about each device id supported by the driver. + type IdInfo: 'static = (); + + /// The table of i2c device ids supported by the driver. + const I2C_DEVICE_ID_TABLE: Option> = None; + + /// The table of OF device ids supported by the driver. + const OF_DEVICE_ID_TABLE: Option> = None; + + /// I2C driver probe. + /// + /// Called when a new i2c client is added or discovered. + /// Implementers should attempt to initialize the client here. + fn probe(client: &mut Client) -> Result; + + /// I2C driver remove. + /// + /// Called when an i2c client is removed. + fn remove(_data: &Self::Data) {} +} + +/// A I2C Client device. +/// +/// # Invariants +/// +/// The field `ptr` is non-null and valid for the lifetime of the object. +pub struct Client { + ptr: *mut bindings::i2c_client, +} + +impl Client { + /// Creates a new client from the given pointer. + /// + /// # Safety + /// + /// `ptr` must be non-null and valid. It must remain valid for the lifetime of the returned + /// instance. + unsafe fn from_ptr(ptr: *mut bindings::i2c_client) -> Self { + // INVARIANT: The safety requirements of the function ensure the lifetime invariant. + Self { ptr } + } + + /// Returns the raw I2C client structure. + pub fn raw_client(&self) -> *mut bindings::i2c_client { + self.ptr + } +} + +impl AsRef for Client { + fn as_ref(&self) -> &Device { + // SAFETY: By the type invariants, we know that `self.ptr` is non-null and valid. + unsafe { Device::as_ref(&mut (*self.ptr).dev) } + } +} + +/// Declares a kernel module that exposes a single i2c driver. +/// +/// # Examples +/// +/// ```ignore +/// # use kernel::{i2c, define_i2c_id_table, module_i2c_driver}; +/// kernel::module_i2c_id_table!(MOD_TABLE, I2C_CLIENT_I2C_ID_TABLE); +/// kernel::define_i2c_id_table! {I2C_CLIENT_I2C_ID_TABLE, (), [ +/// (i2c::DeviceId(b"fpga"), None), +/// ]} +/// struct MyDriver; +/// impl i2c::Driver for MyDriver { +/// kernel::driver_i2c_id_table!(I2C_CLIENT_I2C_ID_TABLE); +/// // [...] +/// # fn probe(_client: &mut i2c::Client) -> Result { +/// # Ok(()) +/// # } +/// } +/// +/// module_i2c_driver! { +/// type: MyDriver, +/// name: "module_name", +/// author: "Author name", +/// license: "GPL", +/// } +/// ``` +#[macro_export] +macro_rules! module_i2c_driver { + ($($f:tt)*) => { + $crate::module_driver!(, $crate::i2c::Adapter, { $($f)* }); + }; +} + +/// Create a I2C `IdTable` with its alias for modpost. +#[macro_export] +macro_rules! i2c_device_table { + ($module_table_name:ident, $table_name:ident, $id_info_type: ty, $table_data: expr) => { + const $table_name: $crate::device_id::IdArray< + $crate::i2c::DeviceId, + $id_info_type, + { $table_data.len() }, + > = $crate::device_id::IdArray::new($table_data); + + $crate::module_device_table!("i2c", $module_table_name, $table_name); + }; +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index a078d36f1eff9e..3e1f48b4392a51 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -45,6 +45,8 @@ pub mod driver; pub mod error; #[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)] pub mod firmware; +#[cfg(CONFIG_I2C)] +pub mod i2c; pub mod init; pub mod ioctl; #[cfg(CONFIG_KUNIT)] From 1cf952ddd13adcd45e99cea861c8b216ce9e4b4b Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Fri, 15 Dec 2023 21:21:54 -0800 Subject: [PATCH 44/51] rust: add function to create masks Create a function that is equivalent to Linux's GENMASK macro. Signed-off-by: Fabien Parent --- rust/kernel/bits.rs | 12 ++++++++++++ rust/kernel/lib.rs | 1 + 2 files changed, 13 insertions(+) create mode 100644 rust/kernel/bits.rs diff --git a/rust/kernel/bits.rs b/rust/kernel/bits.rs new file mode 100644 index 00000000000000..7c111df61d8617 --- /dev/null +++ b/rust/kernel/bits.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Bit manipulation helpers +//! +//! C header: [`include/linux/bits.h`](srctree/include/linux/bits.h) + +/// Generate a mask where all bits >= `h` and <= `l` are set +/// +/// This is a re-implementation in rust of `GENMASK` +pub const fn genmask(h: u32, l: u32) -> u32 { + ((!0u32) - (1 << l) + 1) & ((!0u32) >> (32 - 1 - h)) +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 3e1f48b4392a51..03db904eee9cf9 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -35,6 +35,7 @@ compile_error!("Missing kernel configuration for conditional compilation"); extern crate self as kernel; pub mod alloc; +pub mod bits; #[cfg(CONFIG_BLOCK)] pub mod block; mod build_assert; From 84a8d2ae0a85c822c6501e79bbe679c84d90d21f Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Mon, 18 Dec 2023 17:47:20 -0800 Subject: [PATCH 45/51] rust: macros: paste: add support for using var as part of group The following code is currently working fine and will generate a struct named "Test0": kernel::macros::paste! { struct []; } But if we need to use a "macro variable" like this: macro_rules! test { ($i: literal) => { kernel::macros::paste! { struct []; } }; } test!(0); The code above will make the paste! macro panic: error: proc macro panicked --> drivers/regulator/ncv6336.rs:25:9 | 25 | / kernel::macros::paste! { 26 | | struct []; 27 | | } | |_________^ ... 31 | test!(0); | -------- in this macro invocation | = help: message: unexpected token in paste segments The reason of this panic is that "macro variables" are creating `Groups` [0], with the `None` delimiter [1], and currently `Groups`are not handled by the paste! macro. This commit adds support for TokenTree::Group token where the delimiter is None in order to make the previous code snippet now work. [0] https://doc.rust-lang.org/proc_macro/struct.Group.html [1] https://doc.rust-lang.org/proc_macro/enum.Delimiter.html#variant.None Signed-off-by: Fabien Parent --- rust/macros/paste.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/rust/macros/paste.rs b/rust/macros/paste.rs index f40d42b35b5869..409f5bb7dfb4be 100644 --- a/rust/macros/paste.rs +++ b/rust/macros/paste.rs @@ -2,7 +2,7 @@ use proc_macro::{Delimiter, Group, Ident, Spacing, Span, TokenTree}; -fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree { +fn process_tokens_into_segments(tokens: &[TokenTree]) -> (Vec<(String, Span)>, Option) { let mut tokens = tokens.iter(); let mut segments = Vec::new(); let mut span = None; @@ -46,10 +46,29 @@ fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree { }; segments.push((value, sp)); } + Some(TokenTree::Group(group)) if group.delimiter() == Delimiter::None => { + let group_tokens: Vec<_> = group.stream().into_iter().collect(); + let (group_segments, group_span) = process_tokens_into_segments(&group_tokens); + segments.extend(group_segments); + + assert!( + !(span.is_some() && group_span.is_some()), + "span modifier should only appear at most once" + ); + + if group_span.is_some() { + span = group_span; + } + } _ => panic!("unexpected token in paste segments"), }; } + (segments, span) +} + +fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree { + let (segments, span) = process_tokens_into_segments(tokens); let pasted: String = segments.into_iter().map(|x| x.0).collect(); TokenTree::Ident(Ident::new(&pasted, span.unwrap_or(group_span))) } From 563d61a7ff70c3a229f129a331b2c6f2376b3164 Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Tue, 19 Dec 2023 17:08:24 -0800 Subject: [PATCH 46/51] rust: macros: add foreach! macro to repeat code Add a macro that can be used to repeat code. It allows to use the following syntax: foreach!(i in 0..=4) { paste! { struct []; } } This will be used by the Regmap abstraction in order to prevent redefining the same bit several times, hence prevent fields to overlap. Signed-off-by: Fabien Parent --- rust/macros/foreach.rs | 137 +++++++++++++++++++++++++++++++++++++++++ rust/macros/lib.rs | 25 ++++++++ 2 files changed, 162 insertions(+) create mode 100644 rust/macros/foreach.rs diff --git a/rust/macros/foreach.rs b/rust/macros/foreach.rs new file mode 100644 index 00000000000000..a92280f229d1de --- /dev/null +++ b/rust/macros/foreach.rs @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0 + +use proc_macro::{ + Delimiter, TokenStream, + TokenTree::{self, Group, Ident, Literal}, +}; +use std::ops::Range; + +fn process_group(var: &str, i: isize, tokens: impl Iterator) -> Vec { + let mut tt = Vec::::new(); + let mut tokens = tokens.peekable(); + + while let Some(token) = tokens.next() { + match token { + Group(ref group) => { + let group_tokens = process_group(var, i, group.stream().into_iter()); + let stream = FromIterator::from_iter(group_tokens.into_iter()); + let new_group = proc_macro::Group::new(group.delimiter(), stream); + tt.push(TokenTree::Group(new_group)); + } + TokenTree::Punct(ref punct) => { + if punct.to_string() == "$" { + if let Some(TokenTree::Ident(ident)) = tokens.peek() { + if ident.to_string() == var { + tt.push(TokenTree::Literal(proc_macro::Literal::isize_unsuffixed(i))); + tokens.next(); + continue; + } + } + } + + tt.push(token); + } + _ => tt.push(token), + } + } + + tt +} + +pub(crate) fn expand(input: TokenStream) -> TokenStream { + let mut tokens = input.into_iter().peekable(); + + let var = if let Some(Ident(i)) = tokens.next() { + i.to_string() + } else { + panic!("foreach! first token should be an identifier"); + }; + + let token = tokens.next().expect("missing token, expecting ident 'in'"); + assert!(matches!(token, TokenTree::Ident(x) if x.to_string() == "in")); + + let token = tokens + .next() + .expect("foreach!: missing token, expecting range"); + let token = if let Group(group) = token { + group + .stream() + .into_iter() + .next() + .expect("foreach: missing token, expecting integer") + } else { + token + }; + + let start = if let Literal(lit) = token { + lit.to_string() + .parse::() + .expect("Failed to convert literal to isize") + } else { + panic!("foreach!: unexpected token '{token}'"); + }; + + let token = tokens + .next() + .expect("foreach!: missing token, expecting '.'"); + assert!(matches!(token, TokenTree::Punct(x) if x == '.')); + let token = tokens + .next() + .expect("foreach!: missing token, expecting '.'"); + assert!(matches!(token, TokenTree::Punct(x) if x == '.')); + + let is_inclusive_range = if let Some(TokenTree::Punct(p)) = tokens.peek() { + if p.as_char() == '=' { + tokens.next(); + true + } else { + false + } + } else { + false + }; + + let token = tokens + .next() + .expect("foreach!: missing token, expecting integer"); + let token = if let Group(group) = token { + group + .stream() + .into_iter() + .next() + .expect("foreach: missing token, expecting integer") + } else { + token + }; + + let end = if let Literal(lit) = token { + lit.to_string() + .parse::() + .expect("Failed to convert literal to isize") + } else { + panic!("foreach!: unexpected token '{token}'"); + }; + let range = Range { + start, + end: end + if is_inclusive_range { 1 } else { 0 }, + }; + + let tokens = if let Some(Group(group)) = tokens.next() { + if group.delimiter() != Delimiter::Brace { + panic!("foreach! expected brace"); + } + + group.stream().into_iter() + } else { + panic!("foreach! missing opening brace"); + }; + + let tokens: Vec = tokens.collect(); + let mut tt = Vec::::new(); + + for i in range { + tt.extend_from_slice(&process_group(&var, i, tokens.clone().into_iter())); + } + + FromIterator::from_iter(tt) +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 8d4ac914b48bc8..0b076f9173ef9d 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -9,6 +9,7 @@ #[macro_use] mod quote; mod concat_idents; +mod foreach; mod helpers; mod module; mod paste; @@ -412,6 +413,30 @@ pub fn paste(input: TokenStream) -> TokenStream { tokens.into_iter().collect() } +/// Repeat a fragment of code and provide a numerical index for the current repetition +/// +/// # Examples +/// +/// ```rust,ignore +/// foreach!(i in 0..10) { +/// paste! { +/// fn []() { +/// } +/// } +/// } +/// +/// foreach!(i in 8..=15) { +/// paste! { +/// struct []() { +/// } +/// } +/// } +/// ``` +#[proc_macro] +pub fn foreach(input: TokenStream) -> TokenStream { + foreach::expand(input) +} + /// Derives the [`Zeroable`] trait for the given struct. /// /// This can only be used for structs where every field implements the [`Zeroable`] trait. From f5a2ea21cdabdd113dbc847911fb7712add68473 Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Tue, 28 Nov 2023 19:22:46 -0800 Subject: [PATCH 47/51] rust: Add new abstraction for regmap Add an abstraction to regmap for Rust. Signed-off-by: Fabien Parent --- rust/bindings/bindings_helper.h | 1 + rust/helpers/helpers.c | 1 + rust/helpers/regmap.c | 48 ++ rust/kernel/lib.rs | 2 + rust/kernel/regmap.rs | 874 ++++++++++++++++++++++++++++++++ 5 files changed, 926 insertions(+) create mode 100644 rust/helpers/regmap.c create mode 100644 rust/kernel/regmap.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 3165e96062a14c..d65ce37f033a71 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 999bae8ab2599b..78f6513e66f0d2 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -21,6 +21,7 @@ #include "rbtree.c" #include "rcu.c" #include "refcount.c" +#include "regmap.c" #include "signal.c" #include "slab.c" #include "spinlock.c" diff --git a/rust/helpers/regmap.c b/rust/helpers/regmap.c new file mode 100644 index 00000000000000..2426e563df339a --- /dev/null +++ b/rust/helpers/regmap.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#if IS_BUILTIN(CONFIG_REGMAP_I2C) +struct regmap *rust_helper_regmap_init_i2c(struct i2c_client *i2c, + const struct regmap_config *config) +{ + return regmap_init_i2c(i2c, config); +} +#endif + +int rust_helper_regmap_field_write(struct regmap_field *field, unsigned int val) +{ + return regmap_field_write(field, val); +} + +int rust_helper_regmap_field_force_write(struct regmap_field *field, + unsigned int val) +{ + return regmap_field_force_write(field, val); +} + +int rust_helper_regmap_field_update_bits(struct regmap_field *field, + unsigned int mask, unsigned int val) +{ + return regmap_field_update_bits(field, mask, val); +} + +int rust_helper_regmap_field_set_bits(struct regmap_field *field, + unsigned int bits) +{ + return regmap_field_set_bits(field, bits); +} + +int rust_helper_regmap_field_clear_bits(struct regmap_field *field, + unsigned int bits) +{ + return regmap_field_clear_bits(field, bits); +} + +int rust_helper_regmap_field_force_update_bits(struct regmap_field *field, + unsigned int mask, + unsigned int val) +{ + return regmap_field_force_update_bits(field, mask, val); +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 03db904eee9cf9..cb0aa7fd2fcb0b 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -60,6 +60,8 @@ pub mod page; pub mod prelude; pub mod print; pub mod rbtree; +#[cfg(CONFIG_REGMAP)] +pub mod regmap; pub mod revocable; pub mod sizes; mod static_assert; diff --git a/rust/kernel/regmap.rs b/rust/kernel/regmap.rs new file mode 100644 index 00000000000000..2074e6c789a6c8 --- /dev/null +++ b/rust/kernel/regmap.rs @@ -0,0 +1,874 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Register map access API. +//! +//! C header: [`include/linux/regmap.h`](srctree/include/linux/regmap.h) + +#[cfg(CONFIG_REGMAP_I2C = "y")] +use crate::i2c; +use crate::{ + bindings, + error::{code::*, to_result, Error, Result}, + macros::paste, + sync::Arc, +}; +use core::{marker::PhantomData, mem::MaybeUninit}; + +/// Type of caching +#[repr(u32)] +pub enum CacheType { + /// Don't cache anything + None = bindings::regcache_type_REGCACHE_NONE, + /// Use RbTree caching + RbTree = bindings::regcache_type_REGCACHE_RBTREE, + /// Use Flat caching + Flat = bindings::regcache_type_REGCACHE_FLAT, + /// Use Maple caching + Maple = bindings::regcache_type_REGCACHE_MAPLE, +} + +/// Register map +/// +/// # Examples +/// +/// ``` +/// let regmap = Regmap::init_i2c(i2c, &config); +/// ``` +pub struct Regmap(*mut bindings::regmap); + +impl Regmap { + #[cfg(CONFIG_REGMAP_I2C = "y")] + /// Initialize a [`Regmap`] instance for an `i2c` device. + pub fn init_i2c(i2c: &i2c::Client, config: &Config) -> Self { + let regmap = unsafe { bindings::regmap_init_i2c(i2c.raw_client(), &config.raw) }; + + Self(regmap) + } + + /// Allocate regmap [`Fields`] + /// + /// This function allocate regmap fields from the `reg_fields` descriptors + pub fn alloc_fields( + self: &Arc, + descs: &'static FieldDescs, + ) -> Result> { + let mut rm_fields = [core::ptr::null_mut(); N]; + to_result(unsafe { + bindings::regmap_field_bulk_alloc( + self.0, + &mut rm_fields[0], + descs.0.as_ptr(), + descs.0.len() as i32, + ) + })?; + + Ok(Fields { + rm_fields, + _regmap: self.clone(), + }) + } + + /// Return the raw pointer of this regmap + pub fn as_ptr(&self) -> *mut bindings::regmap { + self.0 + } +} + +impl Drop for Regmap { + fn drop(&mut self) { + unsafe { bindings::regmap_exit(self.0) } + } +} + +/// Field Descriptors +/// +/// FieldDescriptors can be created by calling the [`define_regmap_field_descs`] macro. +pub struct FieldDescs([bindings::reg_field; N]); + +impl FieldDescs { + // macro use only + #[doc(hidden)] + pub const fn new(fields: [bindings::reg_field; N]) -> Self { + Self(fields) + } + + /// Number of fields being held by `FieldDescs` + /// + /// This function should be used to retrieve the number of fields that were + /// created when calling [`define_regmap_field_descs`]. + /// + /// # Examples + /// + /// ``` + /// use kernel::regmap::{define_regmap_field_descs, Fields}; + /// + /// define_regmap_field_descs!(DESCS, { + /// {pid, 0x3, kernel::regmap::access::READ, { value => raw([7:0], ro) }}, + /// }); + /// + /// struct Registrations { + /// fields: Fields<{ DESCS.count() }>, + /// } + /// ``` + pub const fn count(&self) -> usize { + N + } +} + +/// Regmap fields +/// +/// # Invariants +/// +/// `rm_fields` is garanteed to contains valid and initialized `regmap_field`s. +/// +pub struct Fields { + rm_fields: [*mut bindings::regmap_field; N], + + // Each regmap_field hold a pointer to the `struct regmap` instance, so we need to keep a copy + // of the wrapper around. + _regmap: Arc, +} +impl Fields { + /// Get field `index` + pub fn index(&mut self, index: usize) -> *mut bindings::regmap_field { + self.rm_fields[index] + } + + // macro use only + #[doc(hidden)] + pub fn read(&mut self, index: usize) -> Result { + let mut val = 0; + + // Make sure we don't panic if the index is out of bound. + if index >= N { + return Err(EINVAL); + } + + // SAFETY: By the type invariants, we are garanteed that all rm_fields entries point + // to valid and initialized values, hence it is safe to make this FFI call. + let ret = unsafe { bindings::regmap_field_read(self.rm_fields[index], &mut val) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(val) + } +} + +unsafe impl Send for Fields {} + +/// Helper macro for [`Config`] to create methods to set a fields from [`regmap_config`] +/// +/// The following code will create a method named `with_max_register`: +/// ``` +/// config_with!(max_register: u32); +/// ``` +macro_rules! config_with { + ($(#[$meta:meta])* $name:ident: $type:ty) => { + config_with!($(#[$meta])* $name: $type, $name); + }; + + ($(#[$meta:meta])* $name:ident: $type:ty, $e:expr) => { + paste! { + $(#[$meta])* + pub const fn [](mut self, $name: $type) -> Self { + self.raw.$name = $e; + self + } + } + }; +} + +// macro use only +#[doc(hidden)] +pub trait ConfigOps { + fn is_readable_reg(reg: u32) -> bool; + fn is_writeable_reg(reg: u32) -> bool; + fn is_volatile_reg(reg: u32) -> bool; + fn is_precious_reg(reg: u32) -> bool; +} + +/// Regmap Configuration +pub struct Config { + raw: bindings::regmap_config, + _phantom: PhantomData, +} +impl Config { + /// Create a new regmap Config + pub const fn new(reg_bits: i32, val_bits: i32) -> Self { + let cfg = MaybeUninit::::zeroed(); + let mut cfg = unsafe { cfg.assume_init() }; + + cfg.reg_bits = reg_bits; + cfg.val_bits = val_bits; + cfg.writeable_reg = Some(Self::writeable_reg_callback); + cfg.readable_reg = Some(Self::readable_reg_callback); + cfg.volatile_reg = Some(Self::volatile_reg_callback); + cfg.precious_reg = Some(Self::precious_reg_callback); + + Self { + raw: cfg, + _phantom: PhantomData, + } + } + + config_with!( + /// Specifies the maximum valid register address. + max_register: u32 + ); + + config_with!( + /// Type of caching being performed. + cache_type: CacheType, cache_type as _ + ); + + unsafe extern "C" fn writeable_reg_callback(_dev: *mut bindings::device, reg: u32) -> bool { + T::is_writeable_reg(reg) + } + + unsafe extern "C" fn readable_reg_callback(_dev: *mut bindings::device, reg: u32) -> bool { + T::is_readable_reg(reg) + } + + unsafe extern "C" fn volatile_reg_callback(_dev: *mut bindings::device, reg: u32) -> bool { + T::is_volatile_reg(reg) + } + + unsafe extern "C" fn precious_reg_callback(_dev: *mut bindings::device, reg: u32) -> bool { + T::is_precious_reg(reg) + } +} + +/// Definitions describing how registers can be accessed. +pub mod access { + /// Register can be read from. + pub const READ: u32 = 0b000001; + /// Register can be written to. + pub const WRITE: u32 = 0b000010; + /// Register should not be read outside of a call from the driver. + pub const PRECIOUS: u32 = 0b000100; + /// Register value can't be cached. + pub const VOLATILE: u32 = 0b001000; + + /// Register can be read from and written to. + pub const RW: u32 = READ | WRITE; +} + +// macro use only +#[doc(hidden)] +#[macro_export] +macro_rules! regmap_check_access { + ($type:ident, $access:expr, $reg:ident, $addr:literal) => { + if kernel::regmap::access::$type & $access > 0 && $reg == $addr { + return true; + } + }; +} +// macro use only +#[doc(hidden)] +pub use regmap_check_access; + +/// Common operations for all field types +pub trait FieldCommonOps { + /// Get the Mask for the field + fn mask() -> u32; +} + +/// Read operations for fields with `bit` type +pub trait BitFieldReadOps { + /// Returns whether the bit is set + fn is_set(fields: &mut Fields) -> Result; +} + +/// Write operations for fields with `bit` type +pub trait BitFieldWriteOps { + /// Set the bit + fn set(fields: &mut Fields) -> Result; + + /// Force set the bit + fn force_set(fields: &mut Fields) -> Result; + + /// Clear the bit + fn clear(fields: &mut Fields) -> Result; + + /// Force clear the bit + fn force_clear(fields: &mut Fields) -> Result; +} + +/// Read operations for fields with `enum` type +pub trait EnumFieldReadOps { + #[doc(hidden)] + /// Underlying enum type reprensenting the field values + type EnumType; + + /// Read the field + fn read(fields: &mut Fields) -> Result; +} + +/// Write operations for fields with `enum` type +pub trait EnumFieldWriteOps { + #[doc(hidden)] + /// Underlying enum type reprensenting the field values + type EnumType; + + /// Write the field + fn write(fields: &mut Fields, val: Self::EnumType) -> Result; + + /// Force write the field + fn force_write(fields: &mut Fields, val: Self::EnumType) -> Result; +} + +/// Read operations for fields with `raw` type +pub trait RawFieldReadOps { + /// Read the field + fn read(fields: &mut Fields) -> Result; + + /// Test the field bits + fn test_bits(fields: &mut Fields, bits: core::ffi::c_uint) -> Result; +} + +/// Write operations for fields with `raw` type +pub trait RawFieldWriteOps { + /// Write the field + fn write(fields: &mut Fields, val: core::ffi::c_uint) -> Result; + + /// Force write the field + fn force_write(fields: &mut Fields, val: core::ffi::c_uint) -> Result; + + /// Update the field using a mask + fn update_bits( + fields: &mut Fields, + mask: core::ffi::c_uint, + val: core::ffi::c_uint, + ) -> Result; + + /// Force update the field using a mask + fn force_update_bits( + fields: &mut Fields, + mask: core::ffi::c_uint, + val: core::ffi::c_uint, + ) -> Result; + + /// Set field bits + fn set_bits(fields: &mut Fields, bits: core::ffi::c_uint) -> Result; + + /// Clear the field bits + fn clear_bits(fields: &mut Fields, bits: core::ffi::c_uint) -> Result; +} + +// macro use only +#[doc(hidden)] +#[macro_export] +macro_rules! regmap_field_bit { + ($field_name:ident, $access: expr, $reg:literal, $pos:literal, rw) => { + kernel::static_assert!($access & kernel::regmap::access::RW == kernel::regmap::access::RW); + + $crate::regmap_field_bit!($field_name, $reg, $pos, reserved); + $crate::regmap_field_bit!($field_name, _ro); + $crate::regmap_field_bit!($field_name, _wo); + }; + + ($field_name:ident, $access: expr, $reg:literal, $pos:literal, ro) => { + kernel::static_assert!( + $access & kernel::regmap::access::READ == kernel::regmap::access::READ + ); + + $crate::regmap_field_bit!($field_name, $reg, $pos, reserved); + $crate::regmap_field_bit!($field_name, _ro); + }; + + ($field_name:ident, $access: expr, $reg:literal, $pos:literal, wo) => { + kernel::static_assert!( + $access & kernel::regmap::access::WRITE == kernel::regmap::access::WRITE + ); + + $crate::regmap_field_bit!($field_name, $reg, $pos, reserved); + $crate::regmap_field_bit!($field_name, _wo); + }; + + ($field_name:ident, $reg:literal, $pos:literal, reserved) => { + kernel::macros::paste! { + struct [<_Bit $pos >]; + } + + impl $field_name { + pub(crate) const fn reg_field() -> bindings::reg_field { + bindings::reg_field { + reg: $reg, + lsb: $pos, + msb: $pos + 1, + id_offset: 0, + id_size: 0, + } + } + + #[allow(dead_code)] + pub(crate) const fn mask() -> u32 { + kernel::bits::genmask($pos, $pos) + } + } + }; + + ($field_name:ident, _ro) => { + impl super::BitFieldReadOps for $field_name { + fn is_set(fields: &mut regmap::Fields) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + let mut val: core::ffi::c_uint = 0; + kernel::error::to_result(unsafe { bindings::regmap_field_read(field, &mut val) })?; + Ok(val == 1) + } + } + }; + + ($field_name:ident, _wo) => { + impl super::BitFieldWriteOps for $field_name { + fn set(fields: &mut regmap::Fields) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_write(field, 1) }) + } + + fn force_set(fields: &mut regmap::Fields) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_force_write(field, 1) }) + } + + fn clear(fields: &mut regmap::Fields) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_write(field, 0) }) + } + + fn force_clear(fields: &mut regmap::Fields) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_force_write(field, 0) }) + } + } + }; +} + +// macro use only +#[doc(hidden)] +#[macro_export] +macro_rules! regmap_field_enum { + ($field_name:ident, $access: expr, $reg:literal, [$msb:literal:$lsb:literal], ro, { + $($k:ident = $v:literal,)+ }) => { + kernel::static_assert!( + $access & kernel::regmap::access::READ == kernel::regmap::access::READ + ); + + $crate::regmap_field_enum!($field_name, $reg, [$msb:$lsb], reserved, { $($k = $v,)+ }); + $crate::regmap_field_enum!($field_name, _ro); + }; + + ($field_name:ident, $access: expr, $reg:literal, [$msb:literal:$lsb:literal], rw, { + $($k:ident = $v:literal,)+ }) => { + kernel::static_assert!($access & kernel::regmap::access::RW == kernel::regmap::access::RW); + + $crate::regmap_field_enum!($field_name, $reg, [$msb:$lsb], reserved, { $($k = $v,)+ }); + $crate::regmap_field_enum!($field_name, _ro); + $crate::regmap_field_enum!($field_name, _wo); + }; + + ($field_name:ident, $access: expr, $reg:literal, [$msb:literal:$lsb:literal], wo, { + $($k:ident = $v:literal,)+ }) => { + kernel::static_assert!( + $access & kernel::regmap::access::WRITE == kernel::regmap::access::WRITE + ); + + $crate::regmap_field_enum!($field_name, $reg, [$msb:$lsb], reserved, { $($k = $v,)+ }); + $crate::regmap_field_enum!($field_name, _wo); + }; + + ($field_name:ident, $reg:literal, [$msb:literal:$lsb:literal], reserved, { + $($k:ident = $v:literal,)+ }) => { + kernel::macros::foreach!(i in $lsb..=$msb { + kernel::macros::paste! { + struct [<_Bit $i>]; + } + }); + + kernel::macros::paste! { + #[repr(u32)] + #[allow(non_camel_case_types)] + pub(crate) enum [<$field_name _enum>] { + $($k = $v,)+ + } + + impl TryFrom for [<$field_name _enum>] { + type Error = kernel::error::Error; + + fn try_from(raw_value: core::ffi::c_uint) -> Result { + match raw_value { + $($v => Ok(Self::$k),)+ + _ => Err(kernel::error::code::EINVAL), + } + } + } + + impl $field_name { + pub(crate) const fn reg_field() -> bindings::reg_field { + bindings::reg_field { + reg: $reg, + lsb: $lsb, + msb: $msb, + id_offset: 0, + id_size: 0, + } + } + + #[allow(dead_code)] + pub(crate) const fn mask() -> u32 { + kernel::bits::genmask($msb, $lsb) + } + } + } + }; + + ($field_name:ident, _ro) => { + impl super::EnumFieldReadOps for $field_name { + type EnumType = kernel::macros::paste! {[<$field_name _enum>]}; + + fn read(fields: &mut regmap::Fields) -> Result { + Self::EnumType::try_from(fields.read(Self::id() as usize)?) + } + } + }; + + ($field_name:ident, _wo) => { + impl super::EnumFieldWriteOps for $field_name { + type EnumType = kernel::macros::paste! {[<$field_name _enum>]}; + + fn write( + fields: &mut regmap::Fields, + val: Self::EnumType + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + let ret = unsafe { bindings::regmap_field_write(field, val as _) }; + kernel::error::to_result(ret) + } + + fn force_write( + fields: &mut regmap::Fields, + val: Self::EnumType + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + let ret = unsafe { bindings::regmap_field_force_write(field, val as _) }; + kernel::error::to_result(ret) + } + } + }; +} + +// macro use only +#[doc(hidden)] +#[macro_export] +macro_rules! regmap_field_raw { + ($field_name:ident, $access: expr, $reg:literal, [$msb:literal:$lsb:literal], rw) => { + kernel::static_assert!($access & kernel::regmap::access::RW == kernel::regmap::access::RW); + + $crate::regmap_field_raw!($field_name, $reg, [$msb:$lsb], reserved); + $crate::regmap_field_raw!($field_name, $reg, [$msb:$lsb], _ro); + $crate::regmap_field_raw!($field_name, $reg, [$msb:$lsb], _wo); + }; + + ($field_name:ident, $access: expr, $reg:literal, [$msb:literal:$lsb:literal], ro) => { + kernel::static_assert!( + $access & kernel::regmap::access::READ == kernel::regmap::access::READ + ); + + $crate::regmap_field_raw!($field_name, $reg, [$msb:$lsb], reserved); + $crate::regmap_field_raw!($field_name, $reg, [$msb:$lsb], _ro); + }; + + ($field_name:ident, $access: expr, $reg:literal, [$msb:literal:$lsb:literal], wo) => { + kernel::static_assert!( + $access & kernel::regmap::access::WRITE == kernel::regmap::access::WRITE + ); + + $crate::regmap_field_raw!($field_name, $reg, [$msb:$lsb], reserved); + $crate::regmap_field_raw!($field_name, $reg, [$msb:$lsb], _wo); + }; + + ($field_name:ident, $reg:literal, [$msb:literal:$lsb:literal], reserved) => { + kernel::macros::foreach!(i in $lsb..=$msb { + kernel::macros::paste! { + struct [<_Bit $i>]; + } + }); + + impl $field_name { + pub(crate) const fn reg_field() -> bindings::reg_field { + bindings::reg_field { + reg: $reg, + lsb: $lsb, + msb: $msb, + id_offset: 0, + id_size: 0, + } + } + + #[allow(dead_code)] + pub(crate) const fn mask() -> u32 { + kernel::bits::genmask($msb, $lsb) + } + } + }; + + ($field_name:ident, $reg:literal, [$msb:literal:$lsb:literal], _ro) => { + impl super::RawFieldReadOps for $field_name { + fn read(fields: &mut regmap::Fields) -> Result { + fields.read(Self::id() as usize) + } + + fn test_bits( + fields: &mut regmap::Fields, + bits: core::ffi::c_uint, + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_test_bits(field, bits) }) + } + } + }; + + ($field_name:ident, $reg:literal, [$msb:literal:$lsb:literal], _wo) => { + impl super::RawFieldWriteOps for $field_name { + fn write( + fields: &mut regmap::Fields, + val: core::ffi::c_uint, + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_write(field, val as _) }) + } + + fn force_write( + fields: &mut regmap::Fields, + val: core::ffi::c_uint, + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { + bindings::regmap_field_force_write(field, val as _) + }) + } + + fn update_bits( + fields: &mut regmap::Fields, + mask: core::ffi::c_uint, + val: core::ffi::c_uint, + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { + bindings::regmap_field_update_bits(field, mask, val) + }) + } + + fn force_update_bits( + fields: &mut regmap::Fields, + mask: core::ffi::c_uint, + val: core::ffi::c_uint, + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { + bindings::regmap_field_force_update_bits(field, mask, val) + }) + } + + fn set_bits( + fields: &mut regmap::Fields, + bits: core::ffi::c_uint, + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_set_bits(field, bits) }) + } + + fn clear_bits( + fields: &mut regmap::Fields, + bits: core::ffi::c_uint, + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_clear_bits(field, bits) }) + } + } + }; +} + +// macro use only +#[doc(hidden)] +#[macro_export] +macro_rules! regmap_fields { + ($type:ident, $reg:ident, $access:expr, $name:ident, $($t:tt)*) => { + kernel::macros::paste! { + #[allow(non_camel_case_types)] + pub(crate) struct $name; + + impl $name { + #[allow(dead_code)] + pub(crate) const fn id() -> super::Fields { + super::Fields::[<$reg _ $name>] + } + } + + $crate::[]!($name, $access, $($t)*); + } + }; +} + +// macro use only +#[doc(hidden)] +#[macro_export] +macro_rules! regmap_reg_field { + ($reg_name:ident, $field_name:ident) => { + register::$reg_name::$field_name::reg_field() + }; +} + +// macro use only +#[doc(hidden)] +#[macro_export] +macro_rules! regmap_count_fields { + () => { 0usize }; + ($type:ident $($rhs:ident)*) => { 1 + $crate::regmap_count_fields!($($rhs)*) }; +} + +/// Define regmap field descriptors +/// +/// # Syntax +/// +/// ```ignore +/// define_regmap_field_desc!(VAR_NAME, { , [, ...] }); +/// +/// : +/// (name, address, access_permission, { , [, ...] }) +/// +/// : +/// field_name => (...), +/// +/// : +/// * raw(, ) +/// * bit(, ) +/// * enum(, , { +/// EnumKind = , [EnumKind2 = , ...] +/// }) +/// +/// : +/// * ro: read-only +/// * rw: read-write +/// * wo: write-only +/// ``` +/// +/// # Examples +/// +/// ```ignore +/// regmap::define_regmap_field_descs!(FIELD_DESCS, { +/// (pid, 0x3, READ, { value => raw([7:0], ro) }), +/// (limconf, 0x16, RW, { +/// rearm => bit(0, rw), +/// rststatus => bit(1, rw), +/// tpwth => enum([5:4], rw, { +/// Temp83C = 0x0, +/// Temp94C = 0x1, +/// Temp105C = 0x2, +/// Temp116C = 0x3, +/// }), +/// }) +/// }); +/// +/// fn probe(client: &mut i2c::Client) -> Result { +/// # let config = regmap::Config::::new(8, 8) +/// # .with_max_register(0x16) +/// # .with_cache_type(regmap::CacheType::RbTree); +/// # let regmap = Arc::try_new(regmap::Regmap::init_i2c(client, &config))?; +/// # let mut fields = regmap.alloc_fields(&FIELD_DESCS)?; +/// // ... +/// dev_info!(client, "PID: {:#x}", pid::value::read(&mut fields)?); +/// // ... +/// } +/// ``` +#[macro_export] +macro_rules! define_regmap_field_descs { + ($name:ident, { + $(( + $reg_name:ident, $reg_addr:literal, $access:expr, { + $($field_name:ident => $type:ident($($x:tt),*)),* $(,)? + } + )),+ + }) => { + mod register { + use kernel::regmap::{ + access::*, + BitFieldReadOps, BitFieldWriteOps, + ConfigOps, + EnumFieldReadOps, EnumFieldWriteOps, + RawFieldReadOps, RawFieldWriteOps + }; + + kernel::macros::paste! { + $( + pub(crate) mod $reg_name { + use kernel::{bindings, error::{Result}, regmap::{self, access::*}}; + $( + $crate::regmap_fields!($type, $reg_name, $access, $field_name, + $reg_addr, $($x),*); + )* + + #[allow(dead_code)] + pub(crate) const fn addr() -> u32 { + $reg_addr + } + } + )+ + + #[repr(u32)] + #[allow(non_camel_case_types)] + pub(crate) enum Fields { + $($( + [<$reg_name _ $field_name>], + )*)+ + } + + pub(crate) struct AccessOps; + impl ConfigOps for AccessOps { + fn is_readable_reg(reg: u32) -> bool { + $( + kernel::regmap::regmap_check_access!(READ, $access, reg, $reg_addr); + )+ + + false + } + + fn is_writeable_reg(reg: u32) -> bool { + $( + kernel::regmap::regmap_check_access!(WRITE, $access, reg, $reg_addr); + )+ + + false + } + + fn is_volatile_reg(reg: u32) -> bool { + $( + kernel::regmap::regmap_check_access!(VOLATILE, $access, reg, $reg_addr); + )+ + + false + } + + fn is_precious_reg(reg: u32) -> bool { + $( + kernel::regmap::regmap_check_access!(PRECIOUS, $access, reg, $reg_addr); + )+ + + false + } + } + } + } + + const $name: regmap::FieldDescs<{$crate::regmap_count_fields!($($($type)*)+)}> = + regmap::FieldDescs::new([ + $( + $( + $crate::regmap_reg_field!($reg_name, $field_name) + ),* + ),+ + ]); + }; +} +pub use define_regmap_field_descs; From c197ae4ec9f4a49659fd359e30389c79b9d2a20e Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Wed, 7 Feb 2024 21:33:11 -0800 Subject: [PATCH 48/51] rust: error: Add declaration for ENOTRECOVERABLE error Add a missing errno for ENOTRECOVERABLE. Signed-off-by: Fabien Parent --- rust/kernel/error.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 7cd3bbab52f208..ff48a8b3f3a5d5 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -82,6 +82,7 @@ pub mod code { declare_err!(EIOCBQUEUED, "iocb queued, will get completion event."); declare_err!(ERECALLCONFLICT, "Conflict with recalled state."); declare_err!(ENOGRACE, "NFS file lock reclaim refused."); + declare_err!(ENOTRECOVERABLE, "State not recoverable."); } /// Generic integer kernel error. From dfc9c6a463a1c676bbe907f2f1e06d9a3002630e Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Mon, 13 Nov 2023 14:54:07 -0800 Subject: [PATCH 49/51] rust: regulator: add regulator consumer abstractions Add a rust abstraction for the regulator consumer API. Signed-off-by: Fabien Parent --- rust/bindings/bindings_helper.h | 1 + rust/kernel/lib.rs | 2 + rust/kernel/regulator.rs | 44 ++++ rust/kernel/regulator/consumer.rs | 393 ++++++++++++++++++++++++++++++ 4 files changed, 440 insertions(+) create mode 100644 rust/kernel/regulator.rs create mode 100644 rust/kernel/regulator/consumer.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index d65ce37f033a71..fd9430524fe4a8 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index cb0aa7fd2fcb0b..e4ffa8d2074ccc 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -62,6 +62,8 @@ pub mod print; pub mod rbtree; #[cfg(CONFIG_REGMAP)] pub mod regmap; +#[cfg(CONFIG_REGULATOR)] +pub mod regulator; pub mod revocable; pub mod sizes; mod static_assert; diff --git a/rust/kernel/regulator.rs b/rust/kernel/regulator.rs new file mode 100644 index 00000000000000..a00b8577ca8121 --- /dev/null +++ b/rust/kernel/regulator.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! SoC Regulators + +pub mod consumer; + +use crate::{ + bindings, + error::{code::*, Error, Result}, +}; + +/// [`consumer::Regulator`] and [`driver::Regulator`] operating modes +#[derive(Copy, Clone)] +#[repr(u32)] +pub enum Mode { + /// Invalid mode + Invalid = bindings::REGULATOR_MODE_INVALID, + /// Regulator can handle fast changes in it's load + Fast = bindings::REGULATOR_MODE_FAST, + /// Normal regulator power supply mode + Normal = bindings::REGULATOR_MODE_NORMAL, + /// Regulator runs in a more efficient mode for light loads + Idle = bindings::REGULATOR_MODE_IDLE, + /// Regulator runs in the most efficient mode for very light loads + Standby = bindings::REGULATOR_MODE_STANDBY, +} + +impl TryFrom for Mode { + type Error = Error; + + /// Convert a mode represented as an unsigned integer into its Rust enum equivalent + /// + /// If the integer does not match any of the [`Mode`], then [`EINVAL`] is returned + fn try_from(mode: core::ffi::c_uint) -> Result { + match mode { + bindings::REGULATOR_MODE_FAST => Ok(Self::Fast), + bindings::REGULATOR_MODE_NORMAL => Ok(Self::Normal), + bindings::REGULATOR_MODE_IDLE => Ok(Self::Idle), + bindings::REGULATOR_MODE_STANDBY => Ok(Self::Standby), + bindings::REGULATOR_MODE_INVALID => Ok(Self::Invalid), + _ => Err(EINVAL), + } + } +} diff --git a/rust/kernel/regulator/consumer.rs b/rust/kernel/regulator/consumer.rs new file mode 100644 index 00000000000000..ea85aa30a49a2a --- /dev/null +++ b/rust/kernel/regulator/consumer.rs @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! SoC Regulator consumer abstractions. +//! +//! C header: [`include/linux/regulator/consumer.h`](srctree/include/linux/regulator/consumer.h) +//! +//! Reference: + +use crate::{ + bindings, + device::Device, + error::{code::*, from_err_ptr, to_result, Error, Result}, + regulator::Mode, + str::CStr, +}; +use core::{ + cmp::min, + ffi::{c_int, c_uint}, + mem::ManuallyDrop, + time::Duration, +}; + +/// [`Regulator`] in its default state (disabled) +/// +/// # Invariants +/// - [`self.0`] is valid and non-null +pub struct Regulator(*mut bindings::regulator); + +impl Regulator { + /// Lookup and obtain an instance of a regulator + /// + /// If the supply does not exists a dummy one will be + /// created + pub fn get>(dev: T, id: &'static CStr) -> Result { + // SAFETY: `dev.as_raw() is valid and non-null by the type invariant and + // id has a static lifetime so it lives indefinitely + let reg = from_err_ptr(unsafe { + bindings::regulator_get(dev.as_ref().as_raw(), id.as_char_ptr()) + })?; + + // This should not happen: in case of error `regulator_get` returns an + // error encoded into the pointer. And in case the device does not + // exists, a dummy regulator is returned + if reg.is_null() { + return Err(ENODEV); + } + + Ok(Self(reg)) + } + + /// Same as `get`, but if the regulator does not exists + /// an error will be returned instead of a dummy regulator + pub fn get_optional>(dev: T, id: &'static CStr) -> Result { + // SAFETY: `dev.as_raw() is valid and non-null by the type invariant and + // id has a static lifetime so it lives indefinitely + let reg = from_err_ptr(unsafe { + bindings::regulator_get_optional(dev.as_ref().as_raw(), id.as_char_ptr()) + })?; + + // does not exists `regulator_get_optional` returns an + // error encoded into the pointer. + if reg.is_null() { + return Err(ENODEV); + } + + Ok(Self(reg)) + } + + /// Same as `get` but ensure that we have exclusive access to the regulator + pub fn get_exclusive>(dev: T, id: &'static CStr) -> Result { + // SAFETY: `dev.as_raw() is valid and non-null by the type invariant and + // id has a static lifetime so it lives indefinitely + let reg = from_err_ptr(unsafe { + bindings::regulator_get_exclusive(dev.as_ref().as_raw(), id.as_char_ptr()) + })?; + + // This should not happen: in case of error `regulator_get` returns an + // error encoded into the pointer. And in case the device does not + // exists, a dummy regulator is returned + if reg.is_null() { + return Err(ENODEV); + } + + Ok(Self(reg)) + } + + /// Enable the regulator + pub fn enable(self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + to_result(unsafe { bindings::regulator_enable(self.0) })?; + Ok(EnabledRegulator(self)) + } + + /// Force disable the regulator. Even if other consumer + /// have enabled it, the regulator will be forcibly disabled. + pub fn force_disable(&mut self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + to_result(unsafe { bindings::regulator_force_disable(self.0) }) + } + + /// Check if the voltage range can be supported + pub fn is_supported_voltage(&self, min_uv: c_int, max_uv: c_int) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_is_supported_voltage(self.0, min_uv, max_uv) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(ret > 0) + } + + /// Returns the number of selectors supported by the regulator + pub fn count_voltages(&self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_count_voltages(self.0) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(ret as _) + } + + /// Returns the voltage corresponding to the `selector` + pub fn list_voltage(&self, selector: c_uint) -> Result> { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_list_voltage(self.0, selector) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(if ret == 0 { None } else { Some(ret) }) + } + + /// Returns the voltage step size between VSEL values + pub fn get_linear_step(&self) -> Option { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_get_linear_step(self.0) }; + if ret == 0 { + None + } else { + Some(ret) + } + } + + /// Returns the regulator output voltage + pub fn get_voltage(&self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_get_voltage(self.0) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(ret) + } + + /// Set the regulator output voltage + pub fn set_voltage(&mut self, min_uv: c_int, max_uv: c_int) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + to_result(unsafe { bindings::regulator_set_voltage(self.0, min_uv, max_uv) }) + } + + /// Get the raise/fall time required for switching voltage + pub fn set_voltage_time(&mut self, old_uv: c_int, new_uv: c_int) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_set_voltage_time(self.0, old_uv, new_uv) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(ret) + } + + /// Re-apply last regulator output voltage + pub fn sync_voltage(&mut self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + to_result(unsafe { bindings::regulator_sync_voltage(self.0) }) + } + + /// Get regulator output current + pub fn get_current_limit(&self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_get_current_limit(self.0) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(ret) + } + + /// Set regulator output current limit + pub fn set_current_limit(&mut self, min_ua: c_int, max_ua: c_int) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_set_current_limit(self.0, min_ua, max_ua) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(ret) + } + + /// Set regulator load + pub fn set_load(&mut self, load_ua: c_int) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + to_result(unsafe { bindings::regulator_set_load(self.0, load_ua) }) + } + + /// Allow the regulator to go into bypass mode + pub fn allow_bypass(&mut self, allow: bool) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + to_result(unsafe { bindings::regulator_allow_bypass(self.0, allow) }) + } + + /// Set the mode of the regulator + pub fn set_mode(&mut self, mode: Mode) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + to_result(unsafe { bindings::regulator_set_mode(self.0, mode as _) }) + } + + /// Get the current mode of the regulator + pub fn get_mode(&mut self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + Mode::try_from(unsafe { bindings::regulator_get_mode(self.0) }) + } +} + +impl Drop for Regulator { + fn drop(&mut self) { + // SAFETY: The pointer is valid and non-null by the type invariant + unsafe { bindings::regulator_put(self.0) } + } +} + +// SAFETY: `Regulator` is not restricted to a single thread so it is safe +// to move it between threads +unsafe impl Send for Regulator {} + +/// [`Regulator`] that has been enabled +pub struct EnabledRegulator(Regulator); + +impl EnabledRegulator { + /// Disable the regulator + pub fn disable(self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_disable(self.0 .0) }; + if ret < 0 { + let mut reg = ManuallyDrop::new(self); + Ok(core::mem::replace( + &mut reg.0, + Regulator(core::ptr::null_mut()), + )) + } else { + Err(Error::from_errno(ret)) + } + } + + /// Disable the regulator with a specified delay + /// + /// Every non-zero delay < 1ms will be rounded up to 1ms, and any delay + /// longer than [`core::ffi::c_int`] will become [`core::ffi::c_int::MAX`] + pub fn disable_deferred(self, duration: Duration) -> Result { + let ms = min(duration.as_millis(), c_int::MAX as u128) as c_int; + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_disable_deferred(self.0 .0, ms) }; + if ret < 0 { + let mut reg = core::mem::ManuallyDrop::new(self); + Ok(core::mem::replace( + &mut reg.0, + Regulator(core::ptr::null_mut()), + )) + } else { + Err(Error::from_errno(ret)) + } + } + + /* Shared functions */ + + /// See [`Regulator::force_disable`] + pub fn force_disable(self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_force_disable(self.0 .0) }; + if ret < 0 { + let mut reg = core::mem::ManuallyDrop::new(self); + Ok(core::mem::replace( + &mut reg.0, + Regulator(core::ptr::null_mut()), + )) + } else { + Err(Error::from_errno(ret)) + } + } + + /// See [`Regulator::is_supported_voltage`] + pub fn is_supported_voltage(&self, min_uv: c_int, max_uv: c_int) -> Result { + self.0.is_supported_voltage(min_uv, max_uv) + } + + /// See [`Regulator::count_voltages`] + pub fn count_voltages(&self) -> Result { + self.0.count_voltages() + } + + /// See [`Regulator::list_voltage`] + pub fn list_voltage(&self, selector: c_uint) -> Result> { + self.0.list_voltage(selector) + } + + /// See [`Regulator::get_linear_step`] + pub fn get_linear_step(&self) -> Option { + self.0.get_linear_step() + } + + /// See [`Regulator::get_voltage`] + pub fn get_voltage(&self) -> Result { + self.0.get_voltage() + } + + /// See [`Regulator::set_voltage`] + pub fn set_voltage(&mut self, min_uv: c_int, max_uv: c_int) -> Result { + self.0.set_voltage(min_uv, max_uv) + } + + /// See [`Regulator::set_voltage_time`] + pub fn set_voltage_time(&mut self, old_uv: c_int, new_uv: c_int) -> Result { + self.0.set_voltage_time(old_uv, new_uv) + } + + /// See [`Regulator::sync_voltage`] + pub fn sync_voltage(&mut self) -> Result { + self.0.sync_voltage() + } + + /// See [`Regulator::get_current_limit`] + pub fn get_current_limit(&self) -> Result { + self.0.get_current_limit() + } + + /// See [`Regulator::set_current_limit`] + pub fn set_current_limit(&mut self, min_ua: c_int, max_ua: c_int) -> Result { + self.0.set_current_limit(min_ua, max_ua) + } + + /// See [`Regulator::set_load`] + pub fn set_load(&mut self, load_ua: c_int) -> Result { + self.0.set_load(load_ua) + } + + /// See [`Regulator::allow_bypass`] + pub fn allow_bypass(&mut self, allow: bool) -> Result { + self.0.allow_bypass(allow) + } + + /// See [`Regulator::set_mode`] + pub fn set_mode(&mut self, mode: Mode) -> Result { + self.0.set_mode(mode) + } + + /// See [`Regulator::get_mode`] + pub fn get_mode(&mut self) -> Result { + self.0.get_mode() + } +} + +impl Drop for EnabledRegulator { + fn drop(&mut self) { + // SAFETY: The pointer is valid and non-null by the type invariant + let _ = unsafe { bindings::regulator_disable(self.0 .0) }; + } +} + +impl PartialEq for Regulator { + fn eq(&self, other: &Regulator) -> bool { + // SAFETY: The pointers are valid and non-null by the type invariant + unsafe { bindings::regulator_is_equal(self.0, other.0) } + } +} + +impl PartialEq for Regulator { + fn eq(&self, other: &EnabledRegulator) -> bool { + self.eq(&other.0) + } +} + +impl PartialEq for EnabledRegulator { + fn eq(&self, other: &EnabledRegulator) -> bool { + self.0.eq(&other.0) + } +} + +impl PartialEq for EnabledRegulator { + fn eq(&self, other: &Regulator) -> bool { + self.0.eq(other) + } +} From 31a0a6b4780ca8dd448dec9d1bf54708f44103b6 Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Tue, 5 Dec 2023 18:05:59 -0800 Subject: [PATCH 50/51] rust: regulator: add Regulator Driver abstraction This commit adds a Rust abstraction to write Regulator drivers. Only the features used by the NCV6336 driver were added to this abstraction. Signed-off-by: Fabien Parent --- rust/bindings/bindings_helper.h | 1 + rust/kernel/regulator.rs | 1 + rust/kernel/regulator/driver.rs | 750 ++++++++++++++++++++++++++++++++ 3 files changed, 752 insertions(+) create mode 100644 rust/kernel/regulator/driver.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index fd9430524fe4a8..11e11b81391114 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/kernel/regulator.rs b/rust/kernel/regulator.rs index a00b8577ca8121..c932794030e45f 100644 --- a/rust/kernel/regulator.rs +++ b/rust/kernel/regulator.rs @@ -3,6 +3,7 @@ //! SoC Regulators pub mod consumer; +pub mod driver; use crate::{ bindings, diff --git a/rust/kernel/regulator/driver.rs b/rust/kernel/regulator/driver.rs new file mode 100644 index 00000000000000..b8b27d5fdbd7f2 --- /dev/null +++ b/rust/kernel/regulator/driver.rs @@ -0,0 +1,750 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! SoC Regulator Driver Interface +//! +//! C header: [`include/linux/regulator/driver.h`](srctree/include/linux/regulator/driver.h) +//! +//! # Examples +//! +//! ```ignore +//! use kernel::regulator::driver::{Config, Desc, Driver, Registration, Type}; +//! # use kernel::{c_str, device::Device, module_platform_driver, of, platform, prelude::*}; +//! # +//! # module_platform_driver! { +//! # type: MyRegulatorDriver, +//! # name: "my_regulator_driver", +//! # license: "GPL", +//! # } +//! # +//! # kernel::module_of_id_table!(OF_MOD_TABLE, MY_REG_DRIVER_OF_ID_TABLE); +//! # kernel::define_of_id_table! {MY_REG_DRIVER_OF_ID_TABLE, (), [ +//! # (of::DeviceId::Compatible(b"my-regulator-driver"), None), +//! # ]} +//! +//! static DESC: Desc = +//! Desc::new::(c_str!("my-regulator-driver"), Type::Voltage) +//! .with_owner(&THIS_MODULE); +//! +//! struct MyRegulatorDriver; +//! +//! impl platform::Driver for MyRegulatorDriver { +//! # kernel::driver_of_id_table!(MY_REG_DRIVER_OF_ID_TABLE); +//! # +//! fn probe(pdev: &mut platform::Device, _id_info: Option<&Self::IdInfo>) -> Result { +//! let reg = Registration::register(pdev, &DESC, Config::::new(pdev))?; +//! Ok(()) +//! } +//! } +//! +//! #[vtable] +//! impl Driver for MyRegulatorDriver { +//! // Implement supported `Driver`'s operations here. +//! } +//! ``` + +use crate::{ + device::Device, + error::{code::*, from_err_ptr, from_result, Error, Result}, + macros::vtable, + regulator::Mode, + str::CStr, + types::ForeignOwnable, + ThisModule, +}; +use core::{ + marker::PhantomData, + mem::{ManuallyDrop, MaybeUninit}, +}; + +/// [`Regulator`]'s status +/// +/// Corresponds to the kernel's [`enum regulator_status`]. +/// +/// [`enum regulator_status`]: srctree/include/linux/regulator/driver.h +#[derive(Eq, PartialEq)] +pub enum Status { + /// Regulator is off + Off, + /// Regulator is on + On, + /// Regulator is in an error state + Error, + /// Regulator is on and in Fast mode + Fast, + /// Regulator is on and in Normal mode + Normal, + /// Regulator is on and in Idle mode + Idle, + /// Regulator is on and in Standby mode + Standby, + /// Regulator is enabled but not regulating + Bypass, + /// Regulator is any other status + Undefined, +} + +impl TryFrom for Status { + type Error = Error; + + fn try_from(status: core::ffi::c_uint) -> Result { + match status { + bindings::regulator_status_REGULATOR_STATUS_OFF => Ok(Self::Off), + bindings::regulator_status_REGULATOR_STATUS_ON => Ok(Self::On), + bindings::regulator_status_REGULATOR_STATUS_ERROR => Ok(Self::Error), + bindings::regulator_status_REGULATOR_STATUS_FAST => Ok(Self::Fast), + bindings::regulator_status_REGULATOR_STATUS_NORMAL => Ok(Self::Normal), + bindings::regulator_status_REGULATOR_STATUS_IDLE => Ok(Self::Idle), + bindings::regulator_status_REGULATOR_STATUS_STANDBY => Ok(Self::Standby), + bindings::regulator_status_REGULATOR_STATUS_BYPASS => Ok(Self::Bypass), + bindings::regulator_status_REGULATOR_STATUS_UNDEFINED => Ok(Self::Undefined), + _ => Err(EINVAL), + } + } +} + +impl From for Status { + fn from(mode: Mode) -> Self { + // SAFETY: `regulator_mode_to_status` is a pure function that is only doing integer + // to integer conversion, hence this function call is safe. + let status = unsafe { bindings::regulator_mode_to_status(mode as _) }; + + if status < 0 { + Self::Undefined + } else { + Self::try_from(status as core::ffi::c_uint).unwrap_or(Self::Undefined) + } + } +} + +/// Regulator's operations +#[vtable] +pub trait Driver { + /// User data that will be accessible to all operations + type Data: ForeignOwnable + Send + Sync = (); + + /// Return one of the supported voltages, in microvolt; zero if the selector indicates a + /// voltage that is unusable by the system; or negative errno. Selectors range from zero to one + /// less than the number of voltages supported by the system. + fn list_voltage(_rdev: &Regulator, _selector: u32) -> Result { + Err(ENOTSUPP) + } + + /// Set the voltage for the regulator within the range specified. The driver should select the + /// voltage closest to `min_uv`. + fn set_voltage(_rdev: &Regulator, _min_uv: i32, _max_uv: i32) -> Result { + Err(ENOTSUPP) + } + + /// Set the voltage for the regulator using the specified selector. + fn set_voltage_sel(_rdev: &Regulator, _selector: u32) -> Result { + Err(ENOTSUPP) + } + + /// Convert a voltage into a selector. + fn map_voltage(_rdev: &Regulator, _min_uv: i32, _max_uv: i32) -> Result { + Err(ENOTSUPP) + } + + /// Get the currently configured voltage for the regulator; Returns + /// [`ENOTRECOVERABLE`] if the regulator can't be read at bootup and hasn't been + /// set yet. + fn get_voltage(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Get the currently configured voltage selector for the regulator; Returns + /// [`ENOTRECOVERABLE`] if the regulator can't be read at bootup and hasn't been + /// set yet. + fn get_voltage_sel(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Configure a limit for a current-limited regulator. + /// + /// The driver should select the current closest to `max_ua`. + fn set_current_limit(_rdev: &Regulator, _min_ua: i32, _max_ua: i32) -> Result { + Err(ENOTSUPP) + } + + /// Get the configured limit for a current-limited regulator. + fn get_current_limit(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Enable or disable the active discharge of the regulator. + fn set_active_discharge(_rdev: &Regulator, _enable: bool) -> Result { + Err(ENOTSUPP) + } + + /// Configure the regulator as enabled. + fn enable(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Configure the regulator as disabled. + fn disable(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Returns enablement state of the regulator. + fn is_enabled(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Set the configured operating [`Mode`] for the regulator. + fn set_mode(_rdev: &Regulator, _mode: Mode) -> Result { + Err(ENOTSUPP) + } + + /// Get the configured operating [`Mode`] for the regulator + fn get_mode(_rdev: &Regulator) -> Mode { + Mode::Invalid + } + + /// Report the regulator [`Status`]. + fn get_status(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Set the voltage for the regaultor when the system is suspended. + fn set_suspend_voltage(_rdev: &Regulator, _uv: i32) -> Result { + Err(ENOTSUPP) + } + + /// Mark the regulator as enabled when the system is suspended. + fn set_suspend_enable(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Mark the regulator as disabled when the system is suspended. + fn set_suspend_disable(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Set the operating mode for the regulator when the system is suspended. + fn set_suspend_mode(_rdev: &Regulator, _mode: Mode) -> Result { + Err(ENOTSUPP) + } +} + +/// [`Regulator`]'s descriptor +/// +/// # Invariants +/// +/// `self.0` has always valid data. +pub struct Desc(bindings::regulator_desc); +impl Desc { + /// Create a new [`Regulator`] descriptor + pub const fn new(name: &'static CStr, reg_type: Type) -> Self { + // SAFETY: `desc` is a C structure holding data that has been initialized with 0s, + // hence it is safe to use as-is. + let mut desc = unsafe { MaybeUninit::::zeroed().assume_init() }; + desc.name = name.as_char_ptr(); + desc.type_ = match reg_type { + Type::Voltage => bindings::regulator_type_REGULATOR_VOLTAGE, + Type::Current => bindings::regulator_type_REGULATOR_CURRENT, + }; + desc.ops = Adapter::::build(); + Self(desc) + } + + /// Setup the register address, mask, and {en,dis}able values + pub const fn with_enable(mut self, reg: u32, mask: u32, en_val: u32, dis_val: u32) -> Self { + self.0.enable_reg = reg; + self.0.enable_mask = mask; + self.0.enable_val = en_val; + self.0.disable_val = dis_val; + self + } + + /// Setup the register address, mask, and {en,dis}able values. {En,Dis}able values are + /// inverted, i.e. `dis_val` will be use to enable the regulator while `en_val` will be used + /// to disable the regulator. + pub const fn with_inverted_enable( + mut self, + reg: u32, + mask: u32, + en_val: u32, + dis_val: u32, + ) -> Self { + self.0.enable_is_inverted = true; + self.with_enable(reg, mask, en_val, dis_val) + } + + /// Setup the active discharge regiter address, mask, on/off values. + pub const fn with_active_discharge(mut self, reg: u32, mask: u32, on: u32, off: u32) -> Self { + self.0.active_discharge_on = on; + self.0.active_discharge_off = off; + self.0.active_discharge_reg = reg; + self.0.active_discharge_mask = mask; + self + } + + /// Setup the current selection register address, mask, and current table + pub const fn with_csel(mut self, reg: u32, mask: u32, table: &'static [u32]) -> Self { + self.0.csel_reg = reg; + self.0.csel_mask = mask; + self.0.curr_table = table.as_ptr(); + self + } + + /// Voltages are a linear mapping + pub const fn with_linear_mapping( + mut self, + reg: u32, + mask: u32, + min_uv: u32, + uv_step: u32, + n_voltages: u32, + linear_min_sel: u32, + ) -> Self { + self.0.vsel_reg = reg; + self.0.vsel_mask = mask; + self.0.n_voltages = n_voltages; + self.0.min_uV = min_uv; + self.0.uV_step = uv_step; + self.0.linear_min_sel = linear_min_sel; + self + } + + /// Set the regulator owner + pub const fn with_owner(mut self, owner: &'static ThisModule) -> Self { + self.0.owner = owner.as_ptr(); + self + } +} + +// SAFETY: `Desc` cannot be modified after its declaration, hence it is safe to share references +// between threads. +unsafe impl Sync for Desc {} + +/// [`Regulator`]'s Config +/// +/// # Examples +/// +/// ``` +/// use kernel::regulator::driver::Config; +/// # use kernel::regulator::driver::{Desc, Registration}; +/// # use kernel::{device::Device, sync::Arc}; +/// +/// struct DriverData(u32); +/// +/// # fn probe(dev: &Device, desc: &'static Desc) -> Result { +/// let config = Config::>::new(&dev, Arc::try_new(DriverData(128))?); +/// let reg = Registration::register(dev, desc, config)?; +/// # Ok(()) +/// # } +/// ``` +/// +/// # Invariants +/// +/// `self.cfg` always hold valid data. +pub struct Config { + cfg: bindings::regulator_config, + data: T, +} + +impl Config { + /// Create a [`Regulator`] config. + pub fn new(dev: impl AsRef, data: T) -> Self { + Self { + cfg: bindings::regulator_config { + dev: dev.as_ref().as_raw(), + ..Default::default() + }, + data, + } + } +} + +/// Registration structure for Regulator drivers. +pub struct Registration(#[allow(dead_code)] Regulator); + +impl Registration { + /// register a Regulator driver + pub fn register( + dev: impl AsRef, + desc: &'static Desc, + cfg: Config, + ) -> Result { + Ok(Self(Regulator::register(dev, desc, cfg)?)) + } +} + +/// Regulator device +/// +/// Wraps the C structure `regulator_dev`. +/// +/// # Invariants +/// +/// * `self.rdev` is valid and non-null. +/// * [`Self`] has ownership of `self.rdev` memory allocation. +pub struct Regulator { + rdev: *mut bindings::regulator_dev, +} + +impl Regulator { + /// register a Regulator driver + fn register( + dev: impl AsRef, + desc: &'static Desc, + mut config: Config, + ) -> Result { + config.cfg.driver_data = config.data.into_foreign() as _; + + // SAFETY: By the type invariants, we know that `dev.as_ref().as_raw()` is always + // valid and non-null, and the descriptor and config are guaranteed to be valid values, + // hence it is safe to perform the FFI call. + let rdev = from_err_ptr(unsafe { + bindings::regulator_register(dev.as_ref().as_raw(), &desc.0, &config.cfg) + })?; + + if rdev.is_null() { + Err(EINVAL) + } else { + Ok(Self { rdev }) + } + } + + /// List voltages when the regulator is using linear mapping + pub fn list_voltage_linear(&self, selector: u32) -> Result { + // SAFETY: By the type invariants, we know that `self.rdev` is always valid and non-null. + // The C function is safe to call with any selector values. + let ret = unsafe { bindings::regulator_list_voltage_linear(self.rdev, selector) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + Ok(ret) + } + + /// Get regulator's name + pub fn get_name(&self) -> &'static CStr { + // SAFETY: By the type invariants, we know that `self.rdev` is always valid and non-null. + // The C function is guaranteed to return a valid string. + unsafe { CStr::from_char_ptr(bindings::rdev_get_name(self.rdev)) } + } + + /// Get regulator's ID + pub fn get_id(&self) -> i32 { + // SAFETY: By the type invariants, we know that `self.rdev` is always valid and non-null. + unsafe { bindings::rdev_get_id(self.rdev) } + } + + /// Retrieve driver data associated to `self` + pub fn data(&self) -> T::Borrowed<'_> { + // SAFETY: By the type invariants, we know that `self.rdev` is always valid and non-null. + unsafe { T::borrow(bindings::rdev_get_drvdata(self.rdev)) } + } +} + +impl Drop for Regulator { + fn drop(&mut self) { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + unsafe { bindings::regulator_unregister(self.rdev) } + } +} + +// SAFETY: `Regulator` has sole ownership of `self.rdev` and is never read outside of the C +// implementation. It is safe to use it from any thread. +unsafe impl Send for Regulator {} + +/// [`Regulator`] type +pub enum Type { + /// Voltage regulator + Voltage, + /// Current regulator + Current, +} + +pub(crate) struct Adapter(PhantomData); + +impl Adapter { + unsafe extern "C" fn list_voltage_callback( + rdev: *mut bindings::regulator_dev, + selector: core::ffi::c_uint, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| T::list_voltage(&rdev, selector)) + } + + unsafe extern "C" fn set_voltage_callback( + rdev: *mut bindings::regulator_dev, + min_uv: core::ffi::c_int, + max_uv: core::ffi::c_int, + selector: *mut core::ffi::c_uint, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + match T::set_voltage(&rdev, min_uv, max_uv) { + Ok(v) => { + unsafe { *selector = v as _ }; + 0 + } + Err(e) => e.to_errno(), + } + } + + unsafe extern "C" fn map_voltage_callback( + rdev: *mut bindings::regulator_dev, + min_uv: core::ffi::c_int, + max_uv: core::ffi::c_int, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| T::map_voltage(&rdev, min_uv, max_uv)) + } + + unsafe extern "C" fn set_voltage_sel_callback( + rdev: *mut bindings::regulator_dev, + selector: core::ffi::c_uint, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| { + T::set_voltage_sel(&rdev, selector)?; + Ok(0) + }) + } + + unsafe extern "C" fn get_voltage_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| T::get_voltage(&rdev)) + } + + unsafe extern "C" fn get_voltage_sel_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| T::get_voltage_sel(&rdev)) + } + + unsafe extern "C" fn set_current_limit_callback( + rdev: *mut bindings::regulator_dev, + min_ua: core::ffi::c_int, + max_ua: core::ffi::c_int, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| { + T::set_current_limit(&rdev, min_ua, max_ua)?; + Ok(0) + }) + } + + unsafe extern "C" fn get_current_limit_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| T::get_current_limit(&rdev)) + } + + unsafe extern "C" fn set_active_discharge_callback( + rdev: *mut bindings::regulator_dev, + enable: bool, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| { + T::set_active_discharge(&rdev, enable)?; + Ok(0) + }) + } + + unsafe extern "C" fn enable_callback(rdev: *mut bindings::regulator_dev) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| { + T::enable(&rdev)?; + Ok(0) + }) + } + + unsafe extern "C" fn disable_callback(rdev: *mut bindings::regulator_dev) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| { + T::disable(&rdev)?; + Ok(0) + }) + } + + unsafe extern "C" fn is_enabled_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| { + T::is_enabled(&rdev)?; + Ok(0) + }) + } + + unsafe extern "C" fn set_mode_callback( + rdev: *mut bindings::regulator_dev, + mode: core::ffi::c_uint, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| { + let mode = Mode::try_from(mode).unwrap_or(Mode::Invalid); + T::set_mode(&rdev, mode)?; + Ok(0) + }) + } + + unsafe extern "C" fn get_mode_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_uint { + let rdev = ManuallyDrop::new(Regulator { rdev }); + T::get_mode(&rdev) as _ + } + + unsafe extern "C" fn get_status_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| Ok(T::get_status(&rdev)? as _)) + } + + unsafe extern "C" fn set_suspend_voltage_callback( + rdev: *mut bindings::regulator_dev, + uv: core::ffi::c_int, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| { + T::set_suspend_voltage(&rdev, uv)?; + Ok(0) + }) + } + + unsafe extern "C" fn set_suspend_enable_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| { + T::set_suspend_enable(&rdev)?; + Ok(0) + }) + } + + unsafe extern "C" fn set_suspend_disable_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| { + T::set_suspend_disable(&rdev)?; + Ok(0) + }) + } + + unsafe extern "C" fn set_suspend_mode_callback( + rdev: *mut bindings::regulator_dev, + mode: core::ffi::c_uint, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { rdev }); + from_result(|| { + let mode = Mode::try_from(mode).unwrap_or(Mode::Invalid); + T::set_suspend_mode(&rdev, mode)?; + Ok(0) + }) + } + + const VTABLE: bindings::regulator_ops = bindings::regulator_ops { + list_voltage: if T::HAS_LIST_VOLTAGE { + Some(Adapter::::list_voltage_callback) + } else { + None + }, + set_voltage: if T::HAS_SET_VOLTAGE { + Some(Adapter::::set_voltage_callback) + } else { + None + }, + map_voltage: if T::HAS_MAP_VOLTAGE { + Some(Adapter::::map_voltage_callback) + } else { + None + }, + set_voltage_sel: if T::HAS_SET_VOLTAGE_SEL { + Some(Adapter::::set_voltage_sel_callback) + } else { + None + }, + get_voltage: if T::HAS_GET_VOLTAGE { + Some(Adapter::::get_voltage_callback) + } else { + None + }, + get_voltage_sel: if T::HAS_GET_VOLTAGE_SEL { + Some(Adapter::::get_voltage_sel_callback) + } else { + None + }, + set_current_limit: if T::HAS_SET_CURRENT_LIMIT { + Some(Adapter::::set_current_limit_callback) + } else { + None + }, + get_current_limit: if T::HAS_GET_CURRENT_LIMIT { + Some(Adapter::::get_current_limit_callback) + } else { + None + }, + set_active_discharge: if T::HAS_SET_ACTIVE_DISCHARGE { + Some(Adapter::::set_active_discharge_callback) + } else { + None + }, + enable: if T::HAS_ENABLE { + Some(Adapter::::enable_callback) + } else { + None + }, + disable: if T::HAS_DISABLE { + Some(Adapter::::disable_callback) + } else { + None + }, + is_enabled: if T::HAS_IS_ENABLED { + Some(Adapter::::is_enabled_callback) + } else { + None + }, + set_mode: if T::HAS_SET_MODE { + Some(Adapter::::set_mode_callback) + } else { + None + }, + get_mode: if T::HAS_GET_MODE { + Some(Adapter::::get_mode_callback) + } else { + None + }, + get_status: if T::HAS_GET_STATUS { + Some(Adapter::::get_status_callback) + } else { + None + }, + set_suspend_voltage: if T::HAS_SET_SUSPEND_VOLTAGE { + Some(Adapter::::set_suspend_voltage_callback) + } else { + None + }, + set_suspend_enable: if T::HAS_SET_SUSPEND_ENABLE { + Some(Adapter::::set_suspend_enable_callback) + } else { + None + }, + set_suspend_disable: if T::HAS_SET_SUSPEND_DISABLE { + Some(Adapter::::set_suspend_disable_callback) + } else { + None + }, + set_suspend_mode: if T::HAS_SET_SUSPEND_MODE { + Some(Adapter::::set_suspend_mode_callback) + } else { + None + }, + // SAFETY: The rest is zeroed out to initialize `struct regulator_ops`, + // sets `Option<&F>` to be `None`. + ..unsafe { core::mem::MaybeUninit::::zeroed().assume_init() } + }; + + const fn build() -> &'static bindings::regulator_ops { + &Self::VTABLE + } +} From 0b4251203bf0c2c603d73e1e460c7f069c08e156 Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Fri, 1 Mar 2024 13:00:07 -0800 Subject: [PATCH 51/51] rust: regulator: Add support for regmap The regulator API offer many helpers to help simplifies drivers that use the regmap API. This commit adds partial support for it, only the function needed by the NCV6336 driver were added. In case CONFIG_REGMAP=n, we create a dummy Regmap implementation in order to avoid to have to surround a lot of the declarations and statements with #[cfg(CONFIG_REGMAP)]. Signed-off-by: Fabien Parent --- rust/kernel/regulator/driver.rs | 247 +++++++++++++++++++++++++++++--- 1 file changed, 227 insertions(+), 20 deletions(-) diff --git a/rust/kernel/regulator/driver.rs b/rust/kernel/regulator/driver.rs index b8b27d5fdbd7f2..235b45b38b1752 100644 --- a/rust/kernel/regulator/driver.rs +++ b/rust/kernel/regulator/driver.rs @@ -48,14 +48,27 @@ use crate::{ macros::vtable, regulator::Mode, str::CStr, + sync::Arc, types::ForeignOwnable, ThisModule, }; +#[cfg(CONFIG_REGMAP)] +use crate::{error::to_result, regmap::Regmap}; use core::{ marker::PhantomData, mem::{ManuallyDrop, MaybeUninit}, }; +#[cfg(not(CONFIG_REGMAP))] +struct Regmap; + +#[cfg(not(CONFIG_REGMAP))] +impl Regmap { + fn as_ptr(&self) -> *mut bindings::regmap { + core::ptr::null_mut() + } +} + /// [`Regulator`]'s status /// /// Corresponds to the kernel's [`enum regulator_status`]. @@ -342,6 +355,7 @@ unsafe impl Sync for Desc {} pub struct Config { cfg: bindings::regulator_config, data: T, + regmap: Option>, } impl Config { @@ -353,8 +367,16 @@ impl Config { ..Default::default() }, data, + regmap: None, } } + + /// Assign a regmap device to the config + #[cfg(CONFIG_REGMAP)] + pub fn with_regmap(mut self, regmap: Arc) -> Self { + self.regmap = Some(regmap); + self + } } /// Registration structure for Regulator drivers. @@ -381,6 +403,7 @@ impl Registration { /// * [`Self`] has ownership of `self.rdev` memory allocation. pub struct Regulator { rdev: *mut bindings::regulator_dev, + _regmap: Option>, } impl Regulator { @@ -392,6 +415,11 @@ impl Regulator { ) -> Result { config.cfg.driver_data = config.data.into_foreign() as _; + let regmap = config.regmap.take(); + if let Some(regmap) = ®map { + config.cfg.regmap = regmap.as_ptr(); + }; + // SAFETY: By the type invariants, we know that `dev.as_ref().as_raw()` is always // valid and non-null, and the descriptor and config are guaranteed to be valid values, // hence it is safe to perform the FFI call. @@ -402,7 +430,10 @@ impl Regulator { if rdev.is_null() { Err(EINVAL) } else { - Ok(Self { rdev }) + Ok(Self { + rdev, + _regmap: regmap, + }) } } @@ -439,9 +470,17 @@ impl Regulator { impl Drop for Regulator { fn drop(&mut self) { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + let regmap = unsafe { bindings::rdev_get_regmap(self.rdev) }; + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, // so it is safe to perform the FFI call. unsafe { bindings::regulator_unregister(self.rdev) } + + if !regmap.is_null() { + drop(unsafe { Arc::from_raw(regmap) }); + } } } @@ -449,6 +488,117 @@ impl Drop for Regulator { // implementation. It is safe to use it from any thread. unsafe impl Send for Regulator {} +mod sealed { + pub trait Sealed {} +} + +impl sealed::Sealed for Regulator {} + +/// Helper functions to implement some of the [`Driver`] trait methods using [`Regmap`]. +/// +/// This trait is implemented by [`Regulator`] and is Sealed to prevent +/// to be implemented by anyone else. +#[cfg(CONFIG_REGMAP)] +pub trait RegmapHelpers: sealed::Sealed { + /// Implementation of [`Driver::get_voltage_sel`] using [`Regmap`]. + fn get_voltage_sel_regmap(&self) -> Result; + /// Implementation of [`Driver::set_voltage_sel`] using [`Regmap`]. + fn set_voltage_sel_regmap(&self, sel: u32) -> Result; + + /// Implementation of [`Driver::is_enabled`] using [`Regmap`]. + /// + /// [`Desc::with_enable`] or [`Desc::with_inverted_enable`] must have been called + /// to setup the fields required by regmap. + fn is_enabled_regmap(&self) -> Result; + + /// Implementation of [`Driver::enable`] using [`Regmap`]. + /// + /// [`Desc::with_enable`] or [`Desc::with_inverted_enable`] must have been called + /// to setup the fields required by regmap. + fn enable_regmap(&self) -> Result; + + /// Implementation of [`Driver::disable`] using [`Regmap`]. + /// + /// [`Desc::with_enable`] or [`Desc::with_inverted_enable`] must have been called + /// to setup the fields required by regmap. + fn disable_regmap(&self) -> Result; + + /// Implementation of [`Driver::set_active_discharge`] using [`Regmap`]. + /// + /// [`Desc::with_active_discharge`] must have been called to setup the fields required + /// by regmap. + fn set_active_discharge_regmap(&self, enable: bool) -> Result; + /// Implementation of [`Driver::set_current_limit`] using [`Regmap`]. + fn set_current_limit_regmap(&self, min_ua: i32, max_ua: i32) -> Result; + /// Implementation of [`Driver::get_current_limit`] using [`Regmap`]. + fn get_current_limit_regmap(&self) -> Result; +} + +#[cfg(CONFIG_REGMAP)] +impl RegmapHelpers for Regulator { + fn get_voltage_sel_regmap(&self) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + let ret = unsafe { bindings::regulator_get_voltage_sel_regmap(self.rdev) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + Ok(ret) + } + + fn set_voltage_sel_regmap(&self, sel: u32) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + to_result(unsafe { bindings::regulator_set_voltage_sel_regmap(self.rdev, sel) }) + } + + fn is_enabled_regmap(&self) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + let ret = unsafe { bindings::regulator_is_enabled_regmap(self.rdev) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + Ok(ret > 0) + } + + fn enable_regmap(&self) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + to_result(unsafe { bindings::regulator_enable_regmap(self.rdev) }) + } + + fn disable_regmap(&self) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + to_result(unsafe { bindings::regulator_disable_regmap(self.rdev) }) + } + + fn set_active_discharge_regmap(&self, enable: bool) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + to_result(unsafe { bindings::regulator_set_active_discharge_regmap(self.rdev, enable) }) + } + + fn set_current_limit_regmap(&self, min_ua: i32, max_ua: i32) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + to_result(unsafe { + bindings::regulator_set_current_limit_regmap(self.rdev, min_ua, max_ua) + }) + } + + fn get_current_limit_regmap(&self) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + let ret = unsafe { bindings::regulator_get_current_limit_regmap(self.rdev) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + Ok(ret) + } +} + /// [`Regulator`] type pub enum Type { /// Voltage regulator @@ -464,7 +614,10 @@ impl Adapter { rdev: *mut bindings::regulator_dev, selector: core::ffi::c_uint, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| T::list_voltage(&rdev, selector)) } @@ -474,7 +627,10 @@ impl Adapter { max_uv: core::ffi::c_int, selector: *mut core::ffi::c_uint, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); match T::set_voltage(&rdev, min_uv, max_uv) { Ok(v) => { unsafe { *selector = v as _ }; @@ -489,7 +645,10 @@ impl Adapter { min_uv: core::ffi::c_int, max_uv: core::ffi::c_int, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| T::map_voltage(&rdev, min_uv, max_uv)) } @@ -497,7 +656,10 @@ impl Adapter { rdev: *mut bindings::regulator_dev, selector: core::ffi::c_uint, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| { T::set_voltage_sel(&rdev, selector)?; Ok(0) @@ -507,14 +669,20 @@ impl Adapter { unsafe extern "C" fn get_voltage_callback( rdev: *mut bindings::regulator_dev, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| T::get_voltage(&rdev)) } unsafe extern "C" fn get_voltage_sel_callback( rdev: *mut bindings::regulator_dev, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| T::get_voltage_sel(&rdev)) } @@ -523,7 +691,10 @@ impl Adapter { min_ua: core::ffi::c_int, max_ua: core::ffi::c_int, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| { T::set_current_limit(&rdev, min_ua, max_ua)?; Ok(0) @@ -533,7 +704,10 @@ impl Adapter { unsafe extern "C" fn get_current_limit_callback( rdev: *mut bindings::regulator_dev, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| T::get_current_limit(&rdev)) } @@ -541,7 +715,10 @@ impl Adapter { rdev: *mut bindings::regulator_dev, enable: bool, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| { T::set_active_discharge(&rdev, enable)?; Ok(0) @@ -549,7 +726,10 @@ impl Adapter { } unsafe extern "C" fn enable_callback(rdev: *mut bindings::regulator_dev) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| { T::enable(&rdev)?; Ok(0) @@ -557,7 +737,10 @@ impl Adapter { } unsafe extern "C" fn disable_callback(rdev: *mut bindings::regulator_dev) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| { T::disable(&rdev)?; Ok(0) @@ -567,7 +750,10 @@ impl Adapter { unsafe extern "C" fn is_enabled_callback( rdev: *mut bindings::regulator_dev, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| { T::is_enabled(&rdev)?; Ok(0) @@ -578,7 +764,10 @@ impl Adapter { rdev: *mut bindings::regulator_dev, mode: core::ffi::c_uint, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| { let mode = Mode::try_from(mode).unwrap_or(Mode::Invalid); T::set_mode(&rdev, mode)?; @@ -589,14 +778,20 @@ impl Adapter { unsafe extern "C" fn get_mode_callback( rdev: *mut bindings::regulator_dev, ) -> core::ffi::c_uint { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); T::get_mode(&rdev) as _ } unsafe extern "C" fn get_status_callback( rdev: *mut bindings::regulator_dev, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| Ok(T::get_status(&rdev)? as _)) } @@ -604,7 +799,10 @@ impl Adapter { rdev: *mut bindings::regulator_dev, uv: core::ffi::c_int, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| { T::set_suspend_voltage(&rdev, uv)?; Ok(0) @@ -614,7 +812,10 @@ impl Adapter { unsafe extern "C" fn set_suspend_enable_callback( rdev: *mut bindings::regulator_dev, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| { T::set_suspend_enable(&rdev)?; Ok(0) @@ -624,7 +825,10 @@ impl Adapter { unsafe extern "C" fn set_suspend_disable_callback( rdev: *mut bindings::regulator_dev, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| { T::set_suspend_disable(&rdev)?; Ok(0) @@ -635,7 +839,10 @@ impl Adapter { rdev: *mut bindings::regulator_dev, mode: core::ffi::c_uint, ) -> core::ffi::c_int { - let rdev = ManuallyDrop::new(Regulator { rdev }); + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); from_result(|| { let mode = Mode::try_from(mode).unwrap_or(Mode::Invalid); T::set_suspend_mode(&rdev, mode)?;