diff --git a/MAINTAINERS b/MAINTAINERS index 3380b87dcbb983..b77f4495dcf427 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -20230,6 +20230,13 @@ F: scripts/*rust* F: tools/testing/selftests/rust/ K: \b(?i:rust)\b +RUST [ALLOC] +M: Danilo Krummrich +L: rust-for-linux@vger.kernel.org +S: Maintained +F: rust/kernel/alloc.rs +F: rust/kernel/alloc/ + RXRPC SOCKETS (AF_RXRPC) M: David Howells M: Marc Dionne diff --git a/drivers/block/rnull.rs b/drivers/block/rnull.rs index b0227cf9ddd387..0e0e9ed7851e96 100644 --- a/drivers/block/rnull.rs +++ b/drivers/block/rnull.rs @@ -32,11 +32,11 @@ module! { } struct NullBlkModule { - _disk: Pin>>>, + _disk: Pin>>>, } impl kernel::Module for NullBlkModule { - fn init(_module: &'static ThisModule) -> Result { + fn init(_name: &'static CStr, _module: &'static ThisModule) -> Result { pr_info!("Rust null_blk loaded\n"); let tagset = Arc::pin_init(TagSet::new(1, 256, 1), flags::GFP_KERNEL)?; @@ -47,7 +47,7 @@ impl kernel::Module for NullBlkModule { .rotational(false) .build(format_args!("rnullb{}", 0), tagset)?; - let disk = Box::pin_init(new_mutex!(disk, "nullb:disk"), flags::GFP_KERNEL)?; + let disk = KBox::pin_init(new_mutex!(disk, "nullb:disk"), flags::GFP_KERNEL)?; Ok(Self { _disk: disk }) } diff --git a/mm/kasan/kasan_test_rust.rs b/mm/kasan/kasan_test_rust.rs index 47bcf033dd4f06..5b34edf30e7244 100644 --- a/mm/kasan/kasan_test_rust.rs +++ b/mm/kasan/kasan_test_rust.rs @@ -11,7 +11,7 @@ use kernel::prelude::*; /// drop the vector, and touch it. #[no_mangle] pub extern "C" fn kasan_test_rust_uaf() -> u8 { - let mut v: Vec = Vec::new(); + let mut v: KVec = KVec::new(); for _ in 0..4096 { v.push(0x42, GFP_KERNEL).unwrap(); } diff --git a/rust/Makefile b/rust/Makefile index 0856fd6bc61057..3678e79317f12d 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -15,8 +15,8 @@ always-$(CONFIG_RUST) += libmacros.so no-clean-files += libmacros.so always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs -obj-$(CONFIG_RUST) += alloc.o bindings.o kernel.o -always-$(CONFIG_RUST) += exports_alloc_generated.h exports_helpers_generated.h \ +obj-$(CONFIG_RUST) += bindings.o kernel.o +always-$(CONFIG_RUST) += exports_helpers_generated.h \ exports_bindings_generated.h exports_kernel_generated.h always-$(CONFIG_RUST) += uapi/uapi_generated.rs @@ -53,11 +53,6 @@ endif core-cfgs = \ --cfg no_fp_fmt_parse -alloc-cfgs = \ - --cfg no_global_oom_handling \ - --cfg no_rc \ - --cfg no_sync - quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< cmd_rustdoc = \ OBJTREE=$(abspath $(objtree)) \ @@ -81,7 +76,7 @@ quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< # command-like flags to solve the issue. Meanwhile, we use the non-custom case # and then retouch the generated files. rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \ - rustdoc-alloc rustdoc-kernel + rustdoc-kernel $(Q)cp $(srctree)/Documentation/images/logo.svg $(rustdoc_output)/static.files/ $(Q)cp $(srctree)/Documentation/images/COPYING-logo $(rustdoc_output)/static.files/ $(Q)find $(rustdoc_output) -name '*.html' -type f -print0 | xargs -0 sed -Ei \ @@ -108,20 +103,11 @@ rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs FORCE rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE +$(call if_changed,rustdoc) -# We need to allow `rustdoc::broken_intra_doc_links` because some -# `no_global_oom_handling` functions refer to non-`no_global_oom_handling` -# functions. Ideally `rustdoc` would have a way to distinguish broken links -# due to things that are "configured out" vs. entirely non-existing ones. -rustdoc-alloc: private rustc_target_flags = $(alloc-cfgs) \ - -Arustdoc::broken_intra_doc_links -rustdoc-alloc: $(RUST_LIB_SRC)/alloc/src/lib.rs rustdoc-core rustdoc-compiler_builtins FORCE - +$(call if_changed,rustdoc) - -rustdoc-kernel: private rustc_target_flags = --extern alloc \ +rustdoc-kernel: private rustc_target_flags = \ --extern build_error --extern macros=$(objtree)/$(obj)/libmacros.so \ --extern bindings --extern uapi rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-macros \ - rustdoc-compiler_builtins rustdoc-alloc $(obj)/libmacros.so \ + rustdoc-compiler_builtins $(obj)/libmacros.so \ $(obj)/bindings.o FORCE +$(call if_changed,rustdoc) @@ -165,7 +151,7 @@ quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $< mkdir -p $(objtree)/$(obj)/test/doctests/kernel; \ OBJTREE=$(abspath $(objtree)) \ $(RUSTDOC) --test $(rust_flags) \ - -L$(objtree)/$(obj) --extern alloc --extern kernel \ + -L$(objtree)/$(obj) --extern kernel \ --extern build_error --extern macros \ --extern bindings --extern uapi \ --no-run --crate-name kernel -Zunstable-options \ @@ -201,7 +187,7 @@ rusttest-macros: $(src)/macros/lib.rs FORCE +$(call if_changed,rustc_test) +$(call if_changed,rustdoc_test) -rusttest-kernel: private rustc_target_flags = --extern alloc \ +rusttest-kernel: private rustc_target_flags = \ --extern build_error --extern macros --extern bindings --extern uapi rusttest-kernel: $(src)/kernel/lib.rs \ rusttestlib-build_error rusttestlib-macros rusttestlib-bindings \ @@ -314,9 +300,6 @@ quiet_cmd_exports = EXPORTS $@ $(obj)/exports_core_generated.h: $(obj)/core.o FORCE $(call if_changed,exports) -$(obj)/exports_alloc_generated.h: $(obj)/alloc.o FORCE - $(call if_changed,exports) - # Even though Rust kernel modules should never use the bindings directly, # symbols from the `bindings` crate and the C helpers need to be exported # because Rust generics and inlined functions may not get their code generated @@ -363,7 +346,7 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L rust-analyzer: $(Q)$(srctree)/scripts/generate_rust_analyzer.py \ - --cfgs='core=$(core-cfgs)' --cfgs='alloc=$(alloc-cfgs)' \ + --cfgs='core=$(core-cfgs)' \ $(realpath $(srctree)) $(realpath $(objtree)) \ $(rustc_sysroot) $(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \ $(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json @@ -401,12 +384,6 @@ $(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' $(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE +$(call if_changed_rule,rustc_library) -$(obj)/alloc.o: private skip_clippy = 1 -$(obj)/alloc.o: private skip_flags = -Wunreachable_pub -$(obj)/alloc.o: private rustc_target_flags = $(alloc-cfgs) -$(obj)/alloc.o: $(RUST_LIB_SRC)/alloc/src/lib.rs $(obj)/compiler_builtins.o FORCE - +$(call if_changed_rule,rustc_library) - $(obj)/build_error.o: $(src)/build_error.rs $(obj)/compiler_builtins.o FORCE +$(call if_changed_rule,rustc_library) @@ -421,9 +398,9 @@ $(obj)/uapi.o: $(src)/uapi/lib.rs \ $(obj)/uapi/uapi_generated.rs FORCE +$(call if_changed_rule,rustc_library) -$(obj)/kernel.o: private rustc_target_flags = --extern alloc \ +$(obj)/kernel.o: private rustc_target_flags = \ --extern build_error --extern macros --extern bindings --extern uapi -$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/alloc.o $(obj)/build_error.o \ +$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/build_error.o \ $(obj)/libmacros.so $(obj)/bindings.o $(obj)/uapi.o FORCE +$(call if_changed_rule,rustc_library) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index ae82e9c941afa1..11e11b81391114 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -13,10 +13,14 @@ #include #include #include +#include #include #include #include #include +#include +#include +#include #include #include #include @@ -31,4 +35,5 @@ const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT; const gfp_t RUST_CONST_HELPER_GFP_NOWAIT = GFP_NOWAIT; const gfp_t RUST_CONST_HELPER___GFP_ZERO = __GFP_ZERO; const gfp_t RUST_CONST_HELPER___GFP_HIGHMEM = ___GFP_HIGHMEM; +const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; diff --git a/rust/exports.c b/rust/exports.c index e5695f3b45b7aa..82a037381798d7 100644 --- a/rust/exports.c +++ b/rust/exports.c @@ -16,7 +16,6 @@ #define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym) #include "exports_core_generated.h" -#include "exports_alloc_generated.h" #include "exports_helpers_generated.h" #include "exports_bindings_generated.h" #include "exports_kernel_generated.h" diff --git a/rust/helpers/device.c b/rust/helpers/device.c new file mode 100644 index 00000000000000..b2135c6686b027 --- /dev/null +++ b/rust/helpers/device.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +int rust_helper_devm_add_action(struct device *dev, + void (*action)(void *), + void *data) +{ + return devm_add_action(dev, action, data); +} diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 30f40149f3a969..78f6513e66f0d2 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -11,16 +11,22 @@ #include "bug.c" #include "build_assert.c" #include "build_bug.c" +#include "device.c" #include "err.c" +#include "i2c.c" +#include "io.c" #include "kunit.c" #include "mutex.c" #include "page.c" #include "rbtree.c" +#include "rcu.c" #include "refcount.c" +#include "regmap.c" #include "signal.c" #include "slab.c" #include "spinlock.c" #include "task.c" #include "uaccess.c" +#include "vmalloc.c" #include "wait.c" #include "workqueue.c" diff --git a/rust/helpers/i2c.c b/rust/helpers/i2c.c new file mode 100644 index 00000000000000..8ffdc454e7597c --- /dev/null +++ b/rust/helpers/i2c.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void *rust_helper_i2c_get_clientdata(const struct i2c_client *client) +{ + return i2c_get_clientdata(client); +} + +void rust_helper_i2c_set_clientdata(struct i2c_client *client, void *data) +{ + i2c_set_clientdata(client, data); +} diff --git a/rust/helpers/io.c b/rust/helpers/io.c new file mode 100644 index 00000000000000..f9bb1bbf1fd5da --- /dev/null +++ b/rust/helpers/io.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +u8 rust_helper_readb(const volatile void __iomem *addr) +{ + return readb(addr); +} + +u16 rust_helper_readw(const volatile void __iomem *addr) +{ + return readw(addr); +} + +u32 rust_helper_readl(const volatile void __iomem *addr) +{ + return readl(addr); +} + +#ifdef CONFIG_64BIT +u64 rust_helper_readq(const volatile void __iomem *addr) +{ + return readq(addr); +} +#endif + +void rust_helper_writeb(u8 value, volatile void __iomem *addr) +{ + writeb(value, addr); +} + +void rust_helper_writew(u16 value, volatile void __iomem *addr) +{ + writew(value, addr); +} + +void rust_helper_writel(u32 value, volatile void __iomem *addr) +{ + writel(value, addr); +} + +#ifdef CONFIG_64BIT +void rust_helper_writeq(u64 value, volatile void __iomem *addr) +{ + writeq(value, addr); +} +#endif + +u8 rust_helper_readb_relaxed(const volatile void __iomem *addr) +{ + return readb_relaxed(addr); +} + +u16 rust_helper_readw_relaxed(const volatile void __iomem *addr) +{ + return readw_relaxed(addr); +} + +u32 rust_helper_readl_relaxed(const volatile void __iomem *addr) +{ + return readl_relaxed(addr); +} + +#ifdef CONFIG_64BIT +u64 rust_helper_readq_relaxed(const volatile void __iomem *addr) +{ + return readq_relaxed(addr); +} +#endif + +void rust_helper_writeb_relaxed(u8 value, volatile void __iomem *addr) +{ + writeb_relaxed(value, addr); +} + +void rust_helper_writew_relaxed(u16 value, volatile void __iomem *addr) +{ + writew_relaxed(value, addr); +} + +void rust_helper_writel_relaxed(u32 value, volatile void __iomem *addr) +{ + writel_relaxed(value, addr); +} + +#ifdef CONFIG_64BIT +void rust_helper_writeq_relaxed(u64 value, volatile void __iomem *addr) +{ + writeq_relaxed(value, addr); +} +#endif diff --git a/rust/helpers/rcu.c b/rust/helpers/rcu.c new file mode 100644 index 00000000000000..f1cec658351349 --- /dev/null +++ b/rust/helpers/rcu.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void rust_helper_rcu_read_lock(void) +{ + rcu_read_lock(); +} + +void rust_helper_rcu_read_unlock(void) +{ + rcu_read_unlock(); +} diff --git a/rust/helpers/regmap.c b/rust/helpers/regmap.c new file mode 100644 index 00000000000000..2426e563df339a --- /dev/null +++ b/rust/helpers/regmap.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#if IS_BUILTIN(CONFIG_REGMAP_I2C) +struct regmap *rust_helper_regmap_init_i2c(struct i2c_client *i2c, + const struct regmap_config *config) +{ + return regmap_init_i2c(i2c, config); +} +#endif + +int rust_helper_regmap_field_write(struct regmap_field *field, unsigned int val) +{ + return regmap_field_write(field, val); +} + +int rust_helper_regmap_field_force_write(struct regmap_field *field, + unsigned int val) +{ + return regmap_field_force_write(field, val); +} + +int rust_helper_regmap_field_update_bits(struct regmap_field *field, + unsigned int mask, unsigned int val) +{ + return regmap_field_update_bits(field, mask, val); +} + +int rust_helper_regmap_field_set_bits(struct regmap_field *field, + unsigned int bits) +{ + return regmap_field_set_bits(field, bits); +} + +int rust_helper_regmap_field_clear_bits(struct regmap_field *field, + unsigned int bits) +{ + return regmap_field_clear_bits(field, bits); +} + +int rust_helper_regmap_field_force_update_bits(struct regmap_field *field, + unsigned int mask, + unsigned int val) +{ + return regmap_field_force_update_bits(field, mask, val); +} diff --git a/rust/helpers/slab.c b/rust/helpers/slab.c index f043e087f9d666..a842bfbddcba91 100644 --- a/rust/helpers/slab.c +++ b/rust/helpers/slab.c @@ -7,3 +7,9 @@ rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) { return krealloc(objp, new_size, flags); } + +void * __must_check __realloc_size(2) +rust_helper_kvrealloc(const void *p, size_t size, gfp_t flags) +{ + return kvrealloc(p, size, flags); +} diff --git a/rust/helpers/vmalloc.c b/rust/helpers/vmalloc.c new file mode 100644 index 00000000000000..80d34501bbc010 --- /dev/null +++ b/rust/helpers/vmalloc.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void * __must_check __realloc_size(2) +rust_helper_vrealloc(const void *p, size_t size, gfp_t flags) +{ + return vrealloc(p, size, flags); +} diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 998779cc69769f..f2f7f3a53d298c 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -1,12 +1,29 @@ // SPDX-License-Identifier: GPL-2.0 -//! Extensions to the [`alloc`] crate. +//! Implementation of the kernel's memory allocation infrastructure. -#[cfg(not(test))] -#[cfg(not(testlib))] -mod allocator; -pub mod box_ext; -pub mod vec_ext; +#[cfg(not(any(test, testlib)))] +pub mod allocator; +pub mod kbox; +pub mod kvec; +pub mod layout; + +#[cfg(any(test, testlib))] +pub mod allocator_test; + +#[cfg(any(test, testlib))] +pub use self::allocator_test as allocator; + +pub use self::kbox::Box; +pub use self::kbox::KBox; +pub use self::kbox::KVBox; +pub use self::kbox::VBox; + +pub use self::kvec::IntoIter; +pub use self::kvec::KVVec; +pub use self::kvec::KVec; +pub use self::kvec::VVec; +pub use self::kvec::Vec; /// Indicates an allocation error. #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -18,7 +35,7 @@ use core::{alloc::Layout, ptr::NonNull}; /// They can be combined with the operators `|`, `&`, and `!`. /// /// Values can be used from the [`flags`] module. -#[derive(Clone, Copy)] +#[derive(Clone, Copy, PartialEq)] pub struct Flags(u32); impl Flags { @@ -26,6 +43,11 @@ impl Flags { pub(crate) fn as_raw(self) -> u32 { self.0 } + + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Flags) -> bool { + (self & flags) == flags + } } impl core::ops::BitOr for Flags { @@ -86,6 +108,11 @@ pub mod flags { /// use any filesystem callback. It is very likely to fail to allocate memory, even for very /// small allocations. pub const GFP_NOWAIT: Flags = Flags(bindings::GFP_NOWAIT); + + /// Suppresses allocation failure reports. + /// + /// This is normally or'd with other flags. + pub const __GFP_NOWARN: Flags = Flags(bindings::__GFP_NOWARN); } /// The kernel's [`Allocator`] trait. @@ -187,3 +214,11 @@ pub unsafe trait Allocator { let _ = unsafe { Self::realloc(Some(ptr), Layout::new::<()>(), layout, Flags(0)) }; } } + +/// Returns a properly aligned dangling pointer from the given `layout`. +pub(crate) fn dangling_from_layout(layout: Layout) -> NonNull { + let ptr = layout.align() as *mut u8; + + // SAFETY: `layout.align()` (and hence `ptr`) is guaranteed to be non-zero. + unsafe { NonNull::new_unchecked(ptr) } +} diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 3b1c735ba409fb..067719f11d131c 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -1,12 +1,47 @@ // SPDX-License-Identifier: GPL-2.0 //! Allocator support. +//! +//! Documentation for the kernel's memory allocators can found in the "Memory Allocation Guide" +//! linked below. For instance, this includes the concept of "get free page" (GFP) flags and the +//! typical application of the different kernel allocators. +//! +//! Reference: -use super::{flags::*, Flags}; -use core::alloc::{GlobalAlloc, Layout}; +use super::Flags; +use core::alloc::Layout; use core::ptr; +use core::ptr::NonNull; -struct Kmalloc; +use crate::alloc::{AllocError, Allocator}; +use crate::bindings; +use crate::pr_warn; + +/// The contiguous kernel allocator. +/// +/// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also +/// supports larger allocations up to `bindings::KMALLOC_MAX_SIZE`, which is hardware specific. +/// +/// For more details see [self]. +pub struct Kmalloc; + +/// The virtually contiguous kernel allocator. +/// +/// `Vmalloc` allocates pages from the page level allocator and maps them into the contiguous kernel +/// virtual space. It is typically used for large allocations. The memory allocated with this +/// allocator is not physically contiguous. +/// +/// For more details see [self]. +pub struct Vmalloc; + +/// The kvmalloc kernel allocator. +/// +/// `KVmalloc` attempts to allocate memory with `Kmalloc` first, but falls back to `Vmalloc` upon +/// failure. This allocator is typically used when the size for the requested allocation is not +/// known and may exceed the capabilities of `Kmalloc`. +/// +/// For more details see [self]. +pub struct KVmalloc; /// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment. fn aligned_size(new_layout: Layout) -> usize { @@ -19,63 +54,135 @@ fn aligned_size(new_layout: Layout) -> usize { layout.size() } -/// Calls `krealloc` with a proper size to alloc a new object aligned to `new_layout`'s alignment. +/// # Invariants /// -/// # Safety -/// -/// - `ptr` can be either null or a pointer which has been allocated by this allocator. -/// - `new_layout` must have a non-zero size. -pub(crate) unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: Flags) -> *mut u8 { - let size = aligned_size(new_layout); - - // SAFETY: - // - `ptr` is either null or a pointer returned from a previous `k{re}alloc()` by the - // function safety requirement. - // - `size` is greater than 0 since it's from `layout.size()` (which cannot be zero according - // to the function safety requirement) - unsafe { bindings::krealloc(ptr as *const core::ffi::c_void, size, flags.0) as *mut u8 } -} +/// One of the following `krealloc`, `vrealloc`, `kvrealloc`. +struct ReallocFunc( + unsafe extern "C" fn(*const core::ffi::c_void, usize, u32) -> *mut core::ffi::c_void, +); -// SAFETY: TODO. -unsafe impl GlobalAlloc for Kmalloc { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety - // requirement. - unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL) } - } +impl ReallocFunc { + // INVARIANT: `krealloc` satisfies the type invariants. + const KREALLOC: Self = Self(bindings::krealloc); - unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { - // SAFETY: TODO. - unsafe { - bindings::kfree(ptr as *const core::ffi::c_void); - } - } + // INVARIANT: `vrealloc` satisfies the type invariants. + const VREALLOC: Self = Self(bindings::vrealloc); - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - // SAFETY: - // - `new_size`, when rounded up to the nearest multiple of `layout.align()`, will not - // overflow `isize` by the function safety requirement. - // - `layout.align()` is a proper alignment (i.e. not zero and must be a power of two). - let layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + // INVARIANT: `kvrealloc` satisfies the type invariants. + const KVREALLOC: Self = Self(bindings::kvrealloc); + + /// # Safety + /// + /// This method has the same safety requirements as [`Allocator::realloc`]. + /// + /// # Guarantees + /// + /// This method has the same guarantees as `Allocator::realloc`. Additionally + /// - it accepts any pointer to a valid memory allocation allocated by this function. + /// - memory allocated by this function remains valid until it is passed to this function. + unsafe fn call( + &self, + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError> { + let size = aligned_size(layout); + let ptr = match ptr { + Some(ptr) => { + if old_layout.size() == 0 { + ptr::null() + } else { + ptr.as_ptr() + } + } + None => ptr::null(), + }; // SAFETY: - // - `ptr` is either null or a pointer allocated by this allocator by the function safety - // requirement. - // - the size of `layout` is not zero because `new_size` is not zero by the function safety - // requirement. - unsafe { krealloc_aligned(ptr, layout, GFP_KERNEL) } + // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that + // `ptr` is NULL or valid. + // - `ptr` is either NULL or valid by the safety requirements of this function. + // + // GUARANTEE: + // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc`. + // - Those functions provide the guarantees of this function. + let raw_ptr = unsafe { + // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. + self.0(ptr.cast(), size, flags.0).cast() + }; + + let ptr = if size == 0 { + crate::alloc::dangling_from_layout(layout) + } else { + NonNull::new(raw_ptr).ok_or(AllocError)? + }; + + Ok(NonNull::slice_from_raw_parts(ptr, size)) } +} - unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety - // requirement. - unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL | __GFP_ZERO) } +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for Kmalloc { + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError> { + // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`. + unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags) } } } -#[global_allocator] -static ALLOCATOR: Kmalloc = Kmalloc; +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for Vmalloc { + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError> { + // TODO: Support alignments larger than PAGE_SIZE. + if layout.align() > bindings::PAGE_SIZE { + pr_warn!("Vmalloc does not support alignments larger than PAGE_SIZE yet.\n"); + return Err(AllocError); + } -// See . -#[no_mangle] -static __rust_no_alloc_shim_is_unstable: u8 = 0; + // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously + // allocated with this `Allocator`. + unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags) } + } +} + +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for KVmalloc { + #[inline] + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError> { + // TODO: Support alignments larger than PAGE_SIZE. + if layout.align() > bindings::PAGE_SIZE { + pr_warn!("KVmalloc does not support alignments larger than PAGE_SIZE yet.\n"); + return Err(AllocError); + } + + // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously + // allocated with this `Allocator`. + unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags) } + } +} diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs new file mode 100644 index 00000000000000..20b7dcb20376a9 --- /dev/null +++ b/rust/kernel/alloc/allocator_test.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! So far the kernel's `Box` and `Vec` types can't be used by userspace test cases, since all users +//! of those types (e.g. `CString`) use kernel allocators for instantiation. +//! +//! In order to allow userspace test cases to make use of such types as well, implement the +//! `Cmalloc` allocator within the allocator_test module and type alias all kernel allocators to +//! `Cmalloc`. The `Cmalloc` allocator uses libc's realloc() function as allocator backend. + +#![allow(missing_docs)] + +use super::{flags::*, AllocError, Allocator, Flags}; +use core::alloc::Layout; +use core::cmp; +use core::ptr; +use core::ptr::NonNull; + +/// The userspace allocator based on libc. +pub struct Cmalloc; + +pub type Kmalloc = Cmalloc; +pub type Vmalloc = Kmalloc; +pub type KVmalloc = Kmalloc; + +extern "C" { + #[link_name = "aligned_alloc"] + fn libc_aligned_alloc(align: usize, size: usize) -> *mut core::ffi::c_void; + + #[link_name = "free"] + fn libc_free(ptr: *mut core::ffi::c_void); +} + +// SAFETY: +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation created by this `Allocator` is always OK, +// - `realloc` provides the guarantees as provided in the `# Guarantees` section. +unsafe impl Allocator for Cmalloc { + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError> { + let src = match ptr { + Some(src) => { + if old_layout.size() == 0 { + ptr::null_mut() + } else { + src.as_ptr() + } + } + None => ptr::null_mut(), + }; + + if layout.size() == 0 { + // SAFETY: `src` is either NULL or was previously allocated with this `Allocator` + unsafe { libc_free(src.cast()) }; + + return Ok(NonNull::slice_from_raw_parts( + crate::alloc::dangling_from_layout(layout), + 0, + )); + } + + // SAFETY: Returns either NULL or a pointer to a memory allocation that satisfies or + // exceeds the given size and alignment requirements. + let dst = unsafe { libc_aligned_alloc(layout.align(), layout.size()) } as *mut u8; + let dst = NonNull::new(dst).ok_or(AllocError)?; + + if flags.contains(__GFP_ZERO) { + // SAFETY: The preceding calls to `libc_aligned_alloc` and `NonNull::new` + // guarantee that `dst` points to memory of at least `layout.size()` bytes. + unsafe { dst.as_ptr().write_bytes(0, layout.size()) }; + } + + if !src.is_null() { + // SAFETY: + // - `src` has previously been allocated with this `Allocator`; `dst` has just been + // newly allocated, hence the memory regions do not overlap. + // - both` src` and `dst` are properly aligned and valid for reads and writes + unsafe { + ptr::copy_nonoverlapping( + src, + dst.as_ptr(), + cmp::min(layout.size(), old_layout.size()), + ) + }; + } + + // SAFETY: `src` is either NULL or was previously allocated with this `Allocator` + unsafe { libc_free(src.cast()) }; + + Ok(NonNull::slice_from_raw_parts(dst, layout.size())) + } +} diff --git a/rust/kernel/alloc/box_ext.rs b/rust/kernel/alloc/box_ext.rs deleted file mode 100644 index 7009ad78d4e082..00000000000000 --- a/rust/kernel/alloc/box_ext.rs +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! Extensions to [`Box`] for fallible allocations. - -use super::{AllocError, Flags}; -use alloc::boxed::Box; -use core::{mem::MaybeUninit, ptr, result::Result}; - -/// Extensions to [`Box`]. -pub trait BoxExt: Sized { - /// Allocates a new box. - /// - /// The allocation may fail, in which case an error is returned. - fn new(x: T, flags: Flags) -> Result; - - /// Allocates a new uninitialised box. - /// - /// The allocation may fail, in which case an error is returned. - fn new_uninit(flags: Flags) -> Result>, AllocError>; - - /// Drops the contents, but keeps the allocation. - /// - /// # Examples - /// - /// ``` - /// use kernel::alloc::{flags, box_ext::BoxExt}; - /// let value = Box::new([0; 32], flags::GFP_KERNEL)?; - /// assert_eq!(*value, [0; 32]); - /// let mut value = Box::drop_contents(value); - /// // Now we can re-use `value`: - /// value.write([1; 32]); - /// // SAFETY: We just wrote to it. - /// let value = unsafe { value.assume_init() }; - /// assert_eq!(*value, [1; 32]); - /// # Ok::<(), Error>(()) - /// ``` - fn drop_contents(this: Self) -> Box>; -} - -impl BoxExt for Box { - fn new(x: T, flags: Flags) -> Result { - let mut b = >::new_uninit(flags)?; - b.write(x); - // SAFETY: We just wrote to it. - Ok(unsafe { b.assume_init() }) - } - - #[cfg(any(test, testlib))] - fn new_uninit(_flags: Flags) -> Result>, AllocError> { - Ok(Box::new_uninit()) - } - - #[cfg(not(any(test, testlib)))] - fn new_uninit(flags: Flags) -> Result>, AllocError> { - let ptr = if core::mem::size_of::>() == 0 { - core::ptr::NonNull::<_>::dangling().as_ptr() - } else { - let layout = core::alloc::Layout::new::>(); - - // SAFETY: Memory is being allocated (first arg is null). The only other source of - // safety issues is sleeping on atomic context, which is addressed by klint. Lastly, - // the type is not a SZT (checked above). - let ptr = - unsafe { super::allocator::krealloc_aligned(core::ptr::null_mut(), layout, flags) }; - if ptr.is_null() { - return Err(AllocError); - } - - ptr.cast::>() - }; - - // SAFETY: For non-zero-sized types, we allocate above using the global allocator. For - // zero-sized types, we use `NonNull::dangling`. - Ok(unsafe { Box::from_raw(ptr) }) - } - - fn drop_contents(this: Self) -> Box> { - let ptr = Box::into_raw(this); - // SAFETY: `ptr` is valid, because it came from `Box::into_raw`. - unsafe { ptr::drop_in_place(ptr) }; - - // CAST: `MaybeUninit` is a transparent wrapper of `T`. - let ptr = ptr.cast::>(); - - // SAFETY: `ptr` is valid for writes, because it came from `Box::into_raw` and it is valid for - // reads, since the pointer came from `Box::into_raw` and the type is `MaybeUninit`. - unsafe { Box::from_raw(ptr) } - } -} diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs new file mode 100644 index 00000000000000..d69c32496b86a2 --- /dev/null +++ b/rust/kernel/alloc/kbox.rs @@ -0,0 +1,456 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of [`Box`]. + +#[allow(unused_imports)] // Used in doc comments. +use super::allocator::{KVmalloc, Kmalloc, Vmalloc}; +use super::{AllocError, Allocator, Flags}; +use core::alloc::Layout; +use core::fmt; +use core::marker::PhantomData; +use core::mem::ManuallyDrop; +use core::mem::MaybeUninit; +use core::ops::{Deref, DerefMut}; +use core::pin::Pin; +use core::ptr::NonNull; +use core::result::Result; + +use crate::init::{InPlaceInit, InPlaceWrite, Init, PinInit}; +use crate::types::ForeignOwnable; + +/// The kernel's [`Box`] type -- a heap allocation for a single value of type `T`. +/// +/// This is the kernel's version of the Rust stdlib's `Box`. There are several differences, +/// for example no `noalias` attribute is emitted and partially moving out of a `Box` is not +/// supported. There are also several API differences, e.g. `Box` always requires an [`Allocator`] +/// implementation to be passed as generic, page [`Flags`] when allocating memory and all functions +/// that may allocate memory are fallible. +/// +/// `Box` works with any of the kernel's allocators, e.g. [`Kmalloc`], [`Vmalloc`] or [`KVmalloc`]. +/// There are aliases for `Box` with these allocators ([`KBox`], [`VBox`], [`KVBox`]). +/// +/// When dropping a [`Box`], the value is also dropped and the heap memory is automatically freed. +/// +/// # Examples +/// +/// ``` +/// let b = KBox::::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// ``` +/// # use kernel::bindings; +/// const SIZE: usize = bindings::KMALLOC_MAX_SIZE as usize + 1; +/// struct Huge([u8; SIZE]); +/// +/// assert!(KBox::::new_uninit(GFP_KERNEL | __GFP_NOWARN).is_err()); +/// ``` +/// +/// ``` +/// # use kernel::bindings; +/// const SIZE: usize = bindings::KMALLOC_MAX_SIZE as usize + 1; +/// struct Huge([u8; SIZE]); +/// +/// assert!(KVBox::::new_uninit(GFP_KERNEL).is_ok()); +/// ``` +/// +/// # Invariants +/// +/// `self.0` is always properly aligned and either points to memory allocated with `A` or, for +/// zero-sized types, is a dangling, well aligned pointer. +#[repr(transparent)] +pub struct Box(NonNull, PhantomData); + +/// Type alias for [`Box`] with a [`Kmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = KBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type KBox = Box; + +/// Type alias for [`Box`] with a [`Vmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = VBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type VBox = Box; + +/// Type alias for [`Box`] with a [`KVmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = KVBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type KVBox = Box; + +// SAFETY: `Box` is `Send` if `T` is `Send` because the `Box` owns a `T`. +unsafe impl Send for Box +where + T: Send + ?Sized, + A: Allocator, +{ +} + +// SAFETY: `Box` is `Sync` if `T` is `Sync` because the `Box` owns a `T`. +unsafe impl Sync for Box +where + T: Sync + ?Sized, + A: Allocator, +{ +} + +impl Box +where + T: ?Sized, + A: Allocator, +{ + /// Creates a new `Box` from a raw pointer. + /// + /// # Safety + /// + /// For non-ZSTs, `raw` must point at an allocation allocated with `A` that is sufficiently + /// aligned for and holds a valid `T`. The caller passes ownership of the allocation to the + /// `Box`. + /// + /// For ZSTs, `raw` must be a dangling, well aligned pointer. + #[inline] + pub const unsafe fn from_raw(raw: *mut T) -> Self { + // INVARIANT: Validity of `raw` is guaranteed by the safety preconditions of this function. + // SAFETY: By the safety preconditions of this function, `raw` is not a NULL pointer. + Self(unsafe { NonNull::new_unchecked(raw) }, PhantomData) + } + + /// Consumes the `Box` and returns a raw pointer. + /// + /// This will not run the destructor of `T` and for non-ZSTs the allocation will stay alive + /// indefinitely. Use [`Box::from_raw`] to recover the [`Box`], drop the value and free the + /// allocation, if any. + /// + /// # Examples + /// + /// ``` + /// let x = KBox::new(24, GFP_KERNEL)?; + /// let ptr = KBox::into_raw(x); + /// // SAFETY: `ptr` comes from a previous call to `KBox::into_raw`. + /// let x = unsafe { KBox::from_raw(ptr) }; + /// + /// assert_eq!(*x, 24); + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + pub fn into_raw(b: Self) -> *mut T { + ManuallyDrop::new(b).0.as_ptr() + } + + /// Consumes and leaks the `Box` and returns a mutable reference. + /// + /// See [`Box::into_raw`] for more details. + #[inline] + pub fn leak<'a>(b: Self) -> &'a mut T { + // SAFETY: `Box::into_raw` always returns a properly aligned and dereferenceable pointer + // which points to an initialized instance of `T`. + unsafe { &mut *Box::into_raw(b) } + } +} + +impl Box, A> +where + A: Allocator, +{ + /// Converts a `Box, A>` to a `Box`. + /// + /// It is undefined behavior to call this function while the value inside of `b` is not yet + /// fully initialized. + /// + /// # Safety + /// + /// Callers must ensure that the value inside of `b` is in an initialized state. + pub unsafe fn assume_init(self) -> Box { + let raw = Self::into_raw(self); + + // SAFETY: `raw` comes from a previous call to `Box::into_raw`. By the safety requirements + // of this function, the value inside the `Box` is in an initialized state. Hence, it is + // safe to reconstruct the `Box` as `Box`. + unsafe { Box::from_raw(raw.cast()) } + } + + /// Writes the value and converts to `Box`. + pub fn write(mut self, value: T) -> Box { + (*self).write(value); + + // SAFETY: We've just initialized `b`'s value. + unsafe { self.assume_init() } + } +} + +impl Box +where + A: Allocator, +{ + /// Creates a new `Box` and initializes its contents with `x`. + /// + /// New memory is allocated with `A`. The allocation may fail, in which case an error is + /// returned. For ZSTs no memory is allocated. + pub fn new(x: T, flags: Flags) -> Result { + let b = Self::new_uninit(flags)?; + Ok(Box::write(b, x)) + } + + /// Creates a new `Box` with uninitialized contents. + /// + /// New memory is allocated with `A`. The allocation may fail, in which case an error is + /// returned. For ZSTs no memory is allocated. + /// + /// # Examples + /// + /// ``` + /// let b = KBox::::new_uninit(GFP_KERNEL)?; + /// let b = KBox::write(b, 24); + /// + /// assert_eq!(*b, 24_u64); + /// # Ok::<(), Error>(()) + /// ``` + pub fn new_uninit(flags: Flags) -> Result, A>, AllocError> { + let layout = Layout::new::>(); + let ptr = A::alloc(layout, flags)?; + + // INVARIANT: `ptr` is either a dangling pointer or points to memory allocated with `A`, + // which is sufficient in size and alignment for storing a `T`. + Ok(Box(ptr.cast(), PhantomData)) + } + + /// Constructs a new `Pin>`. If `T` does not implement [`Unpin`], then `x` will be + /// pinned in memory and can't be moved. + #[inline] + pub fn pin(x: T, flags: Flags) -> Result>, AllocError> + where + A: 'static, + { + Ok(Self::new(x, flags)?.into()) + } + + /// Forgets the contents (does not run the destructor), but keeps the allocation. + fn forget_contents(this: Self) -> Box, A> { + let ptr = Self::into_raw(this); + + // SAFETY: `ptr` is valid, because it came from `Box::into_raw`. + unsafe { Box::from_raw(ptr.cast()) } + } + + /// Drops the contents, but keeps the allocation. + /// + /// # Examples + /// + /// ``` + /// let value = KBox::new([0; 32], GFP_KERNEL)?; + /// assert_eq!(*value, [0; 32]); + /// let value = KBox::drop_contents(value); + /// // Now we can re-use `value`: + /// let value = KBox::write(value, [1; 32]); + /// assert_eq!(*value, [1; 32]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn drop_contents(this: Self) -> Box, A> { + let ptr = this.0.as_ptr(); + + // SAFETY: `ptr` is valid, because it came from `this`. After this call we never access the + // value stored in `this` again. + unsafe { core::ptr::drop_in_place(ptr) }; + + Self::forget_contents(this) + } + + /// Moves the `Box`'s value out of the `Box` and consumes the `Box`. + pub fn into_inner(b: Self) -> T { + // SAFETY: By the type invariant `&*b` is valid for `read`. + let value = unsafe { core::ptr::read(&*b) }; + let _ = Self::forget_contents(b); + value + } +} + +impl From> for Pin> +where + T: ?Sized, + A: Allocator, +{ + /// Converts a `Box` into a `Pin>`. If `T` does not implement [`Unpin`], then + /// `*b` will be pinned in memory and can't be moved. + /// + /// This moves `b` into `Pin` without moving `*b` or allocating and copying any memory. + fn from(b: Box) -> Self { + // SAFETY: The value wrapped inside a `Pin>` cannot be moved or replaced as long + // as `T` does not implement `Unpin`. + unsafe { Pin::new_unchecked(b) } + } +} + +impl InPlaceWrite for Box, A> +where + A: Allocator + 'static, +{ + type Initialized = Box; + + fn write_init(mut self, init: impl Init) -> Result { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { Box::assume_init(self) }) + } + + fn write_pin_init(mut self, init: impl PinInit) -> Result, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__pinned_init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { Box::assume_init(self) }.into()) + } +} + +impl InPlaceInit for Box +where + A: Allocator + 'static, +{ + type PinnedSelf = Pin; + + #[inline] + fn try_pin_init(init: impl PinInit, flags: Flags) -> Result, E> + where + E: From, + { + Box::<_, A>::new_uninit(flags)?.write_pin_init(init) + } + + #[inline] + fn try_init(init: impl Init, flags: Flags) -> Result + where + E: From, + { + Box::<_, A>::new_uninit(flags)?.write_init(init) + } +} + +impl ForeignOwnable for Box +where + A: Allocator, +{ + type Borrowed<'a> = &'a T; + + fn into_foreign(self) -> *const core::ffi::c_void { + Box::into_raw(self) as _ + } + + unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + unsafe { Box::from_raw(ptr as _) } + } + + unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> &'a T { + // SAFETY: The safety requirements of this method ensure that the object remains alive and + // immutable for the duration of 'a. + unsafe { &*ptr.cast() } + } +} + +impl ForeignOwnable for Pin> +where + A: Allocator, +{ + type Borrowed<'a> = Pin<&'a T>; + + fn into_foreign(self) -> *const core::ffi::c_void { + // SAFETY: We are still treating the box as pinned. + Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }) as _ + } + + unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + unsafe { Pin::new_unchecked(Box::from_raw(ptr as _)) } + } + + unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> Pin<&'a T> { + // SAFETY: The safety requirements for this function ensure that the object is still alive, + // so it is safe to dereference the raw pointer. + // The safety requirements of `from_foreign` also ensure that the object remains alive for + // the lifetime of the returned value. + let r = unsafe { &*ptr.cast() }; + + // SAFETY: This pointer originates from a `Pin>`. + unsafe { Pin::new_unchecked(r) } + } +} + +impl Deref for Box +where + T: ?Sized, + A: Allocator, +{ + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: `self.0` is always properly aligned, dereferenceable and points to an initialized + // instance of `T`. + unsafe { self.0.as_ref() } + } +} + +impl DerefMut for Box +where + T: ?Sized, + A: Allocator, +{ + fn deref_mut(&mut self) -> &mut T { + // SAFETY: `self.0` is always properly aligned, dereferenceable and points to an initialized + // instance of `T`. + unsafe { self.0.as_mut() } + } +} + +impl fmt::Debug for Box +where + T: ?Sized + fmt::Debug, + A: Allocator, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl Drop for Box +where + T: ?Sized, + A: Allocator, +{ + fn drop(&mut self) { + let layout = Layout::for_value::(self); + + // SAFETY: The pointer in `self.0` is guaranteed to be valid by the type invariant. + unsafe { core::ptr::drop_in_place::(self.deref_mut()) }; + + // SAFETY: + // - `self.0` was previously allocated with `A`. + // - `layout` is equal to the `Layout´ `self.0` was allocated with. + unsafe { A::free(self.0.cast(), layout) }; + } +} diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs new file mode 100644 index 00000000000000..fb1b2bb4cb360b --- /dev/null +++ b/rust/kernel/alloc/kvec.rs @@ -0,0 +1,914 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of [`Vec`]. + +use super::{ + allocator::{KVmalloc, Kmalloc, Vmalloc}, + layout::ArrayLayout, + AllocError, Allocator, Box, Flags, +}; +use core::{ + fmt, + marker::PhantomData, + mem::{ManuallyDrop, MaybeUninit}, + ops::Deref, + ops::DerefMut, + ops::Index, + ops::IndexMut, + ptr, + ptr::NonNull, + slice, + slice::SliceIndex, +}; + +/// Create a [`KVec`] containing the arguments. +/// +/// New memory is allocated with `GFP_KERNEL`. +/// +/// # Examples +/// +/// ``` +/// let mut v = kernel::kvec![]; +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(v, [1]); +/// +/// let mut v = kernel::kvec![1; 3]?; +/// v.push(4, GFP_KERNEL)?; +/// assert_eq!(v, [1, 1, 1, 4]); +/// +/// let mut v = kernel::kvec![1, 2, 3]?; +/// v.push(4, GFP_KERNEL)?; +/// assert_eq!(v, [1, 2, 3, 4]); +/// +/// # Ok::<(), Error>(()) +/// ``` +#[macro_export] +macro_rules! kvec { + () => ( + $crate::alloc::KVec::new() + ); + ($elem:expr; $n:expr) => ( + $crate::alloc::KVec::from_elem($elem, $n, GFP_KERNEL) + ); + ($($x:expr),+ $(,)?) => ( + match $crate::alloc::KBox::new_uninit(GFP_KERNEL) { + Ok(b) => Ok($crate::alloc::KVec::from($crate::alloc::KBox::write(b, [$($x),+]))), + Err(e) => Err(e), + } + ); +} + +/// The kernel's [`Vec`] type. +/// +/// A contiguous growable array type with contents allocated with the kernel's allocators (e.g. +/// [`Kmalloc`], [`Vmalloc`] or [`KVmalloc`]), written `Vec`. +/// +/// For non-zero-sized values, a [`Vec`] will use the given allocator `A` for its allocation. For +/// the most common allocators the type aliases [`KVec`], [`VVec`] and [`KVVec`] exist. +/// +/// For zero-sized types the [`Vec`]'s pointer must be `dangling_mut::`; no memory is allocated. +/// +/// Generally, [`Vec`] consists of a pointer that represents the vector's backing buffer, the +/// capacity of the vector (the number of elements that currently fit into the vector), its length +/// (the number of elements that are currently stored in the vector) and the `Allocator` type used +/// to allocate (and free) the backing buffer. +/// +/// A [`Vec`] can be deconstructed into and (re-)constructed from its previously named raw parts +/// and manually modified. +/// +/// [`Vec`]'s backing buffer gets, if required, automatically increased (re-allocated) when elements +/// are added to the vector. +/// +/// # Invariants +/// +/// - `self.ptr` is always properly aligned and either points to memory allocated with `A` or, for +/// zero-sized types, is a dangling, well aligned pointer. +/// +/// - `self.len` always represents the exact number of elements stored in the vector. +/// +/// - `self.layout` represents the absolute number of elements that can be stored within the vector +/// without re-allocation. For ZSTs `self.layout`'s capacity is zero. However, it is legal for the +/// backing buffer to be larger than `layout`. +/// +/// - The `Allocator` type `A` of the vector is the exact same `Allocator` type the backing buffer +/// was allocated with (and must be freed with). +pub struct Vec { + ptr: NonNull, + /// Represents the actual buffer size as `cap` times `size_of::` bytes. + /// + /// Note: This isn't quite the same as `Self::capacity`, which in contrast returns the number of + /// elements we can still store without reallocating. + layout: ArrayLayout, + len: usize, + _p: PhantomData, +} + +/// Type alias for [`Vec`] with a [`Kmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = KVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type KVec = Vec; + +/// Type alias for [`Vec`] with a [`Vmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = VVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type VVec = Vec; + +/// Type alias for [`Vec`] with a [`KVmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = KVVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type KVVec = Vec; + +// SAFETY: `Vec` is `Send` if `T` is `Send` because `Vec` owns its elements. +unsafe impl Send for Vec +where + T: Send, + A: Allocator, +{ +} + +// SAFETY: `Vec` is `Sync` if `T` is `Sync` because `Vec` owns its elements. +unsafe impl Sync for Vec +where + T: Sync, + A: Allocator, +{ +} + +impl Vec +where + A: Allocator, +{ + #[inline] + const fn is_zst() -> bool { + core::mem::size_of::() == 0 + } + + /// Returns the number of elements that can be stored within the vector without allocating + /// additional memory. + pub fn capacity(&self) -> usize { + if const { Self::is_zst() } { + usize::MAX + } else { + self.layout.len() + } + } + + /// Returns the number of elements stored within the vector. + #[inline] + pub fn len(&self) -> usize { + self.len + } + + /// Forcefully sets `self.len` to `new_len`. + /// + /// # Safety + /// + /// - `new_len` must be less than or equal to [`Self::capacity`]. + /// - If `new_len` is greater than `self.len`, all elements within the interval + /// [`self.len`,`new_len`) must be initialized. + #[inline] + pub unsafe fn set_len(&mut self, new_len: usize) { + debug_assert!(new_len <= self.capacity()); + self.len = new_len; + } + + /// Returns a slice of the entire vector. + #[inline] + pub fn as_slice(&self) -> &[T] { + self + } + + /// Returns a mutable slice of the entire vector. + #[inline] + pub fn as_mut_slice(&mut self) -> &mut [T] { + self + } + + /// Returns a mutable raw pointer to the vector's backing buffer, or, if `T` is a ZST, a + /// dangling raw pointer. + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut T { + self.ptr.as_ptr() + } + + /// Returns a raw pointer to the vector's backing buffer, or, if `T` is a ZST, a dangling raw + /// pointer. + #[inline] + pub fn as_ptr(&self) -> *const T { + self.ptr.as_ptr() + } + + /// Returns `true` if the vector contains no elements, `false` otherwise. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// assert!(v.is_empty()); + /// + /// v.push(1, GFP_KERNEL); + /// assert!(!v.is_empty()); + /// ``` + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Creates a new, empty `Vec`. + /// + /// This method does not allocate by itself. + #[inline] + pub const fn new() -> Self { + // INVARIANT: Since this is a new, empty `Vec` with no backing memory yet, + // - `ptr` is a properly aligned dangling pointer for type `T`, + // - `layout` is an empty `ArrayLayout` (zero capacity) + // - `len` is zero, since no elements can be or have been stored, + // - `A` is always valid. + Self { + ptr: NonNull::dangling(), + layout: ArrayLayout::empty(), + len: 0, + _p: PhantomData::, + } + } + + /// Returns a slice of `MaybeUninit` for the remaining spare capacity of the vector. + pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit] { + // SAFETY: + // - `self.len` is smaller than `self.capacity` and hence, the resulting pointer is + // guaranteed to be part of the same allocated object. + // - `self.len` can not overflow `isize`. + let ptr = unsafe { self.as_mut_ptr().add(self.len) } as *mut MaybeUninit; + + // SAFETY: The memory between `self.len` and `self.capacity` is guaranteed to be allocated + // and valid, but uninitialized. + unsafe { slice::from_raw_parts_mut(ptr, self.capacity() - self.len) } + } + + /// Appends an element to the back of the [`Vec`] instance. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// assert_eq!(&v, &[1]); + /// + /// v.push(2, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 2]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError> { + self.reserve(1, flags)?; + + // SAFETY: + // - `self.len` is smaller than `self.capacity` and hence, the resulting pointer is + // guaranteed to be part of the same allocated object. + // - `self.len` can not overflow `isize`. + let ptr = unsafe { self.as_mut_ptr().add(self.len) }; + + // SAFETY: + // - `ptr` is properly aligned and valid for writes. + unsafe { core::ptr::write(ptr, v) }; + + // SAFETY: We just initialised the first spare entry, so it is safe to increase the length + // by 1. We also know that the new length is <= capacity because of the previous call to + // `reserve` above. + unsafe { self.set_len(self.len() + 1) }; + Ok(()) + } + + /// Creates a new [`Vec`] instance with at least the given capacity. + /// + /// # Examples + /// + /// ``` + /// let v = KVec::::with_capacity(20, GFP_KERNEL)?; + /// + /// assert!(v.capacity() >= 20); + /// # Ok::<(), Error>(()) + /// ``` + pub fn with_capacity(capacity: usize, flags: Flags) -> Result { + let mut v = Vec::new(); + + v.reserve(capacity, flags)?; + + Ok(v) + } + + /// Creates a `Vec` from a pointer, a length and a capacity using the allocator `A`. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3]?; + /// v.reserve(1, GFP_KERNEL)?; + /// + /// let (mut ptr, mut len, cap) = v.into_raw_parts(); + /// + /// // SAFETY: We've just reserved memory for another element. + /// unsafe { ptr.add(len).write(4) }; + /// len += 1; + /// + /// // SAFETY: We only wrote an additional element at the end of the `KVec`'s buffer and + /// // correspondingly increased the length of the `KVec` by one. Otherwise, we construct it + /// // from the exact same raw parts. + /// let v = unsafe { KVec::from_raw_parts(ptr, len, cap) }; + /// + /// assert_eq!(v, [1, 2, 3, 4]); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// # Safety + /// + /// If `T` is a ZST: + /// + /// - `ptr` must be a dangling, well aligned pointer. + /// + /// Otherwise: + /// + /// - `ptr` must have been allocated with the allocator `A`. + /// - `ptr` must satisfy or exceed the alignment requirements of `T`. + /// - `ptr` must point to memory with a size of at least `size_of::() * capacity`. + /// bytes. + /// - The allocated size in bytes must not be larger than `isize::MAX`. + /// - `length` must be less than or equal to `capacity`. + /// - The first `length` elements must be initialized values of type `T`. + /// + /// It is also valid to create an empty `Vec` passing a dangling pointer for `ptr` and zero for + /// `cap` and `len`. + pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> Self { + let layout = if Self::is_zst() { + ArrayLayout::empty() + } else { + // SAFETY: By the safety requirements of this function, `capacity * size_of::()` is + // smaller than `isize::MAX`. + unsafe { ArrayLayout::new_unchecked(capacity) } + }; + + // INVARIANT: For ZSTs, we store an empty `ArrayLayout`, all other type invariants are + // covered by the safety requirements of this function. + Self { + // SAFETY: By the safety requirements, `ptr` is either dangling or pointing to a valid + // memory allocation, allocated with `A`. + ptr: unsafe { NonNull::new_unchecked(ptr) }, + layout, + len: length, + _p: PhantomData::, + } + } + + /// Consumes the `Vec` and returns its raw components `pointer`, `length` and `capacity`. + /// + /// This will not run the destructor of the contained elements and for non-ZSTs the allocation + /// will stay alive indefinitely. Use [`Vec::from_raw_parts`] to recover the [`Vec`], drop the + /// elements and free the allocation, if any. + pub fn into_raw_parts(self) -> (*mut T, usize, usize) { + let mut me = ManuallyDrop::new(self); + let len = me.len(); + let capacity = me.capacity(); + let ptr = me.as_mut_ptr(); + (ptr, len, capacity) + } + + /// Ensures that the capacity exceeds the length by at least `additional` elements. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.reserve(10, GFP_KERNEL)?; + /// let cap = v.capacity(); + /// assert!(cap >= 10); + /// + /// v.reserve(10, GFP_KERNEL)?; + /// let new_cap = v.capacity(); + /// assert_eq!(new_cap, cap); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError> { + let len = self.len(); + let cap = self.capacity(); + + if cap - len >= additional { + return Ok(()); + } + + if Self::is_zst() { + // The capacity is already `usize::MAX` for ZSTs, we can't go higher. + return Err(AllocError); + } + + // We know that `cap <= isize::MAX` because of the type invariants of `Self`. So the + // multiplication by two won't overflow. + let new_cap = core::cmp::max(cap * 2, len.checked_add(additional).ok_or(AllocError)?); + let layout = ArrayLayout::new(new_cap).map_err(|_| AllocError)?; + + // SAFETY: + // - `ptr` is valid because it's either `None` or comes from a previous call to + // `A::realloc`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + let ptr = unsafe { + A::realloc( + Some(self.ptr.cast()), + layout.into(), + self.layout.into(), + flags, + )? + }; + + // INVARIANT: + // - `layout` is some `ArrayLayout::`, + // - `ptr` has been created by `A::realloc` from `layout`. + self.ptr = ptr.cast(); + self.layout = layout; + + Ok(()) + } +} + +impl Vec { + /// Extend the vector by `n` clones of `value`. + pub fn extend_with(&mut self, n: usize, value: T, flags: Flags) -> Result<(), AllocError> { + if n == 0 { + return Ok(()); + } + + self.reserve(n, flags)?; + + let spare = self.spare_capacity_mut(); + + for item in spare.iter_mut().take(n - 1) { + item.write(value.clone()); + } + + // We can write the last element directly without cloning needlessly. + spare[n - 1].write(value); + + // SAFETY: + // - `self.len() + n < self.capacity()` due to the call to reserve above, + // - the loop and the line above initialized the next `n` elements. + unsafe { self.set_len(self.len() + n) }; + + Ok(()) + } + + /// Pushes clones of the elements of slice into the [`Vec`] instance. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.extend_from_slice(&[20, 30, 40], GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 20, 30, 40]); + /// + /// v.extend_from_slice(&[50, 60], GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 20, 30, 40, 50, 60]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> { + self.reserve(other.len(), flags)?; + for (slot, item) in core::iter::zip(self.spare_capacity_mut(), other) { + slot.write(item.clone()); + } + + // SAFETY: + // - `other.len()` spare entries have just been initialized, so it is safe to increase + // the length by the same number. + // - `self.len() + other.len() <= self.capacity()` is guaranteed by the preceding `reserve` + // call. + unsafe { self.set_len(self.len() + other.len()) }; + Ok(()) + } + + /// Create a new `Vec` and extend it by `n` clones of `value`. + pub fn from_elem(value: T, n: usize, flags: Flags) -> Result { + let mut v = Self::with_capacity(n, flags)?; + + v.extend_with(n, value, flags)?; + + Ok(v) + } +} + +impl Drop for Vec +where + A: Allocator, +{ + fn drop(&mut self) { + // SAFETY: `self.as_mut_ptr` is guaranteed to be valid by the type invariant. + unsafe { + ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( + self.as_mut_ptr(), + self.len, + )) + }; + + // SAFETY: + // - `self.ptr` was previously allocated with `A`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { A::free(self.ptr.cast(), self.layout.into()) }; + } +} + +impl From> for Vec +where + A: Allocator, +{ + fn from(b: Box<[T; N], A>) -> Vec { + let len = b.len(); + let ptr = Box::into_raw(b); + + // SAFETY: + // - `b` has been allocated with `A`, + // - `ptr` fulfills the alignment requirements for `T`, + // - `ptr` points to memory with at least a size of `size_of::() * len`, + // - all elements within `b` are initialized values of `T`, + // - `len` does not exceed `isize::MAX`. + unsafe { Vec::from_raw_parts(ptr as _, len, len) } + } +} + +impl Default for KVec { + #[inline] + fn default() -> Self { + Self::new() + } +} + +impl fmt::Debug for Vec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl Deref for Vec +where + A: Allocator, +{ + type Target = [T]; + + #[inline] + fn deref(&self) -> &[T] { + // SAFETY: The memory behind `self.as_ptr()` is guaranteed to contain `self.len` + // initialized elements of type `T`. + unsafe { slice::from_raw_parts(self.as_ptr(), self.len) } + } +} + +impl DerefMut for Vec +where + A: Allocator, +{ + #[inline] + fn deref_mut(&mut self) -> &mut [T] { + // SAFETY: The memory behind `self.as_ptr()` is guaranteed to contain `self.len` + // initialized elements of type `T`. + unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.len) } + } +} + +impl Eq for Vec where A: Allocator {} + +impl, A> Index for Vec +where + A: Allocator, +{ + type Output = I::Output; + + #[inline] + fn index(&self, index: I) -> &Self::Output { + Index::index(&**self, index) + } +} + +impl, A> IndexMut for Vec +where + A: Allocator, +{ + #[inline] + fn index_mut(&mut self, index: I) -> &mut Self::Output { + IndexMut::index_mut(&mut **self, index) + } +} + +macro_rules! impl_slice_eq { + ($([$($vars:tt)*] $lhs:ty, $rhs:ty,)*) => { + $( + impl PartialEq<$rhs> for $lhs + where + T: PartialEq, + { + #[inline] + fn eq(&self, other: &$rhs) -> bool { self[..] == other[..] } + } + )* + } +} + +impl_slice_eq! { + [A1: Allocator, A2: Allocator] Vec, Vec, + [A: Allocator] Vec, &[U], + [A: Allocator] Vec, &mut [U], + [A: Allocator] &[T], Vec, + [A: Allocator] &mut [T], Vec, + [A: Allocator] Vec, [U], + [A: Allocator] [T], Vec, + [A: Allocator, const N: usize] Vec, [U; N], + [A: Allocator, const N: usize] Vec, &[U; N], +} + +impl<'a, T, A> IntoIterator for &'a Vec +where + A: Allocator, +{ + type Item = &'a T; + type IntoIter = slice::Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a, T, A: Allocator> IntoIterator for &'a mut Vec +where + A: Allocator, +{ + type Item = &'a mut T; + type IntoIter = slice::IterMut<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + +/// An [`Iterator`] implementation for [`Vec`] that moves elements out of a vector. +/// +/// This structure is created by the [`Vec::into_iter`] method on [`Vec`] (provided by the +/// [`IntoIterator`] trait). +/// +/// # Examples +/// +/// ``` +/// let v = kernel::kvec![0, 1, 2]?; +/// let iter = v.into_iter(); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub struct IntoIter { + ptr: *mut T, + buf: NonNull, + len: usize, + layout: ArrayLayout, + _p: PhantomData, +} + +impl IntoIter +where + A: Allocator, +{ + fn into_raw_parts(self) -> (*mut T, NonNull, usize, usize) { + let me = ManuallyDrop::new(self); + let ptr = me.ptr; + let buf = me.buf; + let len = me.len; + let cap = me.layout.len(); + (ptr, buf, len, cap) + } + + /// Same as `Iterator::collect` but specialized for `Vec`'s `IntoIter`. + /// + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2, 3]?; + /// let mut it = v.into_iter(); + /// + /// assert_eq!(it.next(), Some(1)); + /// + /// let v = it.collect(GFP_KERNEL); + /// assert_eq!(v, [2, 3]); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// # Implementation details + /// + /// Currently, we can't implement `FromIterator`. There are a couple of issues with this trait + /// in the kernel, namely: + /// + /// - Rust's specialization feature is unstable. This prevents us to optimize for the special + /// case where `I::IntoIter` equals `Vec`'s `IntoIter` type. + /// - We also can't use `I::IntoIter`'s type ID either to work around this, since `FromIterator` + /// doesn't require this type to be `'static`. + /// - `FromIterator::from_iter` does return `Self` instead of `Result`, hence + /// we can't properly handle allocation failures. + /// - Neither `Iterator::collect` nor `FromIterator::from_iter` can handle additional allocation + /// flags. + /// + /// Instead, provide `IntoIter::collect`, such that we can at least convert a `IntoIter` into a + /// `Vec` again. + /// + /// Note that `IntoIter::collect` doesn't require `Flags`, since it re-uses the existing backing + /// buffer. However, this backing buffer may be shrunk to the actual count of elements. + pub fn collect(self, flags: Flags) -> Vec { + let old_layout = self.layout; + let (mut ptr, buf, len, mut cap) = self.into_raw_parts(); + let has_advanced = ptr != buf.as_ptr(); + + if has_advanced { + // Copy the contents we have advanced to at the beginning of the buffer. + // + // SAFETY: + // - `ptr` is valid for reads of `len * size_of::()` bytes, + // - `buf.as_ptr()` is valid for writes of `len * size_of::()` bytes, + // - `ptr` and `buf.as_ptr()` are not be subject to aliasing restrictions relative to + // each other, + // - both `ptr` and `buf.ptr()` are properly aligned. + unsafe { ptr::copy(ptr, buf.as_ptr(), len) }; + ptr = buf.as_ptr(); + + // SAFETY: `len` is guaranteed to be smaller than `self.layout.len()`. + let layout = unsafe { ArrayLayout::::new_unchecked(len) }; + + // SAFETY: `buf` points to the start of the backing buffer and `len` is guaranteed to be + // smaller than `cap`. Depending on `alloc` this operation may shrink the buffer or leaves + // it as it is. + ptr = match unsafe { + A::realloc(Some(buf.cast()), layout.into(), old_layout.into(), flags) + } { + // If we fail to shrink, which likely can't even happen, continue with the existing + // buffer. + Err(_) => ptr, + Ok(ptr) => { + cap = len; + ptr.as_ptr().cast() + } + }; + } + + // SAFETY: If the iterator has been advanced, the advanced elements have been copied to + // the beginning of the buffer and `len` has been adjusted accordingly. + // + // - `ptr` is guaranteed to point to the start of the backing buffer. + // - `cap` is either the original capacity or, after shrinking the buffer, equal to `len`. + // - `alloc` is guaranteed to be unchanged since `into_iter` has been called on the original + // `Vec`. + unsafe { Vec::from_raw_parts(ptr, len, cap) } + } +} + +impl Iterator for IntoIter +where + A: Allocator, +{ + type Item = T; + + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2, 3]?; + /// let mut it = v.into_iter(); + /// + /// assert_eq!(it.next(), Some(1)); + /// assert_eq!(it.next(), Some(2)); + /// assert_eq!(it.next(), Some(3)); + /// assert_eq!(it.next(), None); + /// + /// # Ok::<(), Error>(()) + /// ``` + fn next(&mut self) -> Option { + if self.len == 0 { + return None; + } + + let current = self.ptr; + + // SAFETY: We can't overflow; decreasing `self.len` by one every time we advance `self.ptr` + // by one guarantees that. + unsafe { self.ptr = self.ptr.add(1) }; + + self.len -= 1; + + // SAFETY: `current` is guaranteed to point at a valid element within the buffer. + Some(unsafe { current.read() }) + } + + /// # Examples + /// + /// ``` + /// let v: KVec = kernel::kvec![1, 2, 3]?; + /// let mut iter = v.into_iter(); + /// let size = iter.size_hint().0; + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 1); + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 2); + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 3); + /// + /// # Ok::<(), Error>(()) + /// ``` + fn size_hint(&self) -> (usize, Option) { + (self.len, Some(self.len)) + } +} + +impl Drop for IntoIter +where + A: Allocator, +{ + fn drop(&mut self) { + // SAFETY: `self.ptr` is guaranteed to be valid by the type invariant. + unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.ptr, self.len)) }; + + // SAFETY: + // - `self.buf` was previously allocated with `A`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { A::free(self.buf.cast(), self.layout.into()) }; + } +} + +impl IntoIterator for Vec +where + A: Allocator, +{ + type Item = T; + type IntoIter = IntoIter; + + /// Consumes the `Vec` and creates an `Iterator`, which moves each value out of the + /// vector (from start to end). + /// + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2]?; + /// let mut v_iter = v.into_iter(); + /// + /// let first_element: Option = v_iter.next(); + /// + /// assert_eq!(first_element, Some(1)); + /// assert_eq!(v_iter.next(), Some(2)); + /// assert_eq!(v_iter.next(), None); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// ``` + /// let v = kernel::kvec![]; + /// let mut v_iter = v.into_iter(); + /// + /// let first_element: Option = v_iter.next(); + /// + /// assert_eq!(first_element, None); + /// + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + fn into_iter(self) -> Self::IntoIter { + let buf = self.ptr; + let layout = self.layout; + let (ptr, len, _) = self.into_raw_parts(); + + IntoIter { + ptr, + buf, + len, + layout, + _p: PhantomData::, + } + } +} diff --git a/rust/kernel/alloc/layout.rs b/rust/kernel/alloc/layout.rs new file mode 100644 index 00000000000000..7e0c2f46157b77 --- /dev/null +++ b/rust/kernel/alloc/layout.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory layout. +//! +//! Custom layout types extending or improving [`Layout`]. + +use core::{alloc::Layout, marker::PhantomData}; + +/// Error when constructing an [`ArrayLayout`]. +pub struct LayoutError; + +/// A layout for an array `[T; n]`. +/// +/// # Invariants +/// +/// - `len * size_of::() <= isize::MAX`. +pub struct ArrayLayout { + len: usize, + _phantom: PhantomData T>, +} + +impl Clone for ArrayLayout { + fn clone(&self) -> Self { + *self + } +} +impl Copy for ArrayLayout {} + +const ISIZE_MAX: usize = isize::MAX as usize; + +impl ArrayLayout { + /// Creates a new layout for `[T; 0]`. + pub const fn empty() -> Self { + // INVARIANT: `0 * size_of::() <= isize::MAX`. + Self { + len: 0, + _phantom: PhantomData, + } + } + + /// Creates a new layout for `[T; len]`. + /// + /// # Errors + /// + /// When `len * size_of::()` overflows or when `len * size_of::() > isize::MAX`. + pub const fn new(len: usize) -> Result { + match len.checked_mul(core::mem::size_of::()) { + Some(len) if len <= ISIZE_MAX => { + // INVARIANT: We checked above that `len * size_of::() <= isize::MAX`. + Ok(Self { + len, + _phantom: PhantomData, + }) + } + _ => Err(LayoutError), + } + } + + /// Creates a new layout for `[T; len]`. + /// + /// # Safety + /// + /// `len` must be a value, for which `len * size_of::() <= isize::MAX` is true. + pub unsafe fn new_unchecked(len: usize) -> Self { + // INVARIANT: By the safety requirements of this function + // `len * size_of::() <= isize::MAX`. + Self { + len, + _phantom: PhantomData, + } + } + + /// Returns the number of array elements represented by this layout. + pub const fn len(&self) -> usize { + self.len + } + + /// Returns `true` when no array elements are represented by this layout. + pub const fn is_empty(&self) -> bool { + self.len == 0 + } +} + +impl From> for Layout { + fn from(value: ArrayLayout) -> Self { + let res = Layout::array::(value.len); + // SAFETY: By the type invariant of `ArrayLayout` we have + // `len * size_of::() <= isize::MAX` and thus the result must be `Ok`. + unsafe { res.unwrap_unchecked() } + } +} diff --git a/rust/kernel/alloc/vec_ext.rs b/rust/kernel/alloc/vec_ext.rs deleted file mode 100644 index 1297a4be32e8c4..00000000000000 --- a/rust/kernel/alloc/vec_ext.rs +++ /dev/null @@ -1,185 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! Extensions to [`Vec`] for fallible allocations. - -use super::{AllocError, Flags}; -use alloc::vec::Vec; - -/// Extensions to [`Vec`]. -pub trait VecExt: Sized { - /// Creates a new [`Vec`] instance with at least the given capacity. - /// - /// # Examples - /// - /// ``` - /// let v = Vec::::with_capacity(20, GFP_KERNEL)?; - /// - /// assert!(v.capacity() >= 20); - /// # Ok::<(), Error>(()) - /// ``` - fn with_capacity(capacity: usize, flags: Flags) -> Result; - - /// Appends an element to the back of the [`Vec`] instance. - /// - /// # Examples - /// - /// ``` - /// let mut v = Vec::new(); - /// v.push(1, GFP_KERNEL)?; - /// assert_eq!(&v, &[1]); - /// - /// v.push(2, GFP_KERNEL)?; - /// assert_eq!(&v, &[1, 2]); - /// # Ok::<(), Error>(()) - /// ``` - fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError>; - - /// Pushes clones of the elements of slice into the [`Vec`] instance. - /// - /// # Examples - /// - /// ``` - /// let mut v = Vec::new(); - /// v.push(1, GFP_KERNEL)?; - /// - /// v.extend_from_slice(&[20, 30, 40], GFP_KERNEL)?; - /// assert_eq!(&v, &[1, 20, 30, 40]); - /// - /// v.extend_from_slice(&[50, 60], GFP_KERNEL)?; - /// assert_eq!(&v, &[1, 20, 30, 40, 50, 60]); - /// # Ok::<(), Error>(()) - /// ``` - fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> - where - T: Clone; - - /// Ensures that the capacity exceeds the length by at least `additional` elements. - /// - /// # Examples - /// - /// ``` - /// let mut v = Vec::new(); - /// v.push(1, GFP_KERNEL)?; - /// - /// v.reserve(10, GFP_KERNEL)?; - /// let cap = v.capacity(); - /// assert!(cap >= 10); - /// - /// v.reserve(10, GFP_KERNEL)?; - /// let new_cap = v.capacity(); - /// assert_eq!(new_cap, cap); - /// - /// # Ok::<(), Error>(()) - /// ``` - fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError>; -} - -impl VecExt for Vec { - fn with_capacity(capacity: usize, flags: Flags) -> Result { - let mut v = Vec::new(); - >::reserve(&mut v, capacity, flags)?; - Ok(v) - } - - fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError> { - >::reserve(self, 1, flags)?; - let s = self.spare_capacity_mut(); - s[0].write(v); - - // SAFETY: We just initialised the first spare entry, so it is safe to increase the length - // by 1. We also know that the new length is <= capacity because of the previous call to - // `reserve` above. - unsafe { self.set_len(self.len() + 1) }; - Ok(()) - } - - fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> - where - T: Clone, - { - >::reserve(self, other.len(), flags)?; - for (slot, item) in core::iter::zip(self.spare_capacity_mut(), other) { - slot.write(item.clone()); - } - - // SAFETY: We just initialised the `other.len()` spare entries, so it is safe to increase - // the length by the same amount. We also know that the new length is <= capacity because - // of the previous call to `reserve` above. - unsafe { self.set_len(self.len() + other.len()) }; - Ok(()) - } - - #[cfg(any(test, testlib))] - fn reserve(&mut self, additional: usize, _flags: Flags) -> Result<(), AllocError> { - Vec::reserve(self, additional); - Ok(()) - } - - #[cfg(not(any(test, testlib)))] - fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError> { - let len = self.len(); - let cap = self.capacity(); - - if cap - len >= additional { - return Ok(()); - } - - if core::mem::size_of::() == 0 { - // The capacity is already `usize::MAX` for SZTs, we can't go higher. - return Err(AllocError); - } - - // We know cap is <= `isize::MAX` because `Layout::array` fails if the resulting byte size - // is greater than `isize::MAX`. So the multiplication by two won't overflow. - let new_cap = core::cmp::max(cap * 2, len.checked_add(additional).ok_or(AllocError)?); - let layout = core::alloc::Layout::array::(new_cap).map_err(|_| AllocError)?; - - let (old_ptr, len, cap) = destructure(self); - - // We need to make sure that `ptr` is either NULL or comes from a previous call to - // `krealloc_aligned`. A `Vec`'s `ptr` value is not guaranteed to be NULL and might be - // dangling after being created with `Vec::new`. Instead, we can rely on `Vec`'s capacity - // to be zero if no memory has been allocated yet. - let ptr = if cap == 0 { - core::ptr::null_mut() - } else { - old_ptr - }; - - // SAFETY: `ptr` is valid because it's either NULL or comes from a previous call to - // `krealloc_aligned`. We also verified that the type is not a ZST. - let new_ptr = unsafe { super::allocator::krealloc_aligned(ptr.cast(), layout, flags) }; - if new_ptr.is_null() { - // SAFETY: We are just rebuilding the existing `Vec` with no changes. - unsafe { rebuild(self, old_ptr, len, cap) }; - Err(AllocError) - } else { - // SAFETY: `ptr` has been reallocated with the layout for `new_cap` elements. New cap - // is greater than `cap`, so it continues to be >= `len`. - unsafe { rebuild(self, new_ptr.cast::(), len, new_cap) }; - Ok(()) - } - } -} - -#[cfg(not(any(test, testlib)))] -fn destructure(v: &mut Vec) -> (*mut T, usize, usize) { - let mut tmp = Vec::new(); - core::mem::swap(&mut tmp, v); - let mut tmp = core::mem::ManuallyDrop::new(tmp); - let len = tmp.len(); - let cap = tmp.capacity(); - (tmp.as_mut_ptr(), len, cap) -} - -/// Rebuilds a `Vec` from a pointer, length, and capacity. -/// -/// # Safety -/// -/// The same as [`Vec::from_raw_parts`]. -#[cfg(not(any(test, testlib)))] -unsafe fn rebuild(v: &mut Vec, ptr: *mut T, len: usize, cap: usize) { - // SAFETY: The safety requirements from this function satisfy those of `from_raw_parts`. - let mut tmp = unsafe { Vec::from_raw_parts(ptr, len, cap) }; - core::mem::swap(&mut tmp, v); -} diff --git a/rust/kernel/bits.rs b/rust/kernel/bits.rs new file mode 100644 index 00000000000000..7c111df61d8617 --- /dev/null +++ b/rust/kernel/bits.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Bit manipulation helpers +//! +//! C header: [`include/linux/bits.h`](srctree/include/linux/bits.h) + +/// Generate a mask where all bits >= `h` and <= `l` are set +/// +/// This is a re-implementation in rust of `GENMASK` +pub const fn genmask(h: u32, l: u32) -> u32 { + ((!0u32) - (1 << l) + 1) & ((!0u32) >> (32 - 1 - h)) +} diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 851018eef885e7..e20f1c7eb272d2 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -8,7 +8,10 @@ use crate::{ bindings, types::{ARef, Opaque}, }; -use core::ptr; +use core::{fmt, ptr}; + +#[cfg(CONFIG_PRINTK)] +use crate::c_str; /// A reference-counted device. /// @@ -76,12 +79,116 @@ impl Device { /// /// Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count, /// i.e. it must be ensured that the reference count of the C `struct device` `ptr` points to - /// can't drop to zero, for the duration of this function call and the entire duration when the + /// can't drop to zero, for the duration of this function callhe entire duration when the /// returned reference exists. pub unsafe fn as_ref<'a>(ptr: *mut bindings::device) -> &'a Self { // SAFETY: Guaranteed by the safety requirements of the function. unsafe { &*ptr.cast() } } + + /// Prints an emergency-level message (level 0) prefixed with device information. + /// + /// More details are available from [`dev_emerg`]. + /// + /// [`dev_emerg`]: crate::dev_emerg + pub fn pr_emerg(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_EMERG, args) }; + } + + /// Prints an alert-level message (level 1) prefixed with device information. + /// + /// More details are available from [`dev_alert`]. + /// + /// [`dev_alert`]: crate::dev_alert + pub fn pr_alert(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_ALERT, args) }; + } + + /// Prints a critical-level message (level 2) prefixed with device information. + /// + /// More details are available from [`dev_crit`]. + /// + /// [`dev_crit`]: crate::dev_crit + pub fn pr_crit(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_CRIT, args) }; + } + + /// Prints an error-level message (level 3) prefixed with device information. + /// + /// More details are available from [`dev_err`]. + /// + /// [`dev_err`]: crate::dev_err + pub fn pr_err(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_ERR, args) }; + } + + /// Prints a warning-level message (level 4) prefixed with device information. + /// + /// More details are available from [`dev_warn`]. + /// + /// [`dev_warn`]: crate::dev_warn + pub fn pr_warn(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_WARNING, args) }; + } + + /// Prints a notice-level message (level 5) prefixed with device information. + /// + /// More details are available from [`dev_notice`]. + /// + /// [`dev_notice`]: crate::dev_notice + pub fn pr_notice(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_NOTICE, args) }; + } + + /// Prints an info-level message (level 6) prefixed with device information. + /// + /// More details are available from [`dev_info`]. + /// + /// [`dev_info`]: crate::dev_info + pub fn pr_info(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_INFO, args) }; + } + + /// Prints a debug-level message (level 7) prefixed with device information. + /// + /// More details are available from [`dev_dbg`]. + /// + /// [`dev_dbg`]: crate::dev_dbg + pub fn pr_dbg(&self, args: fmt::Arguments<'_>) { + if cfg!(debug_assertions) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_DEBUG, args) }; + } + } + + /// Prints the provided message to the console. + /// + /// # Safety + /// + /// Callers must ensure that `klevel` is null-terminated; in particular, one of the + /// `KERN_*`constants, for example, `KERN_CRIT`, `KERN_ALERT`, etc. + #[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] + unsafe fn printk(&self, klevel: &[u8], msg: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminatedne of the kernel constants. `self.as_raw` + // is valid because `self` is valid. The "%pA" format string expects a pointer to + // `fmt::Arguments`, which is what we're passing as the last argument. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_dev_printk( + klevel as *const _ as *const core::ffi::c_char, + self.as_raw(), + c_str!("%pA").as_char_ptr(), + &msg as *const _ as *const core::ffi::c_void, + ) + }; + } } // SAFETY: Instances of `Device` are always reference-counted. @@ -103,3 +210,213 @@ unsafe impl Send for Device {} // SAFETY: `Device` can be shared among threads because all immutable methods are protected by the // synchronization in `struct device`. unsafe impl Sync for Device {} + +#[doc(hidden)] +#[macro_export] +macro_rules! dev_printk { + ($method:ident, $dev:expr, $($f:tt)*) => { + { + ($dev).$method(core::format_args!($($f)*)); + } + } +} + +/// Prints an emergency-level message (level 0) prefixed with device information. +/// +/// This level should be used if the system is unusable. +/// +/// Equivalent to the kernel's `dev_emerg` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_emerg!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_emerg { + ($($f:tt)*) => { $crate::dev_printk!(pr_emerg, $($f)*); } +} + +/// Prints an alert-level message (level 1) prefixed with device information. +/// +/// This level should be used if action must be taken immediately. +/// +/// Equivalent to the kernel's `dev_alert` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_alert!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_alert { + ($($f:tt)*) => { $crate::dev_printk!(pr_alert, $($f)*); } +} + +/// Prints a critical-level message (level 2) prefixed with device information. +/// +/// This level should be used in critical conditions. +/// +/// Equivalent to the kernel's `dev_crit` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_crit!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_crit { + ($($f:tt)*) => { $crate::dev_printk!(pr_crit, $($f)*); } +} + +/// Prints an error-level message (level 3) prefixed with device information. +/// +/// This level should be used in error conditions. +/// +/// Equivalent to the kernel's `dev_err` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_err!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_err { + ($($f:tt)*) => { $crate::dev_printk!(pr_err, $($f)*); } +} + +/// Prints a warning-level message (level 4) prefixed with device information. +/// +/// This level should be used in warning conditions. +/// +/// Equivalent to the kernel's `dev_warn` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_warn!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_warn { + ($($f:tt)*) => { $crate::dev_printk!(pr_warn, $($f)*); } +} + +/// Prints a notice-level message (level 5) prefixed with device information. +/// +/// This level should be used in normal but significant conditions. +/// +/// Equivalent to the kernel's `dev_notice` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_notice!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_notice { + ($($f:tt)*) => { $crate::dev_printk!(pr_notice, $($f)*); } +} + +/// Prints an info-level message (level 6) prefixed with device information. +/// +/// This level should be used for informational messages. +/// +/// Equivalent to the kernel's `dev_info` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_info!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_info { + ($($f:tt)*) => { $crate::dev_printk!(pr_info, $($f)*); } +} + +/// Prints a debug-level message (level 7) prefixed with device information. +/// +/// This level should be used for debug messages. +/// +/// Equivalent to the kernel's `dev_dbg` macro, except that it doesn't support dynamic debug yet. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_dbg!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_dbg { + ($($f:tt)*) => { $crate::dev_printk!(pr_dbg, $($f)*); } +} diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs new file mode 100644 index 00000000000000..c84e06f774e778 --- /dev/null +++ b/rust/kernel/device_id.rs @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic implementation of device IDs. +//! +//! Each bus / subsystem that matches device and driver through a bus / subsystem specific ID is +//! expected to implement [`RawDeviceId`]. + +use core::mem::MaybeUninit; + +/// Marker trait to indicate a Rust device ID type represents a corresponding C device ID type. +/// +/// This is meant to be implemented by buses/subsystems so that they can use [`IdTable`] to +/// guarantee (at compile-time) zero-termination of device id tables provided by drivers. +/// +/// # Safety +/// +/// Implementers must ensure that: +/// - `Self` is layout-compatible with [`RawDeviceId::RawType`]; i.e. it's safe to transmute to +/// `RawDeviceId`. +/// +/// This requirement is needed so `IdArray::new` can convert `Self` to `RawType` when building +/// the ID table. +/// +/// Ideally, this should be achieved using a const function that does conversion instead of +/// transmute; however, const trait functions relies on `const_trait_impl` unstable feature, +/// which is broken/gone in Rust 1.73. +/// +/// - `DRIVER_DATA_OFFSET` is the offset of context/data field of the device ID (usually named +/// `driver_data`) of the device ID, the field is suitable sized to write a `usize` value. +/// +/// Similar to the previous requirement, the data should ideally be added during `Self` to +/// `RawType` conversion, but there's currently no way to do it when using traits in const. +pub unsafe trait RawDeviceId { + /// The raw type that holds the device id. + /// + /// Id tables created from [`Self`] are going to hold this type in its zero-terminated array. + type RawType: Copy; + + /// The offest to the context/data field. + const DRIVER_DATA_OFFSET: usize; +} + +/// A zero-terminated device id array. +#[repr(C)] +pub struct RawIdArray { + ids: [T::RawType; N], + sentinel: MaybeUninit, +} + +impl RawIdArray { + #[doc(hidden)] + pub const fn len(&self) -> usize { + (N + 1) * core::mem::size_of::() + } +} + +/// A zero-terminated device id array, followed by context data. +#[repr(C)] +pub struct IdArray { + #[doc(hidden)] + pub raw_ids: RawIdArray, + id_infos: [U; N], +} + +impl IdArray { + /// Creates a new instance of the array. + /// + /// The contents are derived from the given identifiers and context information. + pub const fn new(ids: [(T, U); N]) -> Self { + let mut raw_ids = [const { MaybeUninit::::uninit() }; N]; + let mut infos = [const { MaybeUninit::uninit() }; N]; + + let mut i = 0usize; + while i < N { + // SAFETY: by the safety requirement of `RawDeviceId`, we're guaranteed that `T` is + // layout-wise compatible with `RawType`. + raw_ids[i] = unsafe { core::mem::transmute_copy(&ids[i].0) }; + // SAFETY: by the safety requirement of `RawDeviceId`, this would be effectively + // `raw_ids[i].driver_data = i;`. + unsafe { + raw_ids[i] + .as_mut_ptr() + .byte_offset(T::DRIVER_DATA_OFFSET as _) + .cast::() + .write(i); + } + + // SAFETY: this is effectively a move: `infos[i] = ids[i].1`. We make a copy here but + // later forget `ids`. + infos[i] = MaybeUninit::new(unsafe { core::ptr::read(&ids[i].1) }); + i += 1; + } + + core::mem::forget(ids); + + Self { + // SAFETY: this is effectively `array_assume_init`, which is unstable, so we use + // `transmute_copy` instead. We have initialized all elements of `raw_ids` so this + // `array_assume_init` is safe. + raw_ids: RawIdArray { + ids: unsafe { core::mem::transmute_copy(&raw_ids) }, + sentinel: MaybeUninit::zeroed(), + }, + // SAFETY: We have initialized all elements of `infos` so this `array_assume_init` is + // safe. + id_infos: unsafe { core::mem::transmute_copy(&infos) }, + } + } +} + +/// A device id table. +/// +/// This trait is only implemented by `IdArray`. +/// +/// The purpose of this trait is to allow `&'static dyn IdArray` to be in context when `N` in +/// `IdArray` doesn't matter. +pub trait IdTable { + /// Obtain the pointer to the ID table. + fn as_ptr(&self) -> *const T::RawType; + + /// Obtain the pointer to the bus specific device ID from an index. + fn id(&self, index: usize) -> &T::RawType; + + /// Obtain the pointer to the driver-specific information from an index. + fn info(&self, index: usize) -> &U; +} + +impl IdTable for IdArray { + fn as_ptr(&self) -> *const T::RawType { + // This cannot be `self.ids.as_ptr()`, as the return pointer must have correct provenance + // to access the sentinel. + (self as *const Self).cast() + } + + fn id(&self, index: usize) -> &T::RawType { + &self.raw_ids.ids[index] + } + + fn info(&self, index: usize) -> &U { + &self.id_infos[index] + } +} + +/// Create device table alias for modpost. +#[macro_export] +macro_rules! module_device_table { + ($table_type: literal, $module_table_name:ident, $table_name:ident) => { + #[export_name = concat!("__mod_", $table_type, "__", stringify!($table_name), "_device_table")] + static $module_table_name: [u8; $table_name.raw_ids.len()] = + unsafe { core::mem::transmute_copy(&$table_name.raw_ids) }; + }; +} diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs new file mode 100644 index 00000000000000..b23559f55214ec --- /dev/null +++ b/rust/kernel/devres.rs @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Devres abstraction +//! +//! [`Devres`] represents an abstraction for the kernel devres (device resource management) +//! implementation. + +use crate::{ + alloc::Flags, + bindings, + device::Device, + error::{Error, Result}, + prelude::*, + revocable::Revocable, + sync::Arc, +}; + +use core::ffi::c_void; +use core::ops::Deref; + +#[pin_data] +struct DevresInner { + #[pin] + data: Revocable, +} + +/// This abstraction is meant to be used by subsystems to containerize [`Device`] bound resources to +/// manage their lifetime. +/// +/// [`Device`] bound resources should be freed when either the resource goes out of scope or the +/// [`Device`] is unbound respectively, depending on what happens first. +/// +/// To achieve that [`Devres`] registers a devres callback on creation, which is called once the +/// [`Device`] is unbound, revoking access to the encapsulated resource (see also [`Revocable`]). +/// +/// After the [`Devres`] has been unbound it is not possible to access the encapsulated resource +/// anymore. +/// +/// [`Devres`] users should make sure to simply free the corresponding backing resource in `T`'s +/// [`Drop`] implementation. +/// +/// # Example +/// +/// ```no_run +/// # use kernel::{bindings, c_str, device::Device, devres::Devres, io::Io}; +/// # use core::ops::Deref; +/// +/// // See also [`pci::Bar`] for a real example. +/// struct IoMem(Io); +/// +/// impl IoMem { +/// /// # Safety +/// /// +/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs +/// /// virtual address space. +/// unsafe fn new(paddr: usize) -> Result{ +/// +/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is +/// // valid for `ioremap`. +/// let addr = unsafe { bindings::ioremap(paddr as _, SIZE.try_into().unwrap()) }; +/// if addr.is_null() { +/// return Err(ENOMEM); +/// } +/// +/// // SAFETY: `addr` is guaranteed to be the start of a valid I/O mapped memory region of +/// // size `SIZE`. +/// let io = unsafe { Io::new(addr as _, SIZE)? }; +/// +/// Ok(IoMem(io)) +/// } +/// } +/// +/// impl Drop for IoMem { +/// fn drop(&mut self) { +/// // SAFETY: Safe as by the invariant of `Io`. +/// unsafe { bindings::iounmap(self.0.base_addr() as _); }; +/// } +/// } +/// +/// impl Deref for IoMem { +/// type Target = Io; +/// +/// fn deref(&self) -> &Self::Target { +/// &self.0 +/// } +/// } +/// +/// # fn no_run() -> Result<(), Error> { +/// # // SAFETY: Invalid usage; just for the example to get an `ARef` instance. +/// # let dev = unsafe { Device::from_raw(core::ptr::null_mut()) }; +/// +/// // SAFETY: Invalid usage for example purposes. +/// let iomem = unsafe { IoMem::<{ core::mem::size_of::() }>::new(0xBAAAAAAD)? }; +/// let devres = Devres::new(&dev, iomem, GFP_KERNEL)?; +/// +/// let res = devres.try_access().ok_or(ENXIO)?; +/// res.writel(0x42, 0x0); +/// # Ok(()) +/// # } +/// ``` +pub struct Devres(Arc>); + +impl DevresInner { + fn new(dev: &Device, data: T, flags: Flags) -> Result>> { + let inner = Arc::pin_init( + pin_init!( DevresInner { + data <- Revocable::new(data), + }), + flags, + )?; + + // Convert `Arc` into a raw pointer and make devres own this reference until + // `Self::devres_callback` is called. + let data = inner.clone().into_raw(); + + // SAFETY: `devm_add_action` guarantees to call `Self::devres_callback` once `dev` is + // detached. + let ret = unsafe { + bindings::devm_add_action(dev.as_raw(), Some(Self::devres_callback), data as _) + }; + + if ret != 0 { + // SAFETY: We just created another reference to `inner` in order to pass it to + // `bindings::devm_add_action`. If `bindings::devm_add_action` fails, we have to drop + // this reference accordingly. + let _ = unsafe { Arc::from_raw(data) }; + return Err(Error::from_errno(ret)); + } + + Ok(inner) + } + + #[allow(clippy::missing_safety_doc)] + unsafe extern "C" fn devres_callback(ptr: *mut c_void) { + let ptr = ptr as *mut DevresInner; + // Devres owned this memory; now that we received the callback, drop the `Arc` and hence the + // reference. + // SAFETY: Safe, since we leaked an `Arc` reference to devm_add_action() in + // `DevresInner::new`. + let inner = unsafe { Arc::from_raw(ptr) }; + + inner.data.revoke(); + } +} + +impl Devres { + /// Creates a new [`Devres`] instance of the given `data`. The `data` encapsulated within the + /// returned `Devres` instance' `data` will be revoked once the device is detached. + pub fn new(dev: &Device, data: T, flags: Flags) -> Result { + let inner = DevresInner::new(dev, data, flags)?; + + Ok(Devres(inner)) + } + + /// Same as [`Devres::new`], but does not return a `Devres` instance. Instead the given `data` + /// is owned by devres and will be revoked / dropped, once the device is detached. + pub fn new_foreign_owned(dev: &Device, data: T, flags: Flags) -> Result { + let _ = DevresInner::new(dev, data, flags)?; + + Ok(()) + } +} + +impl Deref for Devres { + type Target = Revocable; + + fn deref(&self) -> &Self::Target { + &self.0.data + } +} + +impl Drop for Devres { + fn drop(&mut self) { + // Revoke the data, such that it gets dropped already and the actual resource is freed. + // `DevresInner` has to stay alive until the devres callback has been called. This is + // necessary since we don't know when `Devres` is dropped and calling + // `devm_remove_action()` instead could race with `devres_release_all()`. + self.revoke(); + } +} diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs new file mode 100644 index 00000000000000..f9766038b14e54 --- /dev/null +++ b/rust/kernel/driver.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic support for drivers of different buses (e.g., PCI, Platform, Amba, etc.). +//! +//! Each bus / subsystem is expected to implement [`RegistrationOps`], which allows drivers to +//! register using the [`Registration`] class. + +use crate::error::{Error, Result}; +use crate::{init::PinInit, str::CStr, try_pin_init, types::Opaque, ThisModule}; +use core::pin::Pin; +use macros::{pin_data, pinned_drop}; + +/// The [`RegistrationOps`] trait serves as generic interface for subsystems (e.g., PCI, Platform, +/// Amba, etc.) to privide the corresponding subsystem specific implementation to register / +/// unregister a driver of the particular type (`RegType`). +/// +/// For instance, the PCI subsystem would set `RegType` to `bindings::pci_driver` and call +/// `bindings::__pci_register_driver` from `RegistrationOps::register` and +/// `bindings::pci_unregister_driver` from `RegistrationOps::unregister`. +pub trait RegistrationOps { + /// The type that holds information about the registration. This is typically a struct defined + /// by the C portion of the kernel. + type RegType: Default; + + /// Registers a driver. + /// + /// On success, `reg` must remain pinned and valid until the matching call to + /// [`RegistrationOps::unregister`]. + fn register( + reg: &mut Self::RegType, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result; + + /// Unregisters a driver previously registered with [`RegistrationOps::register`]. + fn unregister(reg: &mut Self::RegType); +} + +/// A [`Registration`] is a generic type that represents the registration of some driver type (e.g. +/// `bindings::pci_driver`). Therefore a [`Registration`] must be initialized with a type that +/// implements the [`RegistrationOps`] trait, such that the generic `T::register` and `T::unregister` +/// calls result in the subsystem specific registration calls. +/// +///Once the `Registration` structure is dropped, the driver is unregistered. +#[pin_data(PinnedDrop)] +pub struct Registration { + #[pin] + reg: Opaque, +} + +// SAFETY: `Registration` has no fields or methods accessible via `&Registration`, so it is safe to +// share references to it with multiple threads as nothing can be done. +unsafe impl Sync for Registration {} + +// SAFETY: Both registration and unregistration are implemented in C and safe to be performed from +// any thread, so `Registration` is `Send`. +unsafe impl Send for Registration {} + +impl Registration { + /// Creates a new instance of the registration object. + pub fn new(name: &'static CStr, module: &'static ThisModule) -> impl PinInit { + try_pin_init!(Self { + reg <- Opaque::try_ffi_init(|ptr: *mut T::RegType| { + // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write. + unsafe { ptr.write(T::RegType::default()) }; + + // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write, and it has + // just been initialised above, so it's also valid for read. + let drv = unsafe { &mut *ptr }; + + T::register(drv, name, module) + }), + }) + } +} + +#[pinned_drop] +impl PinnedDrop for Registration { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The existance of the `Registration` guarantees that `self.reg.get()` is properly + // aligned and points to a valid value. + let drv = unsafe { &mut *self.reg.get() }; + + T::unregister(drv); + } +} + +/// A kernel module that only registers the given driver on init. +/// +/// This is a helper struct to make it easier to define single-functionality modules, in this case, +/// modules that offer a single driver. +#[pin_data] +pub struct Module { + #[pin] + _driver: Registration, +} + +impl crate::InPlaceModule for Module { + fn init(name: &'static CStr, module: &'static ThisModule) -> impl PinInit { + try_pin_init!(Self { + _driver <- Registration::::new(name, module), + }) + } +} + +/// Declares a kernel module that exposes a single driver. +/// +/// It is meant to be used as a helper by other subsystems so they can more easily expose their own +/// macros. +#[macro_export] +macro_rules! module_driver { + (<$gen_type:ident>, $driver_ops:ty, { type: $type:ty, $($f:tt)* }) => { + type Ops<$gen_type> = $driver_ops; + type ModuleType = $crate::driver::Module>; + $crate::prelude::module! { + type: ModuleType, + $($f)* + } + } +} diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index be6509d5f4a4a7..ff48a8b3f3a5d5 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -6,7 +6,7 @@ use crate::{alloc::AllocError, str::CStr}; -use alloc::alloc::LayoutError; +use core::alloc::LayoutError; use core::fmt; use core::num::NonZeroI32; @@ -82,6 +82,7 @@ pub mod code { declare_err!(EIOCBQUEUED, "iocb queued, will get completion event."); declare_err!(ERECALLCONFLICT, "Conflict with recalled state."); declare_err!(ENOGRACE, "NFS file lock reclaim refused."); + declare_err!(ENOTRECOVERABLE, "State not recoverable."); } /// Generic integer kernel error. @@ -161,7 +162,7 @@ impl Error { } /// Returns a string representing the error, if one exists. - #[cfg(not(testlib))] + #[cfg(not(any(test, testlib)))] pub fn name(&self) -> Option<&'static CStr> { // SAFETY: Just an FFI call, there are no extra safety requirements. let ptr = unsafe { bindings::errname(-self.0.get()) }; @@ -178,7 +179,7 @@ impl Error { /// When `testlib` is configured, this always returns `None` to avoid the dependency on a /// kernel function so that tests that use this (e.g., by calling [`Result::unwrap`]) can still /// run in userspace. - #[cfg(testlib)] + #[cfg(any(test, testlib))] pub fn name(&self) -> Option<&'static CStr> { None } diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs new file mode 100644 index 00000000000000..4e3a6281b7774c --- /dev/null +++ b/rust/kernel/i2c.rs @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! I2C devices and drivers. +//! +//! C header: [`include/linux/i2c.h`](../../../../include/linux/i2c.h) + +use crate::{ + bindings, + device::Device, + device_id::{self, RawDeviceId}, + driver, + error::{from_result, to_result, Result}, + of, + str::{BStr, CStr}, + types::ForeignOwnable, + ThisModule, +}; + +/// An I2C device id. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::i2c_device_id); + +impl DeviceId { + /// Create a new I2C DeviceId + pub const fn new(name: &CStr) -> Self { + let device_id = core::mem::MaybeUninit::::zeroed(); + let mut device_id = unsafe { device_id.assume_init() }; + + let name = BStr::from_bytes(name.as_bytes_with_nul()); + assert!(name.len() <= device_id.name.len()); + + let mut i = 0; + while i < name.len() { + device_id.name[i] = name.deref_const()[i] as _; + i += 1; + } + + Self(device_id) + } +} + +// SAFETY: `ZERO` is all zeroed-out and `to_rawid` stores `offset` in `i2c_device_id::driver_data`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::i2c_device_id; + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::i2c_device_id, driver_data); +} + +/// Alias for `device_id::IdTable` containing I2C's `DeviceId` +pub type IdTable = &'static dyn device_id::IdTable; + +/// An adapter for the registration of i2c drivers. +pub struct Adapter(T); + +impl driver::RegistrationOps for Adapter { + type RegType = bindings::i2c_driver; + + fn register( + i2cdrv: &mut Self::RegType, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + i2cdrv.driver.name = name.as_char_ptr(); + i2cdrv.probe = Some(Self::probe_callback); + i2cdrv.remove = Some(Self::remove_callback); + if let Some(t) = T::I2C_DEVICE_ID_TABLE { + i2cdrv.id_table = t.as_ptr(); + } + if let Some(t) = T::OF_DEVICE_ID_TABLE { + i2cdrv.driver.of_match_table = t.as_ptr(); + } + + // SAFETY: + // - `pdrv` lives at least until the call to `platform_driver_unregister()` returns. + // - `name` pointer has static lifetime. + // - `module.0` lives at least as long as the module. + // - `probe()` and `remove()` are static functions. + // - `of_match_table` is either a raw pointer with static lifetime, + // as guaranteed by the [`device_id::IdTable`] type, or null. + to_result(unsafe { bindings::i2c_register_driver(module.0, i2cdrv) }) + } + + fn unregister(i2cdrv: &mut Self::RegType) { + // SAFETY: By the safety requirements of this function (defined in the trait definition), + // `reg` was passed (and updated) by a previous successful call to + // `i2c_register_driver`. + unsafe { bindings::i2c_del_driver(i2cdrv) }; + } +} + +impl Adapter { + extern "C" fn probe_callback(i2c: *mut bindings::i2c_client) -> core::ffi::c_int { + from_result(|| { + let mut client = unsafe { Client::from_ptr(i2c) }; + let data = T::probe(&mut client)?; + + // SAFETY: `i2c` is guaranteed to be a valid, non-null pointer. + unsafe { bindings::i2c_set_clientdata(i2c, data.into_foreign() as _) }; + Ok(0) + }) + } + + extern "C" fn remove_callback(i2c: *mut bindings::i2c_client) { + // SAFETY: `i2c` is guaranteed to be a valid, non-null pointer + let ptr = unsafe { bindings::i2c_get_clientdata(i2c) }; + // SAFETY: + // - we allocated this pointer using `T::Data::into_pointer`, + // so it is safe to turn back into a `T::Data`. + // - the allocation happened in `probe`, no-one freed the memory, + // `remove` is the canonical kernel location to free driver data. so OK + // to convert the pointer back to a Rust structure here. + let data = unsafe { T::Data::from_foreign(ptr) }; + T::remove(&data); + } +} + +/// A I2C driver. +pub trait Driver { + /// Data stored on device by driver. + /// + /// Corresponds to the data set or retrieved via the kernel's + /// `i2c_{set,get}_clientdata()` functions. + /// + /// Require that `Data` implements `ForeignOwnable`. We guarantee to + /// never move the underlying wrapped data structure. This allows + type Data: ForeignOwnable = (); + + /// The type holding information about each device id supported by the driver. + type IdInfo: 'static = (); + + /// The table of i2c device ids supported by the driver. + const I2C_DEVICE_ID_TABLE: Option> = None; + + /// The table of OF device ids supported by the driver. + const OF_DEVICE_ID_TABLE: Option> = None; + + /// I2C driver probe. + /// + /// Called when a new i2c client is added or discovered. + /// Implementers should attempt to initialize the client here. + fn probe(client: &mut Client) -> Result; + + /// I2C driver remove. + /// + /// Called when an i2c client is removed. + fn remove(_data: &Self::Data) {} +} + +/// A I2C Client device. +/// +/// # Invariants +/// +/// The field `ptr` is non-null and valid for the lifetime of the object. +pub struct Client { + ptr: *mut bindings::i2c_client, +} + +impl Client { + /// Creates a new client from the given pointer. + /// + /// # Safety + /// + /// `ptr` must be non-null and valid. It must remain valid for the lifetime of the returned + /// instance. + unsafe fn from_ptr(ptr: *mut bindings::i2c_client) -> Self { + // INVARIANT: The safety requirements of the function ensure the lifetime invariant. + Self { ptr } + } + + /// Returns the raw I2C client structure. + pub fn raw_client(&self) -> *mut bindings::i2c_client { + self.ptr + } +} + +impl AsRef for Client { + fn as_ref(&self) -> &Device { + // SAFETY: By the type invariants, we know that `self.ptr` is non-null and valid. + unsafe { Device::as_ref(&mut (*self.ptr).dev) } + } +} + +/// Declares a kernel module that exposes a single i2c driver. +/// +/// # Examples +/// +/// ```ignore +/// # use kernel::{i2c, define_i2c_id_table, module_i2c_driver}; +/// kernel::module_i2c_id_table!(MOD_TABLE, I2C_CLIENT_I2C_ID_TABLE); +/// kernel::define_i2c_id_table! {I2C_CLIENT_I2C_ID_TABLE, (), [ +/// (i2c::DeviceId(b"fpga"), None), +/// ]} +/// struct MyDriver; +/// impl i2c::Driver for MyDriver { +/// kernel::driver_i2c_id_table!(I2C_CLIENT_I2C_ID_TABLE); +/// // [...] +/// # fn probe(_client: &mut i2c::Client) -> Result { +/// # Ok(()) +/// # } +/// } +/// +/// module_i2c_driver! { +/// type: MyDriver, +/// name: "module_name", +/// author: "Author name", +/// license: "GPL", +/// } +/// ``` +#[macro_export] +macro_rules! module_i2c_driver { + ($($f:tt)*) => { + $crate::module_driver!(, $crate::i2c::Adapter, { $($f)* }); + }; +} + +/// Create a I2C `IdTable` with its alias for modpost. +#[macro_export] +macro_rules! i2c_device_table { + ($module_table_name:ident, $table_name:ident, $id_info_type: ty, $table_data: expr) => { + const $table_name: $crate::device_id::IdArray< + $crate::i2c::DeviceId, + $id_info_type, + { $table_data.len() }, + > = $crate::device_id::IdArray::new($table_data); + + $crate::module_device_table!("i2c", $module_table_name, $table_name); + }; +} diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 25057cbed40b5f..c9919ba0b68368 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -13,7 +13,7 @@ //! To initialize a `struct` with an in-place constructor you will need two things: //! - an in-place constructor, //! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc`], -//! [`UniqueArc`], [`Box`] or any other smart pointer that implements [`InPlaceInit`]). +//! [`UniqueArc`], [`KBox`] or any other smart pointer that implements [`InPlaceInit`]). //! //! To get an in-place constructor there are generally three options: //! - directly creating an in-place constructor using the [`pin_init!`] macro, @@ -68,7 +68,7 @@ //! # a <- new_mutex!(42, "Foo::a"), //! # b: 24, //! # }); -//! let foo: Result>> = Box::pin_init(foo, GFP_KERNEL); +//! let foo: Result>> = KBox::pin_init(foo, GFP_KERNEL); //! ``` //! //! For more information see the [`pin_init!`] macro. @@ -92,14 +92,14 @@ //! struct DriverData { //! #[pin] //! status: Mutex, -//! buffer: Box<[u8; 1_000_000]>, +//! buffer: KBox<[u8; 1_000_000]>, //! } //! //! impl DriverData { //! fn new() -> impl PinInit { //! try_pin_init!(Self { //! status <- new_mutex!(0, "DriverData::status"), -//! buffer: Box::init(kernel::init::zeroed(), GFP_KERNEL)?, +//! buffer: KBox::init(kernel::init::zeroed(), GFP_KERNEL)?, //! }) //! } //! } @@ -211,13 +211,12 @@ //! [`pin_init!`]: crate::pin_init! use crate::{ - alloc::{box_ext::BoxExt, AllocError, Flags}, + alloc::{AllocError, Flags, KBox}, error::{self, Error}, sync::Arc, sync::UniqueArc, types::{Opaque, ScopeGuard}, }; -use alloc::boxed::Box; use core::{ cell::UnsafeCell, convert::Infallible, @@ -298,7 +297,7 @@ macro_rules! stack_pin_init { /// struct Foo { /// #[pin] /// a: Mutex, -/// b: Box, +/// b: KBox, /// } /// /// struct Bar { @@ -307,7 +306,7 @@ macro_rules! stack_pin_init { /// /// stack_try_pin_init!(let foo: Result, AllocError> = pin_init!(Foo { /// a <- new_mutex!(42), -/// b: Box::new(Bar { +/// b: KBox::new(Bar { /// x: 64, /// }, GFP_KERNEL)?, /// })); @@ -324,7 +323,7 @@ macro_rules! stack_pin_init { /// struct Foo { /// #[pin] /// a: Mutex, -/// b: Box, +/// b: KBox, /// } /// /// struct Bar { @@ -333,7 +332,7 @@ macro_rules! stack_pin_init { /// /// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo { /// a <- new_mutex!(42), -/// b: Box::new(Bar { +/// b: KBox::new(Bar { /// x: 64, /// }, GFP_KERNEL)?, /// })); @@ -391,7 +390,7 @@ macro_rules! stack_try_pin_init { /// }, /// }); /// # initializer } -/// # Box::pin_init(demo(), GFP_KERNEL).unwrap(); +/// # KBox::pin_init(demo(), GFP_KERNEL).unwrap(); /// ``` /// /// Arbitrary Rust expressions can be used to set the value of a variable. @@ -460,7 +459,7 @@ macro_rules! stack_try_pin_init { /// # }) /// # } /// # } -/// let foo = Box::pin_init(Foo::new(), GFP_KERNEL); +/// let foo = KBox::pin_init(Foo::new(), GFP_KERNEL); /// ``` /// /// They can also easily embed it into their own `struct`s: @@ -588,11 +587,10 @@ macro_rules! pin_init { /// # Examples /// /// ```rust -/// # #![feature(new_uninit)] /// use kernel::{init::{self, PinInit}, error::Error}; /// #[pin_data] /// struct BigBuf { -/// big: Box<[u8; 1024 * 1024 * 1024]>, +/// big: KBox<[u8; 1024 * 1024 * 1024]>, /// small: [u8; 1024 * 1024], /// ptr: *mut u8, /// } @@ -600,7 +598,7 @@ macro_rules! pin_init { /// impl BigBuf { /// fn new() -> impl PinInit { /// try_pin_init!(Self { -/// big: Box::init(init::zeroed(), GFP_KERNEL)?, +/// big: KBox::init(init::zeroed(), GFP_KERNEL)?, /// small: [0; 1024 * 1024], /// ptr: core::ptr::null_mut(), /// }? Error) @@ -692,16 +690,16 @@ macro_rules! init { /// # Examples /// /// ```rust -/// use kernel::{init::{PinInit, zeroed}, error::Error}; +/// use kernel::{alloc::KBox, init::{PinInit, zeroed}, error::Error}; /// struct BigBuf { -/// big: Box<[u8; 1024 * 1024 * 1024]>, +/// big: KBox<[u8; 1024 * 1024 * 1024]>, /// small: [u8; 1024 * 1024], /// } /// /// impl BigBuf { /// fn new() -> impl Init { /// try_init!(Self { -/// big: Box::init(zeroed(), GFP_KERNEL)?, +/// big: KBox::init(zeroed(), GFP_KERNEL)?, /// small: [0; 1024 * 1024], /// }? Error) /// } @@ -812,8 +810,8 @@ macro_rules! assert_pinned { /// A pin-initializer for the type `T`. /// /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can -/// be [`Box`], [`Arc`], [`UniqueArc`] or even the stack (see [`stack_pin_init!`]). Use the -/// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc`] on this. +/// be [`KBox`], [`Arc`], [`UniqueArc`] or even the stack (see [`stack_pin_init!`]). Use +/// the [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc`] on this. /// /// Also see the [module description](self). /// @@ -893,7 +891,7 @@ pub unsafe trait PinInit: Sized { } /// An initializer returned by [`PinInit::pin_chain`]. -pub struct ChainPinInit(I, F, __internal::Invariant<(E, Box)>); +pub struct ChainPinInit(I, F, __internal::Invariant<(E, KBox)>); // SAFETY: The `__pinned_init` function is implemented such that it // - returns `Ok(())` on successful initialization, @@ -919,8 +917,8 @@ where /// An initializer for `T`. /// /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can -/// be [`Box`], [`Arc`], [`UniqueArc`] or even the stack (see [`stack_pin_init!`]). Use the -/// [`InPlaceInit::init`] function of a smart pointer like [`Arc`] on this. Because +/// be [`KBox`], [`Arc`], [`UniqueArc`] or even the stack (see [`stack_pin_init!`]). Use +/// the [`InPlaceInit::init`] function of a smart pointer like [`Arc`] on this. Because /// [`PinInit`] is a super trait, you can use every function that takes it as well. /// /// Also see the [module description](self). @@ -992,7 +990,7 @@ pub unsafe trait Init: PinInit { } /// An initializer returned by [`Init::chain`]. -pub struct ChainInit(I, F, __internal::Invariant<(E, Box)>); +pub struct ChainInit(I, F, __internal::Invariant<(E, KBox)>); // SAFETY: The `__init` function is implemented such that it // - returns `Ok(())` on successful initialization, @@ -1076,8 +1074,9 @@ pub fn uninit() -> impl Init, E> { /// # Examples /// /// ```rust -/// use kernel::{error::Error, init::init_array_from_fn}; -/// let array: Box<[usize; 1_000]> = Box::init::(init_array_from_fn(|i| i), GFP_KERNEL).unwrap(); +/// use kernel::{alloc::KBox, error::Error, init::init_array_from_fn}; +/// let array: KBox<[usize; 1_000]> = +/// KBox::init::(init_array_from_fn(|i| i), GFP_KERNEL).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` pub fn init_array_from_fn( @@ -1244,26 +1243,6 @@ impl InPlaceInit for Arc { } } -impl InPlaceInit for Box { - type PinnedSelf = Pin; - - #[inline] - fn try_pin_init(init: impl PinInit, flags: Flags) -> Result - where - E: From, - { - as BoxExt<_>>::new_uninit(flags)?.write_pin_init(init) - } - - #[inline] - fn try_init(init: impl Init, flags: Flags) -> Result - where - E: From, - { - as BoxExt<_>>::new_uninit(flags)?.write_init(init) - } -} - impl InPlaceInit for UniqueArc { type PinnedSelf = Pin; @@ -1300,28 +1279,6 @@ pub trait InPlaceWrite { fn write_pin_init(self, init: impl PinInit) -> Result, E>; } -impl InPlaceWrite for Box> { - type Initialized = Box; - - fn write_init(mut self, init: impl Init) -> Result { - let slot = self.as_mut_ptr(); - // SAFETY: When init errors/panics, slot will get deallocated but not dropped, - // slot is valid. - unsafe { init.__init(slot)? }; - // SAFETY: All fields have been initialized. - Ok(unsafe { self.assume_init() }) - } - - fn write_pin_init(mut self, init: impl PinInit) -> Result, E> { - let slot = self.as_mut_ptr(); - // SAFETY: When init errors/panics, slot will get deallocated but not dropped, - // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; - // SAFETY: All fields have been initialized. - Ok(unsafe { self.assume_init() }.into()) - } -} - impl InPlaceWrite for UniqueArc> { type Initialized = UniqueArc; @@ -1453,7 +1410,7 @@ impl_zeroable! { // // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant. {} Option>, - {} Option>, + {} Option>, // SAFETY: `null` pointer is valid. // diff --git a/rust/kernel/init/__internal.rs b/rust/kernel/init/__internal.rs index 44431fba7aabf7..74329cc3262c05 100644 --- a/rust/kernel/init/__internal.rs +++ b/rust/kernel/init/__internal.rs @@ -105,7 +105,7 @@ pub unsafe trait InitData: Copy { } } -pub struct AllData(PhantomData) -> Box>); +pub struct AllData(PhantomData) -> KBox>); impl Clone for AllData { fn clone(&self) -> Self { diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs new file mode 100644 index 00000000000000..750af938f83e10 --- /dev/null +++ b/rust/kernel/io.rs @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory-mapped IO. +//! +//! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h) + +use crate::error::{code::EINVAL, Result}; +use crate::{bindings, build_assert}; + +/// IO-mapped memory, starting at the base address @addr and spanning @maxlen bytes. +/// +/// The creator (usually a subsystem / bus such as PCI) is responsible for creating the +/// mapping, performing an additional region request etc. +/// +/// # Invariant +/// +/// `addr` is the start and `maxsize` the length of valid I/O mapped memory region of size +/// `maxsize`. +/// +/// # Examples +/// +/// ```no_run +/// # use kernel::{bindings, io::Io}; +/// # use core::ops::Deref; +/// +/// // See also [`pci::Bar`] for a real example. +/// struct IoMem(Io); +/// +/// impl IoMem { +/// /// # Safety +/// /// +/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs +/// /// virtual address space. +/// unsafe fn new(paddr: usize) -> Result{ +/// +/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is +/// // valid for `ioremap`. +/// let addr = unsafe { bindings::ioremap(paddr as _, SIZE.try_into().unwrap()) }; +/// if addr.is_null() { +/// return Err(ENOMEM); +/// } +/// +/// // SAFETY: `addr` is guaranteed to be the start of a valid I/O mapped memory region of +/// // size `SIZE`. +/// let io = unsafe { Io::new(addr as _, SIZE)? }; +/// +/// Ok(IoMem(io)) +/// } +/// } +/// +/// impl Drop for IoMem { +/// fn drop(&mut self) { +/// // SAFETY: Safe as by the invariant of `Io`. +/// unsafe { bindings::iounmap(self.0.base_addr() as _); }; +/// } +/// } +/// +/// impl Deref for IoMem { +/// type Target = Io; +/// +/// fn deref(&self) -> &Self::Target { +/// &self.0 +/// } +/// } +/// +///# fn no_run() -> Result<(), Error> { +/// // SAFETY: Invalid usage for example purposes. +/// let iomem = unsafe { IoMem::<{ core::mem::size_of::() }>::new(0xBAAAAAAD)? }; +/// iomem.writel(0x42, 0x0); +/// assert!(iomem.try_writel(0x42, 0x0).is_ok()); +/// assert!(iomem.try_writel(0x42, 0x4).is_err()); +/// # Ok(()) +/// # } +/// ``` +pub struct Io { + addr: usize, + maxsize: usize, +} + +macro_rules! define_read { + ($(#[$attr:meta])* $name:ident, $try_name:ident, $type_name:ty) => { + /// Read IO data from a given offset known at compile time. + /// + /// Bound checks are performed on compile time, hence if the offset is not known at compile + /// time, the build will fail. + $(#[$attr])* + #[inline] + pub fn $name(&self, offset: usize) -> $type_name { + let addr = self.io_addr_assert::<$type_name>(offset); + + // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. + unsafe { bindings::$name(addr as _) } + } + + /// Read IO data from a given offset. + /// + /// Bound checks are performed on runtime, it fails if the offset (plus the type size) is + /// out of bounds. + $(#[$attr])* + pub fn $try_name(&self, offset: usize) -> Result<$type_name> { + let addr = self.io_addr::<$type_name>(offset)?; + + // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. + Ok(unsafe { bindings::$name(addr as _) }) + } + }; +} + +macro_rules! define_write { + ($(#[$attr:meta])* $name:ident, $try_name:ident, $type_name:ty) => { + /// Write IO data from a given offset known at compile time. + /// + /// Bound checks are performed on compile time, hence if the offset is not known at compile + /// time, the build will fail. + $(#[$attr])* + #[inline] + pub fn $name(&self, value: $type_name, offset: usize) { + let addr = self.io_addr_assert::<$type_name>(offset); + + // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. + unsafe { bindings::$name(value, addr as _, ) } + } + + /// Write IO data from a given offset. + /// + /// Bound checks are performed on runtime, it fails if the offset (plus the type size) is + /// out of bounds. + $(#[$attr])* + pub fn $try_name(&self, value: $type_name, offset: usize) -> Result { + let addr = self.io_addr::<$type_name>(offset)?; + + // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. + unsafe { bindings::$name(value, addr as _) } + Ok(()) + } + }; +} + +impl Io { + /// + /// + /// # Safety + /// + /// Callers must ensure that `addr` is the start of a valid I/O mapped memory region of size + /// `maxsize`. + pub unsafe fn new(addr: usize, maxsize: usize) -> Result { + if maxsize < SIZE { + return Err(EINVAL); + } + + // INVARIANT: Covered by the safety requirements of this function. + Ok(Self { addr, maxsize }) + } + + /// Returns the base address of this mapping. + #[inline] + pub fn base_addr(&self) -> usize { + self.addr + } + + /// Returns the size of this mapping. + #[inline] + pub fn maxsize(&self) -> usize { + self.maxsize + } + + #[inline] + const fn offset_valid(offset: usize, size: usize) -> bool { + let type_size = core::mem::size_of::(); + if let Some(end) = offset.checked_add(type_size) { + end <= size && offset % type_size == 0 + } else { + false + } + } + + #[inline] + fn io_addr(&self, offset: usize) -> Result { + if !Self::offset_valid::(offset, self.maxsize()) { + return Err(EINVAL); + } + + // Probably no need to check, since the safety requirements of `Self::new` guarantee that + // this can't overflow. + self.base_addr().checked_add(offset).ok_or(EINVAL) + } + + #[inline] + fn io_addr_assert(&self, offset: usize) -> usize { + build_assert!(Self::offset_valid::(offset, SIZE)); + + self.base_addr() + offset + } + + define_read!(readb, try_readb, u8); + define_read!(readw, try_readw, u16); + define_read!(readl, try_readl, u32); + define_read!( + #[cfg(CONFIG_64BIT)] + readq, + try_readq, + u64 + ); + + define_read!(readb_relaxed, try_readb_relaxed, u8); + define_read!(readw_relaxed, try_readw_relaxed, u16); + define_read!(readl_relaxed, try_readl_relaxed, u32); + define_read!( + #[cfg(CONFIG_64BIT)] + readq_relaxed, + try_readq_relaxed, + u64 + ); + + define_write!(writeb, try_writeb, u8); + define_write!(writew, try_writew, u16); + define_write!(writel, try_writel, u32); + define_write!( + #[cfg(CONFIG_64BIT)] + writeq, + try_writeq, + u64 + ); + + define_write!(writeb_relaxed, try_writeb_relaxed, u8); + define_write!(writew_relaxed, try_writew_relaxed, u16); + define_write!(writel_relaxed, try_writel_relaxed, u32); + define_write!( + #[cfg(CONFIG_64BIT)] + writeq_relaxed, + try_writeq_relaxed, + u64 + ); +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index dc37aef6a0085a..e4ffa8d2074ccc 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -13,11 +13,18 @@ #![no_std] #![feature(arbitrary_self_types)] +#![feature(associated_type_defaults)] #![feature(coerce_unsized)] +#![feature(const_refs_to_cell)] #![feature(dispatch_from_dyn)] +#![feature(inline_const)] #![feature(lint_reasons)] -#![feature(new_uninit)] #![feature(unsize)] +#![allow(stable_features)] +// Stable in Rust 1.83 +#![feature(const_mut_refs)] +#![feature(const_ptr_write)] +#![feature(const_maybe_uninit_as_mut_ptr)] // Ensure conditional compilation based on the kernel configuration works; // otherwise we may silently break things like initcall handling. @@ -28,13 +35,19 @@ compile_error!("Missing kernel configuration for conditional compilation"); extern crate self as kernel; pub mod alloc; +pub mod bits; #[cfg(CONFIG_BLOCK)] pub mod block; mod build_assert; pub mod device; +pub mod device_id; +pub mod devres; +pub mod driver; pub mod error; #[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)] pub mod firmware; +#[cfg(CONFIG_I2C)] +pub mod i2c; pub mod init; pub mod ioctl; #[cfg(CONFIG_KUNIT)] @@ -42,10 +55,16 @@ pub mod kunit; pub mod list; #[cfg(CONFIG_NET)] pub mod net; +pub mod of; pub mod page; pub mod prelude; pub mod print; pub mod rbtree; +#[cfg(CONFIG_REGMAP)] +pub mod regmap; +#[cfg(CONFIG_REGULATOR)] +pub mod regulator; +pub mod revocable; pub mod sizes; mod static_assert; #[doc(hidden)] @@ -61,6 +80,7 @@ pub mod workqueue; #[doc(hidden)] pub use bindings; +pub mod io; pub use macros; pub use uapi; @@ -80,7 +100,36 @@ pub trait Module: Sized + Sync + Send { /// should do. /// /// Equivalent to the `module_init` macro in the C API. - fn init(module: &'static ThisModule) -> error::Result; + fn init(name: &'static str::CStr, module: &'static ThisModule) -> error::Result; +} + +/// A module that is pinned and initialised in-place. +pub trait InPlaceModule: Sync + Send { + /// Creates an initialiser for the module. + /// + /// It is called when the module is loaded. + fn init( + name: &'static str::CStr, + module: &'static ThisModule, + ) -> impl init::PinInit; +} + +impl InPlaceModule for T { + fn init( + name: &'static str::CStr, + module: &'static ThisModule, + ) -> impl init::PinInit { + let initer = move |slot: *mut Self| { + let m = ::init(name, module)?; + + // SAFETY: `slot` is valid for write per the contract with `pin_init_from_closure`. + unsafe { slot.write(m) }; + Ok(()) + }; + + // SAFETY: On success, `initer` always fully initialises an instance of `Self`. + unsafe { init::pin_init_from_closure(initer) } + } } /// Equivalent to `THIS_MODULE` in the C API. diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs index 910ce867480a8a..26631b0e127ac0 100644 --- a/rust/kernel/net/phy.rs +++ b/rust/kernel/net/phy.rs @@ -899,7 +899,7 @@ macro_rules! module_phy_driver { [$($crate::net::phy::create_phy_driver::<$driver>()),+]; impl $crate::Module for Module { - fn init(module: &'static ThisModule) -> Result { + fn init(_name: &'static CStr, module: &'static ThisModule) -> Result { // SAFETY: The anonymous constant guarantees that nobody else can access // the `DRIVERS` static. The array is used only in the C side. let drivers = unsafe { &mut DRIVERS }; diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs new file mode 100644 index 00000000000000..85b5ccde408160 --- /dev/null +++ b/rust/kernel/of.rs @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Devicetree and Open Firmware abstractions. +//! +//! C header: [`include/linux/of_*.h`](../../../../include/linux/of_*.h) + +use crate::{ + bindings, device_id, + str::{BStr, CStr}, +}; + +/// An open firmware device id. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::of_device_id); + +impl DeviceId { + /// Create a OF `DeviceId` from a compatible string. + pub const fn with_compatible(compatible: &CStr) -> Self { + let device_id = core::mem::MaybeUninit::::zeroed(); + let mut device_id = unsafe { device_id.assume_init() }; + + let compatible = BStr::from_bytes(compatible.as_bytes_with_nul()); + assert!(compatible.len() <= device_id.compatible.len()); + + let mut i = 0; + while i < compatible.len() { + device_id.compatible[i] = compatible.deref_const()[i] as _; + i += 1; + } + + Self(device_id) + } +} + +// SAFETY: `ZERO` is all zeroed-out and `to_rawid` stores `offset` in `of_device_id::data`. +unsafe impl device_id::RawDeviceId for DeviceId { + type RawType = bindings::of_device_id; + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::of_device_id, data); +} + +/// Alias for `device_id::IdTable` containing OF's `DeviceId` +pub type IdTable = &'static dyn device_id::IdTable; + +/// Create an OF `IdTable` with its alias for modpost. +#[macro_export] +macro_rules! of_device_table { + ($module_table_name:ident, $table_name:ident, $id_info_type: ty, $table_data: expr) => { + const $table_name: $crate::device_id::IdArray< + $crate::of::DeviceId, + $id_info_type, + { $table_data.len() }, + > = $crate::device_id::IdArray::new($table_data); + + $crate::module_device_table!("of", $module_table_name, $table_name); + }; +} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 4571daec0961bb..9ab4e0b6cbc912 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -14,10 +14,7 @@ #[doc(no_inline)] pub use core::pin::Pin; -pub use crate::alloc::{box_ext::BoxExt, flags::*, vec_ext::VecExt}; - -#[doc(no_inline)] -pub use alloc::{boxed::Box, vec::Vec}; +pub use crate::alloc::{flags::*, Box, KBox, KVBox, KVVec, KVec, VBox, VVec, Vec}; #[doc(no_inline)] pub use macros::{module, pin_data, pinned_drop, vtable, Zeroable}; @@ -27,6 +24,8 @@ pub use super::build_assert; // `super::std_vendor` is hidden, which makes the macro inline for some reason. #[doc(no_inline)] pub use super::dbg; +pub use super::fmt; +pub use super::{dev_alert, dev_crit, dev_dbg, dev_emerg, dev_err, dev_info, dev_notice, dev_warn}; pub use super::{pr_alert, pr_crit, pr_debug, pr_emerg, pr_err, pr_info, pr_notice, pr_warn}; pub use super::{init, pin_init, try_init, try_pin_init}; diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs index d03e4aa1f4812b..cb4415a1225825 100644 --- a/rust/kernel/rbtree.rs +++ b/rust/kernel/rbtree.rs @@ -7,7 +7,6 @@ //! Reference: use crate::{alloc::Flags, bindings, container_of, error::Result, prelude::*}; -use alloc::boxed::Box; use core::{ cmp::{Ord, Ordering}, marker::PhantomData, @@ -497,7 +496,7 @@ impl Drop for RBTree { // but it is not observable. The loop invariant is still maintained. // SAFETY: `this` is valid per the loop invariant. - unsafe { drop(Box::from_raw(this.cast_mut())) }; + unsafe { drop(KBox::from_raw(this.cast_mut())) }; } } } @@ -764,7 +763,7 @@ impl<'a, K, V> Cursor<'a, K, V> { // point to the links field of `Node` objects. let this = unsafe { container_of!(self.current.as_ptr(), Node, links) }.cast_mut(); // SAFETY: `this` is valid by the type invariants as described above. - let node = unsafe { Box::from_raw(this) }; + let node = unsafe { KBox::from_raw(this) }; let node = RBTreeNode { node }; // SAFETY: The reference to the tree used to create the cursor outlives the cursor, so // the tree cannot change. By the tree invariant, all nodes are valid. @@ -809,7 +808,7 @@ impl<'a, K, V> Cursor<'a, K, V> { // point to the links field of `Node` objects. let this = unsafe { container_of!(neighbor, Node, links) }.cast_mut(); // SAFETY: `this` is valid by the type invariants as described above. - let node = unsafe { Box::from_raw(this) }; + let node = unsafe { KBox::from_raw(this) }; return Some(RBTreeNode { node }); } None @@ -1038,7 +1037,7 @@ impl Iterator for IterRaw { /// It contains the memory needed to hold a node that can be inserted into a red-black tree. One /// can be obtained by directly allocating it ([`RBTreeNodeReservation::new`]). pub struct RBTreeNodeReservation { - node: Box>>, + node: KBox>>, } impl RBTreeNodeReservation { @@ -1046,7 +1045,7 @@ impl RBTreeNodeReservation { /// call to [`RBTree::insert`]. pub fn new(flags: Flags) -> Result> { Ok(RBTreeNodeReservation { - node: as BoxExt<_>>::new_uninit(flags)?, + node: KBox::new_uninit(flags)?, }) } } @@ -1062,14 +1061,15 @@ impl RBTreeNodeReservation { /// Initialises a node reservation. /// /// It then becomes an [`RBTreeNode`] that can be inserted into a tree. - pub fn into_node(mut self, key: K, value: V) -> RBTreeNode { - self.node.write(Node { - key, - value, - links: bindings::rb_node::default(), - }); - // SAFETY: We just wrote to it. - let node = unsafe { self.node.assume_init() }; + pub fn into_node(self, key: K, value: V) -> RBTreeNode { + let node = KBox::write( + self.node, + Node { + key, + value, + links: bindings::rb_node::default(), + }, + ); RBTreeNode { node } } } @@ -1079,7 +1079,7 @@ impl RBTreeNodeReservation { /// The node is fully initialised (with key and value) and can be inserted into a tree without any /// extra allocations or failure paths. pub struct RBTreeNode { - node: Box>, + node: KBox>, } impl RBTreeNode { @@ -1091,7 +1091,9 @@ impl RBTreeNode { /// Get the key and value from inside the node. pub fn to_key_value(self) -> (K, V) { - (self.node.key, self.node.value) + let node = KBox::into_inner(self.node); + + (node.key, node.value) } } @@ -1113,7 +1115,7 @@ impl RBTreeNode { /// may be freed (but only for the key/value; memory for the node itself is kept for reuse). pub fn into_reservation(self) -> RBTreeNodeReservation { RBTreeNodeReservation { - node: Box::drop_contents(self.node), + node: KBox::drop_contents(self.node), } } } @@ -1164,7 +1166,7 @@ impl<'a, K, V> RawVacantEntry<'a, K, V> { /// The `node` must have a key such that inserting it here does not break the ordering of this /// [`RBTree`]. fn insert(self, node: RBTreeNode) -> &'a mut V { - let node = Box::into_raw(node.node); + let node = KBox::into_raw(node.node); // SAFETY: `node` is valid at least until we call `Box::from_raw`, which only happens when // the node is removed or replaced. @@ -1238,21 +1240,24 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> { // SAFETY: The node was a node in the tree, but we removed it, so we can convert it // back into a box. node: unsafe { - Box::from_raw(container_of!(self.node_links, Node, links).cast_mut()) + KBox::from_raw(container_of!(self.node_links, Node, links).cast_mut()) }, } } /// Takes the value of the entry out of the map, and returns it. pub fn remove(self) -> V { - self.remove_node().node.value + let rb_node = self.remove_node(); + let node = KBox::into_inner(rb_node.node); + + node.value } /// Swap the current node for the provided node. /// /// The key of both nodes must be equal. fn replace(self, node: RBTreeNode) -> RBTreeNode { - let node = Box::into_raw(node.node); + let node = KBox::into_raw(node.node); // SAFETY: `node` is valid at least until we call `Box::from_raw`, which only happens when // the node is removed or replaced. @@ -1268,7 +1273,7 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> { // - `self.node_ptr` produces a valid pointer to a node in the tree. // - Now that we removed this entry from the tree, we can convert the node to a box. let old_node = - unsafe { Box::from_raw(container_of!(self.node_links, Node, links).cast_mut()) }; + unsafe { KBox::from_raw(container_of!(self.node_links, Node, links).cast_mut()) }; RBTreeNode { node: old_node } } diff --git a/rust/kernel/regmap.rs b/rust/kernel/regmap.rs new file mode 100644 index 00000000000000..2074e6c789a6c8 --- /dev/null +++ b/rust/kernel/regmap.rs @@ -0,0 +1,874 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Register map access API. +//! +//! C header: [`include/linux/regmap.h`](srctree/include/linux/regmap.h) + +#[cfg(CONFIG_REGMAP_I2C = "y")] +use crate::i2c; +use crate::{ + bindings, + error::{code::*, to_result, Error, Result}, + macros::paste, + sync::Arc, +}; +use core::{marker::PhantomData, mem::MaybeUninit}; + +/// Type of caching +#[repr(u32)] +pub enum CacheType { + /// Don't cache anything + None = bindings::regcache_type_REGCACHE_NONE, + /// Use RbTree caching + RbTree = bindings::regcache_type_REGCACHE_RBTREE, + /// Use Flat caching + Flat = bindings::regcache_type_REGCACHE_FLAT, + /// Use Maple caching + Maple = bindings::regcache_type_REGCACHE_MAPLE, +} + +/// Register map +/// +/// # Examples +/// +/// ``` +/// let regmap = Regmap::init_i2c(i2c, &config); +/// ``` +pub struct Regmap(*mut bindings::regmap); + +impl Regmap { + #[cfg(CONFIG_REGMAP_I2C = "y")] + /// Initialize a [`Regmap`] instance for an `i2c` device. + pub fn init_i2c(i2c: &i2c::Client, config: &Config) -> Self { + let regmap = unsafe { bindings::regmap_init_i2c(i2c.raw_client(), &config.raw) }; + + Self(regmap) + } + + /// Allocate regmap [`Fields`] + /// + /// This function allocate regmap fields from the `reg_fields` descriptors + pub fn alloc_fields( + self: &Arc, + descs: &'static FieldDescs, + ) -> Result> { + let mut rm_fields = [core::ptr::null_mut(); N]; + to_result(unsafe { + bindings::regmap_field_bulk_alloc( + self.0, + &mut rm_fields[0], + descs.0.as_ptr(), + descs.0.len() as i32, + ) + })?; + + Ok(Fields { + rm_fields, + _regmap: self.clone(), + }) + } + + /// Return the raw pointer of this regmap + pub fn as_ptr(&self) -> *mut bindings::regmap { + self.0 + } +} + +impl Drop for Regmap { + fn drop(&mut self) { + unsafe { bindings::regmap_exit(self.0) } + } +} + +/// Field Descriptors +/// +/// FieldDescriptors can be created by calling the [`define_regmap_field_descs`] macro. +pub struct FieldDescs([bindings::reg_field; N]); + +impl FieldDescs { + // macro use only + #[doc(hidden)] + pub const fn new(fields: [bindings::reg_field; N]) -> Self { + Self(fields) + } + + /// Number of fields being held by `FieldDescs` + /// + /// This function should be used to retrieve the number of fields that were + /// created when calling [`define_regmap_field_descs`]. + /// + /// # Examples + /// + /// ``` + /// use kernel::regmap::{define_regmap_field_descs, Fields}; + /// + /// define_regmap_field_descs!(DESCS, { + /// {pid, 0x3, kernel::regmap::access::READ, { value => raw([7:0], ro) }}, + /// }); + /// + /// struct Registrations { + /// fields: Fields<{ DESCS.count() }>, + /// } + /// ``` + pub const fn count(&self) -> usize { + N + } +} + +/// Regmap fields +/// +/// # Invariants +/// +/// `rm_fields` is garanteed to contains valid and initialized `regmap_field`s. +/// +pub struct Fields { + rm_fields: [*mut bindings::regmap_field; N], + + // Each regmap_field hold a pointer to the `struct regmap` instance, so we need to keep a copy + // of the wrapper around. + _regmap: Arc, +} +impl Fields { + /// Get field `index` + pub fn index(&mut self, index: usize) -> *mut bindings::regmap_field { + self.rm_fields[index] + } + + // macro use only + #[doc(hidden)] + pub fn read(&mut self, index: usize) -> Result { + let mut val = 0; + + // Make sure we don't panic if the index is out of bound. + if index >= N { + return Err(EINVAL); + } + + // SAFETY: By the type invariants, we are garanteed that all rm_fields entries point + // to valid and initialized values, hence it is safe to make this FFI call. + let ret = unsafe { bindings::regmap_field_read(self.rm_fields[index], &mut val) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(val) + } +} + +unsafe impl Send for Fields {} + +/// Helper macro for [`Config`] to create methods to set a fields from [`regmap_config`] +/// +/// The following code will create a method named `with_max_register`: +/// ``` +/// config_with!(max_register: u32); +/// ``` +macro_rules! config_with { + ($(#[$meta:meta])* $name:ident: $type:ty) => { + config_with!($(#[$meta])* $name: $type, $name); + }; + + ($(#[$meta:meta])* $name:ident: $type:ty, $e:expr) => { + paste! { + $(#[$meta])* + pub const fn [](mut self, $name: $type) -> Self { + self.raw.$name = $e; + self + } + } + }; +} + +// macro use only +#[doc(hidden)] +pub trait ConfigOps { + fn is_readable_reg(reg: u32) -> bool; + fn is_writeable_reg(reg: u32) -> bool; + fn is_volatile_reg(reg: u32) -> bool; + fn is_precious_reg(reg: u32) -> bool; +} + +/// Regmap Configuration +pub struct Config { + raw: bindings::regmap_config, + _phantom: PhantomData, +} +impl Config { + /// Create a new regmap Config + pub const fn new(reg_bits: i32, val_bits: i32) -> Self { + let cfg = MaybeUninit::::zeroed(); + let mut cfg = unsafe { cfg.assume_init() }; + + cfg.reg_bits = reg_bits; + cfg.val_bits = val_bits; + cfg.writeable_reg = Some(Self::writeable_reg_callback); + cfg.readable_reg = Some(Self::readable_reg_callback); + cfg.volatile_reg = Some(Self::volatile_reg_callback); + cfg.precious_reg = Some(Self::precious_reg_callback); + + Self { + raw: cfg, + _phantom: PhantomData, + } + } + + config_with!( + /// Specifies the maximum valid register address. + max_register: u32 + ); + + config_with!( + /// Type of caching being performed. + cache_type: CacheType, cache_type as _ + ); + + unsafe extern "C" fn writeable_reg_callback(_dev: *mut bindings::device, reg: u32) -> bool { + T::is_writeable_reg(reg) + } + + unsafe extern "C" fn readable_reg_callback(_dev: *mut bindings::device, reg: u32) -> bool { + T::is_readable_reg(reg) + } + + unsafe extern "C" fn volatile_reg_callback(_dev: *mut bindings::device, reg: u32) -> bool { + T::is_volatile_reg(reg) + } + + unsafe extern "C" fn precious_reg_callback(_dev: *mut bindings::device, reg: u32) -> bool { + T::is_precious_reg(reg) + } +} + +/// Definitions describing how registers can be accessed. +pub mod access { + /// Register can be read from. + pub const READ: u32 = 0b000001; + /// Register can be written to. + pub const WRITE: u32 = 0b000010; + /// Register should not be read outside of a call from the driver. + pub const PRECIOUS: u32 = 0b000100; + /// Register value can't be cached. + pub const VOLATILE: u32 = 0b001000; + + /// Register can be read from and written to. + pub const RW: u32 = READ | WRITE; +} + +// macro use only +#[doc(hidden)] +#[macro_export] +macro_rules! regmap_check_access { + ($type:ident, $access:expr, $reg:ident, $addr:literal) => { + if kernel::regmap::access::$type & $access > 0 && $reg == $addr { + return true; + } + }; +} +// macro use only +#[doc(hidden)] +pub use regmap_check_access; + +/// Common operations for all field types +pub trait FieldCommonOps { + /// Get the Mask for the field + fn mask() -> u32; +} + +/// Read operations for fields with `bit` type +pub trait BitFieldReadOps { + /// Returns whether the bit is set + fn is_set(fields: &mut Fields) -> Result; +} + +/// Write operations for fields with `bit` type +pub trait BitFieldWriteOps { + /// Set the bit + fn set(fields: &mut Fields) -> Result; + + /// Force set the bit + fn force_set(fields: &mut Fields) -> Result; + + /// Clear the bit + fn clear(fields: &mut Fields) -> Result; + + /// Force clear the bit + fn force_clear(fields: &mut Fields) -> Result; +} + +/// Read operations for fields with `enum` type +pub trait EnumFieldReadOps { + #[doc(hidden)] + /// Underlying enum type reprensenting the field values + type EnumType; + + /// Read the field + fn read(fields: &mut Fields) -> Result; +} + +/// Write operations for fields with `enum` type +pub trait EnumFieldWriteOps { + #[doc(hidden)] + /// Underlying enum type reprensenting the field values + type EnumType; + + /// Write the field + fn write(fields: &mut Fields, val: Self::EnumType) -> Result; + + /// Force write the field + fn force_write(fields: &mut Fields, val: Self::EnumType) -> Result; +} + +/// Read operations for fields with `raw` type +pub trait RawFieldReadOps { + /// Read the field + fn read(fields: &mut Fields) -> Result; + + /// Test the field bits + fn test_bits(fields: &mut Fields, bits: core::ffi::c_uint) -> Result; +} + +/// Write operations for fields with `raw` type +pub trait RawFieldWriteOps { + /// Write the field + fn write(fields: &mut Fields, val: core::ffi::c_uint) -> Result; + + /// Force write the field + fn force_write(fields: &mut Fields, val: core::ffi::c_uint) -> Result; + + /// Update the field using a mask + fn update_bits( + fields: &mut Fields, + mask: core::ffi::c_uint, + val: core::ffi::c_uint, + ) -> Result; + + /// Force update the field using a mask + fn force_update_bits( + fields: &mut Fields, + mask: core::ffi::c_uint, + val: core::ffi::c_uint, + ) -> Result; + + /// Set field bits + fn set_bits(fields: &mut Fields, bits: core::ffi::c_uint) -> Result; + + /// Clear the field bits + fn clear_bits(fields: &mut Fields, bits: core::ffi::c_uint) -> Result; +} + +// macro use only +#[doc(hidden)] +#[macro_export] +macro_rules! regmap_field_bit { + ($field_name:ident, $access: expr, $reg:literal, $pos:literal, rw) => { + kernel::static_assert!($access & kernel::regmap::access::RW == kernel::regmap::access::RW); + + $crate::regmap_field_bit!($field_name, $reg, $pos, reserved); + $crate::regmap_field_bit!($field_name, _ro); + $crate::regmap_field_bit!($field_name, _wo); + }; + + ($field_name:ident, $access: expr, $reg:literal, $pos:literal, ro) => { + kernel::static_assert!( + $access & kernel::regmap::access::READ == kernel::regmap::access::READ + ); + + $crate::regmap_field_bit!($field_name, $reg, $pos, reserved); + $crate::regmap_field_bit!($field_name, _ro); + }; + + ($field_name:ident, $access: expr, $reg:literal, $pos:literal, wo) => { + kernel::static_assert!( + $access & kernel::regmap::access::WRITE == kernel::regmap::access::WRITE + ); + + $crate::regmap_field_bit!($field_name, $reg, $pos, reserved); + $crate::regmap_field_bit!($field_name, _wo); + }; + + ($field_name:ident, $reg:literal, $pos:literal, reserved) => { + kernel::macros::paste! { + struct [<_Bit $pos >]; + } + + impl $field_name { + pub(crate) const fn reg_field() -> bindings::reg_field { + bindings::reg_field { + reg: $reg, + lsb: $pos, + msb: $pos + 1, + id_offset: 0, + id_size: 0, + } + } + + #[allow(dead_code)] + pub(crate) const fn mask() -> u32 { + kernel::bits::genmask($pos, $pos) + } + } + }; + + ($field_name:ident, _ro) => { + impl super::BitFieldReadOps for $field_name { + fn is_set(fields: &mut regmap::Fields) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + let mut val: core::ffi::c_uint = 0; + kernel::error::to_result(unsafe { bindings::regmap_field_read(field, &mut val) })?; + Ok(val == 1) + } + } + }; + + ($field_name:ident, _wo) => { + impl super::BitFieldWriteOps for $field_name { + fn set(fields: &mut regmap::Fields) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_write(field, 1) }) + } + + fn force_set(fields: &mut regmap::Fields) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_force_write(field, 1) }) + } + + fn clear(fields: &mut regmap::Fields) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_write(field, 0) }) + } + + fn force_clear(fields: &mut regmap::Fields) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_force_write(field, 0) }) + } + } + }; +} + +// macro use only +#[doc(hidden)] +#[macro_export] +macro_rules! regmap_field_enum { + ($field_name:ident, $access: expr, $reg:literal, [$msb:literal:$lsb:literal], ro, { + $($k:ident = $v:literal,)+ }) => { + kernel::static_assert!( + $access & kernel::regmap::access::READ == kernel::regmap::access::READ + ); + + $crate::regmap_field_enum!($field_name, $reg, [$msb:$lsb], reserved, { $($k = $v,)+ }); + $crate::regmap_field_enum!($field_name, _ro); + }; + + ($field_name:ident, $access: expr, $reg:literal, [$msb:literal:$lsb:literal], rw, { + $($k:ident = $v:literal,)+ }) => { + kernel::static_assert!($access & kernel::regmap::access::RW == kernel::regmap::access::RW); + + $crate::regmap_field_enum!($field_name, $reg, [$msb:$lsb], reserved, { $($k = $v,)+ }); + $crate::regmap_field_enum!($field_name, _ro); + $crate::regmap_field_enum!($field_name, _wo); + }; + + ($field_name:ident, $access: expr, $reg:literal, [$msb:literal:$lsb:literal], wo, { + $($k:ident = $v:literal,)+ }) => { + kernel::static_assert!( + $access & kernel::regmap::access::WRITE == kernel::regmap::access::WRITE + ); + + $crate::regmap_field_enum!($field_name, $reg, [$msb:$lsb], reserved, { $($k = $v,)+ }); + $crate::regmap_field_enum!($field_name, _wo); + }; + + ($field_name:ident, $reg:literal, [$msb:literal:$lsb:literal], reserved, { + $($k:ident = $v:literal,)+ }) => { + kernel::macros::foreach!(i in $lsb..=$msb { + kernel::macros::paste! { + struct [<_Bit $i>]; + } + }); + + kernel::macros::paste! { + #[repr(u32)] + #[allow(non_camel_case_types)] + pub(crate) enum [<$field_name _enum>] { + $($k = $v,)+ + } + + impl TryFrom for [<$field_name _enum>] { + type Error = kernel::error::Error; + + fn try_from(raw_value: core::ffi::c_uint) -> Result { + match raw_value { + $($v => Ok(Self::$k),)+ + _ => Err(kernel::error::code::EINVAL), + } + } + } + + impl $field_name { + pub(crate) const fn reg_field() -> bindings::reg_field { + bindings::reg_field { + reg: $reg, + lsb: $lsb, + msb: $msb, + id_offset: 0, + id_size: 0, + } + } + + #[allow(dead_code)] + pub(crate) const fn mask() -> u32 { + kernel::bits::genmask($msb, $lsb) + } + } + } + }; + + ($field_name:ident, _ro) => { + impl super::EnumFieldReadOps for $field_name { + type EnumType = kernel::macros::paste! {[<$field_name _enum>]}; + + fn read(fields: &mut regmap::Fields) -> Result { + Self::EnumType::try_from(fields.read(Self::id() as usize)?) + } + } + }; + + ($field_name:ident, _wo) => { + impl super::EnumFieldWriteOps for $field_name { + type EnumType = kernel::macros::paste! {[<$field_name _enum>]}; + + fn write( + fields: &mut regmap::Fields, + val: Self::EnumType + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + let ret = unsafe { bindings::regmap_field_write(field, val as _) }; + kernel::error::to_result(ret) + } + + fn force_write( + fields: &mut regmap::Fields, + val: Self::EnumType + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + let ret = unsafe { bindings::regmap_field_force_write(field, val as _) }; + kernel::error::to_result(ret) + } + } + }; +} + +// macro use only +#[doc(hidden)] +#[macro_export] +macro_rules! regmap_field_raw { + ($field_name:ident, $access: expr, $reg:literal, [$msb:literal:$lsb:literal], rw) => { + kernel::static_assert!($access & kernel::regmap::access::RW == kernel::regmap::access::RW); + + $crate::regmap_field_raw!($field_name, $reg, [$msb:$lsb], reserved); + $crate::regmap_field_raw!($field_name, $reg, [$msb:$lsb], _ro); + $crate::regmap_field_raw!($field_name, $reg, [$msb:$lsb], _wo); + }; + + ($field_name:ident, $access: expr, $reg:literal, [$msb:literal:$lsb:literal], ro) => { + kernel::static_assert!( + $access & kernel::regmap::access::READ == kernel::regmap::access::READ + ); + + $crate::regmap_field_raw!($field_name, $reg, [$msb:$lsb], reserved); + $crate::regmap_field_raw!($field_name, $reg, [$msb:$lsb], _ro); + }; + + ($field_name:ident, $access: expr, $reg:literal, [$msb:literal:$lsb:literal], wo) => { + kernel::static_assert!( + $access & kernel::regmap::access::WRITE == kernel::regmap::access::WRITE + ); + + $crate::regmap_field_raw!($field_name, $reg, [$msb:$lsb], reserved); + $crate::regmap_field_raw!($field_name, $reg, [$msb:$lsb], _wo); + }; + + ($field_name:ident, $reg:literal, [$msb:literal:$lsb:literal], reserved) => { + kernel::macros::foreach!(i in $lsb..=$msb { + kernel::macros::paste! { + struct [<_Bit $i>]; + } + }); + + impl $field_name { + pub(crate) const fn reg_field() -> bindings::reg_field { + bindings::reg_field { + reg: $reg, + lsb: $lsb, + msb: $msb, + id_offset: 0, + id_size: 0, + } + } + + #[allow(dead_code)] + pub(crate) const fn mask() -> u32 { + kernel::bits::genmask($msb, $lsb) + } + } + }; + + ($field_name:ident, $reg:literal, [$msb:literal:$lsb:literal], _ro) => { + impl super::RawFieldReadOps for $field_name { + fn read(fields: &mut regmap::Fields) -> Result { + fields.read(Self::id() as usize) + } + + fn test_bits( + fields: &mut regmap::Fields, + bits: core::ffi::c_uint, + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_test_bits(field, bits) }) + } + } + }; + + ($field_name:ident, $reg:literal, [$msb:literal:$lsb:literal], _wo) => { + impl super::RawFieldWriteOps for $field_name { + fn write( + fields: &mut regmap::Fields, + val: core::ffi::c_uint, + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_write(field, val as _) }) + } + + fn force_write( + fields: &mut regmap::Fields, + val: core::ffi::c_uint, + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { + bindings::regmap_field_force_write(field, val as _) + }) + } + + fn update_bits( + fields: &mut regmap::Fields, + mask: core::ffi::c_uint, + val: core::ffi::c_uint, + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { + bindings::regmap_field_update_bits(field, mask, val) + }) + } + + fn force_update_bits( + fields: &mut regmap::Fields, + mask: core::ffi::c_uint, + val: core::ffi::c_uint, + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { + bindings::regmap_field_force_update_bits(field, mask, val) + }) + } + + fn set_bits( + fields: &mut regmap::Fields, + bits: core::ffi::c_uint, + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_set_bits(field, bits) }) + } + + fn clear_bits( + fields: &mut regmap::Fields, + bits: core::ffi::c_uint, + ) -> Result { + let field = unsafe { fields.index(Self::id() as usize) }; + kernel::error::to_result(unsafe { bindings::regmap_field_clear_bits(field, bits) }) + } + } + }; +} + +// macro use only +#[doc(hidden)] +#[macro_export] +macro_rules! regmap_fields { + ($type:ident, $reg:ident, $access:expr, $name:ident, $($t:tt)*) => { + kernel::macros::paste! { + #[allow(non_camel_case_types)] + pub(crate) struct $name; + + impl $name { + #[allow(dead_code)] + pub(crate) const fn id() -> super::Fields { + super::Fields::[<$reg _ $name>] + } + } + + $crate::[]!($name, $access, $($t)*); + } + }; +} + +// macro use only +#[doc(hidden)] +#[macro_export] +macro_rules! regmap_reg_field { + ($reg_name:ident, $field_name:ident) => { + register::$reg_name::$field_name::reg_field() + }; +} + +// macro use only +#[doc(hidden)] +#[macro_export] +macro_rules! regmap_count_fields { + () => { 0usize }; + ($type:ident $($rhs:ident)*) => { 1 + $crate::regmap_count_fields!($($rhs)*) }; +} + +/// Define regmap field descriptors +/// +/// # Syntax +/// +/// ```ignore +/// define_regmap_field_desc!(VAR_NAME, { , [, ...] }); +/// +/// : +/// (name, address, access_permission, { , [, ...] }) +/// +/// : +/// field_name => (...), +/// +/// : +/// * raw(, ) +/// * bit(, ) +/// * enum(, , { +/// EnumKind = , [EnumKind2 = , ...] +/// }) +/// +/// : +/// * ro: read-only +/// * rw: read-write +/// * wo: write-only +/// ``` +/// +/// # Examples +/// +/// ```ignore +/// regmap::define_regmap_field_descs!(FIELD_DESCS, { +/// (pid, 0x3, READ, { value => raw([7:0], ro) }), +/// (limconf, 0x16, RW, { +/// rearm => bit(0, rw), +/// rststatus => bit(1, rw), +/// tpwth => enum([5:4], rw, { +/// Temp83C = 0x0, +/// Temp94C = 0x1, +/// Temp105C = 0x2, +/// Temp116C = 0x3, +/// }), +/// }) +/// }); +/// +/// fn probe(client: &mut i2c::Client) -> Result { +/// # let config = regmap::Config::::new(8, 8) +/// # .with_max_register(0x16) +/// # .with_cache_type(regmap::CacheType::RbTree); +/// # let regmap = Arc::try_new(regmap::Regmap::init_i2c(client, &config))?; +/// # let mut fields = regmap.alloc_fields(&FIELD_DESCS)?; +/// // ... +/// dev_info!(client, "PID: {:#x}", pid::value::read(&mut fields)?); +/// // ... +/// } +/// ``` +#[macro_export] +macro_rules! define_regmap_field_descs { + ($name:ident, { + $(( + $reg_name:ident, $reg_addr:literal, $access:expr, { + $($field_name:ident => $type:ident($($x:tt),*)),* $(,)? + } + )),+ + }) => { + mod register { + use kernel::regmap::{ + access::*, + BitFieldReadOps, BitFieldWriteOps, + ConfigOps, + EnumFieldReadOps, EnumFieldWriteOps, + RawFieldReadOps, RawFieldWriteOps + }; + + kernel::macros::paste! { + $( + pub(crate) mod $reg_name { + use kernel::{bindings, error::{Result}, regmap::{self, access::*}}; + $( + $crate::regmap_fields!($type, $reg_name, $access, $field_name, + $reg_addr, $($x),*); + )* + + #[allow(dead_code)] + pub(crate) const fn addr() -> u32 { + $reg_addr + } + } + )+ + + #[repr(u32)] + #[allow(non_camel_case_types)] + pub(crate) enum Fields { + $($( + [<$reg_name _ $field_name>], + )*)+ + } + + pub(crate) struct AccessOps; + impl ConfigOps for AccessOps { + fn is_readable_reg(reg: u32) -> bool { + $( + kernel::regmap::regmap_check_access!(READ, $access, reg, $reg_addr); + )+ + + false + } + + fn is_writeable_reg(reg: u32) -> bool { + $( + kernel::regmap::regmap_check_access!(WRITE, $access, reg, $reg_addr); + )+ + + false + } + + fn is_volatile_reg(reg: u32) -> bool { + $( + kernel::regmap::regmap_check_access!(VOLATILE, $access, reg, $reg_addr); + )+ + + false + } + + fn is_precious_reg(reg: u32) -> bool { + $( + kernel::regmap::regmap_check_access!(PRECIOUS, $access, reg, $reg_addr); + )+ + + false + } + } + } + } + + const $name: regmap::FieldDescs<{$crate::regmap_count_fields!($($($type)*)+)}> = + regmap::FieldDescs::new([ + $( + $( + $crate::regmap_reg_field!($reg_name, $field_name) + ),* + ),+ + ]); + }; +} +pub use define_regmap_field_descs; diff --git a/rust/kernel/regulator.rs b/rust/kernel/regulator.rs new file mode 100644 index 00000000000000..c932794030e45f --- /dev/null +++ b/rust/kernel/regulator.rs @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! SoC Regulators + +pub mod consumer; +pub mod driver; + +use crate::{ + bindings, + error::{code::*, Error, Result}, +}; + +/// [`consumer::Regulator`] and [`driver::Regulator`] operating modes +#[derive(Copy, Clone)] +#[repr(u32)] +pub enum Mode { + /// Invalid mode + Invalid = bindings::REGULATOR_MODE_INVALID, + /// Regulator can handle fast changes in it's load + Fast = bindings::REGULATOR_MODE_FAST, + /// Normal regulator power supply mode + Normal = bindings::REGULATOR_MODE_NORMAL, + /// Regulator runs in a more efficient mode for light loads + Idle = bindings::REGULATOR_MODE_IDLE, + /// Regulator runs in the most efficient mode for very light loads + Standby = bindings::REGULATOR_MODE_STANDBY, +} + +impl TryFrom for Mode { + type Error = Error; + + /// Convert a mode represented as an unsigned integer into its Rust enum equivalent + /// + /// If the integer does not match any of the [`Mode`], then [`EINVAL`] is returned + fn try_from(mode: core::ffi::c_uint) -> Result { + match mode { + bindings::REGULATOR_MODE_FAST => Ok(Self::Fast), + bindings::REGULATOR_MODE_NORMAL => Ok(Self::Normal), + bindings::REGULATOR_MODE_IDLE => Ok(Self::Idle), + bindings::REGULATOR_MODE_STANDBY => Ok(Self::Standby), + bindings::REGULATOR_MODE_INVALID => Ok(Self::Invalid), + _ => Err(EINVAL), + } + } +} diff --git a/rust/kernel/regulator/consumer.rs b/rust/kernel/regulator/consumer.rs new file mode 100644 index 00000000000000..ea85aa30a49a2a --- /dev/null +++ b/rust/kernel/regulator/consumer.rs @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! SoC Regulator consumer abstractions. +//! +//! C header: [`include/linux/regulator/consumer.h`](srctree/include/linux/regulator/consumer.h) +//! +//! Reference: + +use crate::{ + bindings, + device::Device, + error::{code::*, from_err_ptr, to_result, Error, Result}, + regulator::Mode, + str::CStr, +}; +use core::{ + cmp::min, + ffi::{c_int, c_uint}, + mem::ManuallyDrop, + time::Duration, +}; + +/// [`Regulator`] in its default state (disabled) +/// +/// # Invariants +/// - [`self.0`] is valid and non-null +pub struct Regulator(*mut bindings::regulator); + +impl Regulator { + /// Lookup and obtain an instance of a regulator + /// + /// If the supply does not exists a dummy one will be + /// created + pub fn get>(dev: T, id: &'static CStr) -> Result { + // SAFETY: `dev.as_raw() is valid and non-null by the type invariant and + // id has a static lifetime so it lives indefinitely + let reg = from_err_ptr(unsafe { + bindings::regulator_get(dev.as_ref().as_raw(), id.as_char_ptr()) + })?; + + // This should not happen: in case of error `regulator_get` returns an + // error encoded into the pointer. And in case the device does not + // exists, a dummy regulator is returned + if reg.is_null() { + return Err(ENODEV); + } + + Ok(Self(reg)) + } + + /// Same as `get`, but if the regulator does not exists + /// an error will be returned instead of a dummy regulator + pub fn get_optional>(dev: T, id: &'static CStr) -> Result { + // SAFETY: `dev.as_raw() is valid and non-null by the type invariant and + // id has a static lifetime so it lives indefinitely + let reg = from_err_ptr(unsafe { + bindings::regulator_get_optional(dev.as_ref().as_raw(), id.as_char_ptr()) + })?; + + // does not exists `regulator_get_optional` returns an + // error encoded into the pointer. + if reg.is_null() { + return Err(ENODEV); + } + + Ok(Self(reg)) + } + + /// Same as `get` but ensure that we have exclusive access to the regulator + pub fn get_exclusive>(dev: T, id: &'static CStr) -> Result { + // SAFETY: `dev.as_raw() is valid and non-null by the type invariant and + // id has a static lifetime so it lives indefinitely + let reg = from_err_ptr(unsafe { + bindings::regulator_get_exclusive(dev.as_ref().as_raw(), id.as_char_ptr()) + })?; + + // This should not happen: in case of error `regulator_get` returns an + // error encoded into the pointer. And in case the device does not + // exists, a dummy regulator is returned + if reg.is_null() { + return Err(ENODEV); + } + + Ok(Self(reg)) + } + + /// Enable the regulator + pub fn enable(self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + to_result(unsafe { bindings::regulator_enable(self.0) })?; + Ok(EnabledRegulator(self)) + } + + /// Force disable the regulator. Even if other consumer + /// have enabled it, the regulator will be forcibly disabled. + pub fn force_disable(&mut self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + to_result(unsafe { bindings::regulator_force_disable(self.0) }) + } + + /// Check if the voltage range can be supported + pub fn is_supported_voltage(&self, min_uv: c_int, max_uv: c_int) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_is_supported_voltage(self.0, min_uv, max_uv) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(ret > 0) + } + + /// Returns the number of selectors supported by the regulator + pub fn count_voltages(&self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_count_voltages(self.0) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(ret as _) + } + + /// Returns the voltage corresponding to the `selector` + pub fn list_voltage(&self, selector: c_uint) -> Result> { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_list_voltage(self.0, selector) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(if ret == 0 { None } else { Some(ret) }) + } + + /// Returns the voltage step size between VSEL values + pub fn get_linear_step(&self) -> Option { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_get_linear_step(self.0) }; + if ret == 0 { + None + } else { + Some(ret) + } + } + + /// Returns the regulator output voltage + pub fn get_voltage(&self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_get_voltage(self.0) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(ret) + } + + /// Set the regulator output voltage + pub fn set_voltage(&mut self, min_uv: c_int, max_uv: c_int) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + to_result(unsafe { bindings::regulator_set_voltage(self.0, min_uv, max_uv) }) + } + + /// Get the raise/fall time required for switching voltage + pub fn set_voltage_time(&mut self, old_uv: c_int, new_uv: c_int) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_set_voltage_time(self.0, old_uv, new_uv) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(ret) + } + + /// Re-apply last regulator output voltage + pub fn sync_voltage(&mut self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + to_result(unsafe { bindings::regulator_sync_voltage(self.0) }) + } + + /// Get regulator output current + pub fn get_current_limit(&self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_get_current_limit(self.0) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(ret) + } + + /// Set regulator output current limit + pub fn set_current_limit(&mut self, min_ua: c_int, max_ua: c_int) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_set_current_limit(self.0, min_ua, max_ua) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + + Ok(ret) + } + + /// Set regulator load + pub fn set_load(&mut self, load_ua: c_int) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + to_result(unsafe { bindings::regulator_set_load(self.0, load_ua) }) + } + + /// Allow the regulator to go into bypass mode + pub fn allow_bypass(&mut self, allow: bool) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + to_result(unsafe { bindings::regulator_allow_bypass(self.0, allow) }) + } + + /// Set the mode of the regulator + pub fn set_mode(&mut self, mode: Mode) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + to_result(unsafe { bindings::regulator_set_mode(self.0, mode as _) }) + } + + /// Get the current mode of the regulator + pub fn get_mode(&mut self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + Mode::try_from(unsafe { bindings::regulator_get_mode(self.0) }) + } +} + +impl Drop for Regulator { + fn drop(&mut self) { + // SAFETY: The pointer is valid and non-null by the type invariant + unsafe { bindings::regulator_put(self.0) } + } +} + +// SAFETY: `Regulator` is not restricted to a single thread so it is safe +// to move it between threads +unsafe impl Send for Regulator {} + +/// [`Regulator`] that has been enabled +pub struct EnabledRegulator(Regulator); + +impl EnabledRegulator { + /// Disable the regulator + pub fn disable(self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_disable(self.0 .0) }; + if ret < 0 { + let mut reg = ManuallyDrop::new(self); + Ok(core::mem::replace( + &mut reg.0, + Regulator(core::ptr::null_mut()), + )) + } else { + Err(Error::from_errno(ret)) + } + } + + /// Disable the regulator with a specified delay + /// + /// Every non-zero delay < 1ms will be rounded up to 1ms, and any delay + /// longer than [`core::ffi::c_int`] will become [`core::ffi::c_int::MAX`] + pub fn disable_deferred(self, duration: Duration) -> Result { + let ms = min(duration.as_millis(), c_int::MAX as u128) as c_int; + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_disable_deferred(self.0 .0, ms) }; + if ret < 0 { + let mut reg = core::mem::ManuallyDrop::new(self); + Ok(core::mem::replace( + &mut reg.0, + Regulator(core::ptr::null_mut()), + )) + } else { + Err(Error::from_errno(ret)) + } + } + + /* Shared functions */ + + /// See [`Regulator::force_disable`] + pub fn force_disable(self) -> Result { + // SAFETY: The pointer is valid and non-null by the type invariant + let ret = unsafe { bindings::regulator_force_disable(self.0 .0) }; + if ret < 0 { + let mut reg = core::mem::ManuallyDrop::new(self); + Ok(core::mem::replace( + &mut reg.0, + Regulator(core::ptr::null_mut()), + )) + } else { + Err(Error::from_errno(ret)) + } + } + + /// See [`Regulator::is_supported_voltage`] + pub fn is_supported_voltage(&self, min_uv: c_int, max_uv: c_int) -> Result { + self.0.is_supported_voltage(min_uv, max_uv) + } + + /// See [`Regulator::count_voltages`] + pub fn count_voltages(&self) -> Result { + self.0.count_voltages() + } + + /// See [`Regulator::list_voltage`] + pub fn list_voltage(&self, selector: c_uint) -> Result> { + self.0.list_voltage(selector) + } + + /// See [`Regulator::get_linear_step`] + pub fn get_linear_step(&self) -> Option { + self.0.get_linear_step() + } + + /// See [`Regulator::get_voltage`] + pub fn get_voltage(&self) -> Result { + self.0.get_voltage() + } + + /// See [`Regulator::set_voltage`] + pub fn set_voltage(&mut self, min_uv: c_int, max_uv: c_int) -> Result { + self.0.set_voltage(min_uv, max_uv) + } + + /// See [`Regulator::set_voltage_time`] + pub fn set_voltage_time(&mut self, old_uv: c_int, new_uv: c_int) -> Result { + self.0.set_voltage_time(old_uv, new_uv) + } + + /// See [`Regulator::sync_voltage`] + pub fn sync_voltage(&mut self) -> Result { + self.0.sync_voltage() + } + + /// See [`Regulator::get_current_limit`] + pub fn get_current_limit(&self) -> Result { + self.0.get_current_limit() + } + + /// See [`Regulator::set_current_limit`] + pub fn set_current_limit(&mut self, min_ua: c_int, max_ua: c_int) -> Result { + self.0.set_current_limit(min_ua, max_ua) + } + + /// See [`Regulator::set_load`] + pub fn set_load(&mut self, load_ua: c_int) -> Result { + self.0.set_load(load_ua) + } + + /// See [`Regulator::allow_bypass`] + pub fn allow_bypass(&mut self, allow: bool) -> Result { + self.0.allow_bypass(allow) + } + + /// See [`Regulator::set_mode`] + pub fn set_mode(&mut self, mode: Mode) -> Result { + self.0.set_mode(mode) + } + + /// See [`Regulator::get_mode`] + pub fn get_mode(&mut self) -> Result { + self.0.get_mode() + } +} + +impl Drop for EnabledRegulator { + fn drop(&mut self) { + // SAFETY: The pointer is valid and non-null by the type invariant + let _ = unsafe { bindings::regulator_disable(self.0 .0) }; + } +} + +impl PartialEq for Regulator { + fn eq(&self, other: &Regulator) -> bool { + // SAFETY: The pointers are valid and non-null by the type invariant + unsafe { bindings::regulator_is_equal(self.0, other.0) } + } +} + +impl PartialEq for Regulator { + fn eq(&self, other: &EnabledRegulator) -> bool { + self.eq(&other.0) + } +} + +impl PartialEq for EnabledRegulator { + fn eq(&self, other: &EnabledRegulator) -> bool { + self.0.eq(&other.0) + } +} + +impl PartialEq for EnabledRegulator { + fn eq(&self, other: &Regulator) -> bool { + self.0.eq(other) + } +} diff --git a/rust/kernel/regulator/driver.rs b/rust/kernel/regulator/driver.rs new file mode 100644 index 00000000000000..235b45b38b1752 --- /dev/null +++ b/rust/kernel/regulator/driver.rs @@ -0,0 +1,957 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! SoC Regulator Driver Interface +//! +//! C header: [`include/linux/regulator/driver.h`](srctree/include/linux/regulator/driver.h) +//! +//! # Examples +//! +//! ```ignore +//! use kernel::regulator::driver::{Config, Desc, Driver, Registration, Type}; +//! # use kernel::{c_str, device::Device, module_platform_driver, of, platform, prelude::*}; +//! # +//! # module_platform_driver! { +//! # type: MyRegulatorDriver, +//! # name: "my_regulator_driver", +//! # license: "GPL", +//! # } +//! # +//! # kernel::module_of_id_table!(OF_MOD_TABLE, MY_REG_DRIVER_OF_ID_TABLE); +//! # kernel::define_of_id_table! {MY_REG_DRIVER_OF_ID_TABLE, (), [ +//! # (of::DeviceId::Compatible(b"my-regulator-driver"), None), +//! # ]} +//! +//! static DESC: Desc = +//! Desc::new::(c_str!("my-regulator-driver"), Type::Voltage) +//! .with_owner(&THIS_MODULE); +//! +//! struct MyRegulatorDriver; +//! +//! impl platform::Driver for MyRegulatorDriver { +//! # kernel::driver_of_id_table!(MY_REG_DRIVER_OF_ID_TABLE); +//! # +//! fn probe(pdev: &mut platform::Device, _id_info: Option<&Self::IdInfo>) -> Result { +//! let reg = Registration::register(pdev, &DESC, Config::::new(pdev))?; +//! Ok(()) +//! } +//! } +//! +//! #[vtable] +//! impl Driver for MyRegulatorDriver { +//! // Implement supported `Driver`'s operations here. +//! } +//! ``` + +use crate::{ + device::Device, + error::{code::*, from_err_ptr, from_result, Error, Result}, + macros::vtable, + regulator::Mode, + str::CStr, + sync::Arc, + types::ForeignOwnable, + ThisModule, +}; +#[cfg(CONFIG_REGMAP)] +use crate::{error::to_result, regmap::Regmap}; +use core::{ + marker::PhantomData, + mem::{ManuallyDrop, MaybeUninit}, +}; + +#[cfg(not(CONFIG_REGMAP))] +struct Regmap; + +#[cfg(not(CONFIG_REGMAP))] +impl Regmap { + fn as_ptr(&self) -> *mut bindings::regmap { + core::ptr::null_mut() + } +} + +/// [`Regulator`]'s status +/// +/// Corresponds to the kernel's [`enum regulator_status`]. +/// +/// [`enum regulator_status`]: srctree/include/linux/regulator/driver.h +#[derive(Eq, PartialEq)] +pub enum Status { + /// Regulator is off + Off, + /// Regulator is on + On, + /// Regulator is in an error state + Error, + /// Regulator is on and in Fast mode + Fast, + /// Regulator is on and in Normal mode + Normal, + /// Regulator is on and in Idle mode + Idle, + /// Regulator is on and in Standby mode + Standby, + /// Regulator is enabled but not regulating + Bypass, + /// Regulator is any other status + Undefined, +} + +impl TryFrom for Status { + type Error = Error; + + fn try_from(status: core::ffi::c_uint) -> Result { + match status { + bindings::regulator_status_REGULATOR_STATUS_OFF => Ok(Self::Off), + bindings::regulator_status_REGULATOR_STATUS_ON => Ok(Self::On), + bindings::regulator_status_REGULATOR_STATUS_ERROR => Ok(Self::Error), + bindings::regulator_status_REGULATOR_STATUS_FAST => Ok(Self::Fast), + bindings::regulator_status_REGULATOR_STATUS_NORMAL => Ok(Self::Normal), + bindings::regulator_status_REGULATOR_STATUS_IDLE => Ok(Self::Idle), + bindings::regulator_status_REGULATOR_STATUS_STANDBY => Ok(Self::Standby), + bindings::regulator_status_REGULATOR_STATUS_BYPASS => Ok(Self::Bypass), + bindings::regulator_status_REGULATOR_STATUS_UNDEFINED => Ok(Self::Undefined), + _ => Err(EINVAL), + } + } +} + +impl From for Status { + fn from(mode: Mode) -> Self { + // SAFETY: `regulator_mode_to_status` is a pure function that is only doing integer + // to integer conversion, hence this function call is safe. + let status = unsafe { bindings::regulator_mode_to_status(mode as _) }; + + if status < 0 { + Self::Undefined + } else { + Self::try_from(status as core::ffi::c_uint).unwrap_or(Self::Undefined) + } + } +} + +/// Regulator's operations +#[vtable] +pub trait Driver { + /// User data that will be accessible to all operations + type Data: ForeignOwnable + Send + Sync = (); + + /// Return one of the supported voltages, in microvolt; zero if the selector indicates a + /// voltage that is unusable by the system; or negative errno. Selectors range from zero to one + /// less than the number of voltages supported by the system. + fn list_voltage(_rdev: &Regulator, _selector: u32) -> Result { + Err(ENOTSUPP) + } + + /// Set the voltage for the regulator within the range specified. The driver should select the + /// voltage closest to `min_uv`. + fn set_voltage(_rdev: &Regulator, _min_uv: i32, _max_uv: i32) -> Result { + Err(ENOTSUPP) + } + + /// Set the voltage for the regulator using the specified selector. + fn set_voltage_sel(_rdev: &Regulator, _selector: u32) -> Result { + Err(ENOTSUPP) + } + + /// Convert a voltage into a selector. + fn map_voltage(_rdev: &Regulator, _min_uv: i32, _max_uv: i32) -> Result { + Err(ENOTSUPP) + } + + /// Get the currently configured voltage for the regulator; Returns + /// [`ENOTRECOVERABLE`] if the regulator can't be read at bootup and hasn't been + /// set yet. + fn get_voltage(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Get the currently configured voltage selector for the regulator; Returns + /// [`ENOTRECOVERABLE`] if the regulator can't be read at bootup and hasn't been + /// set yet. + fn get_voltage_sel(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Configure a limit for a current-limited regulator. + /// + /// The driver should select the current closest to `max_ua`. + fn set_current_limit(_rdev: &Regulator, _min_ua: i32, _max_ua: i32) -> Result { + Err(ENOTSUPP) + } + + /// Get the configured limit for a current-limited regulator. + fn get_current_limit(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Enable or disable the active discharge of the regulator. + fn set_active_discharge(_rdev: &Regulator, _enable: bool) -> Result { + Err(ENOTSUPP) + } + + /// Configure the regulator as enabled. + fn enable(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Configure the regulator as disabled. + fn disable(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Returns enablement state of the regulator. + fn is_enabled(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Set the configured operating [`Mode`] for the regulator. + fn set_mode(_rdev: &Regulator, _mode: Mode) -> Result { + Err(ENOTSUPP) + } + + /// Get the configured operating [`Mode`] for the regulator + fn get_mode(_rdev: &Regulator) -> Mode { + Mode::Invalid + } + + /// Report the regulator [`Status`]. + fn get_status(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Set the voltage for the regaultor when the system is suspended. + fn set_suspend_voltage(_rdev: &Regulator, _uv: i32) -> Result { + Err(ENOTSUPP) + } + + /// Mark the regulator as enabled when the system is suspended. + fn set_suspend_enable(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Mark the regulator as disabled when the system is suspended. + fn set_suspend_disable(_rdev: &Regulator) -> Result { + Err(ENOTSUPP) + } + + /// Set the operating mode for the regulator when the system is suspended. + fn set_suspend_mode(_rdev: &Regulator, _mode: Mode) -> Result { + Err(ENOTSUPP) + } +} + +/// [`Regulator`]'s descriptor +/// +/// # Invariants +/// +/// `self.0` has always valid data. +pub struct Desc(bindings::regulator_desc); +impl Desc { + /// Create a new [`Regulator`] descriptor + pub const fn new(name: &'static CStr, reg_type: Type) -> Self { + // SAFETY: `desc` is a C structure holding data that has been initialized with 0s, + // hence it is safe to use as-is. + let mut desc = unsafe { MaybeUninit::::zeroed().assume_init() }; + desc.name = name.as_char_ptr(); + desc.type_ = match reg_type { + Type::Voltage => bindings::regulator_type_REGULATOR_VOLTAGE, + Type::Current => bindings::regulator_type_REGULATOR_CURRENT, + }; + desc.ops = Adapter::::build(); + Self(desc) + } + + /// Setup the register address, mask, and {en,dis}able values + pub const fn with_enable(mut self, reg: u32, mask: u32, en_val: u32, dis_val: u32) -> Self { + self.0.enable_reg = reg; + self.0.enable_mask = mask; + self.0.enable_val = en_val; + self.0.disable_val = dis_val; + self + } + + /// Setup the register address, mask, and {en,dis}able values. {En,Dis}able values are + /// inverted, i.e. `dis_val` will be use to enable the regulator while `en_val` will be used + /// to disable the regulator. + pub const fn with_inverted_enable( + mut self, + reg: u32, + mask: u32, + en_val: u32, + dis_val: u32, + ) -> Self { + self.0.enable_is_inverted = true; + self.with_enable(reg, mask, en_val, dis_val) + } + + /// Setup the active discharge regiter address, mask, on/off values. + pub const fn with_active_discharge(mut self, reg: u32, mask: u32, on: u32, off: u32) -> Self { + self.0.active_discharge_on = on; + self.0.active_discharge_off = off; + self.0.active_discharge_reg = reg; + self.0.active_discharge_mask = mask; + self + } + + /// Setup the current selection register address, mask, and current table + pub const fn with_csel(mut self, reg: u32, mask: u32, table: &'static [u32]) -> Self { + self.0.csel_reg = reg; + self.0.csel_mask = mask; + self.0.curr_table = table.as_ptr(); + self + } + + /// Voltages are a linear mapping + pub const fn with_linear_mapping( + mut self, + reg: u32, + mask: u32, + min_uv: u32, + uv_step: u32, + n_voltages: u32, + linear_min_sel: u32, + ) -> Self { + self.0.vsel_reg = reg; + self.0.vsel_mask = mask; + self.0.n_voltages = n_voltages; + self.0.min_uV = min_uv; + self.0.uV_step = uv_step; + self.0.linear_min_sel = linear_min_sel; + self + } + + /// Set the regulator owner + pub const fn with_owner(mut self, owner: &'static ThisModule) -> Self { + self.0.owner = owner.as_ptr(); + self + } +} + +// SAFETY: `Desc` cannot be modified after its declaration, hence it is safe to share references +// between threads. +unsafe impl Sync for Desc {} + +/// [`Regulator`]'s Config +/// +/// # Examples +/// +/// ``` +/// use kernel::regulator::driver::Config; +/// # use kernel::regulator::driver::{Desc, Registration}; +/// # use kernel::{device::Device, sync::Arc}; +/// +/// struct DriverData(u32); +/// +/// # fn probe(dev: &Device, desc: &'static Desc) -> Result { +/// let config = Config::>::new(&dev, Arc::try_new(DriverData(128))?); +/// let reg = Registration::register(dev, desc, config)?; +/// # Ok(()) +/// # } +/// ``` +/// +/// # Invariants +/// +/// `self.cfg` always hold valid data. +pub struct Config { + cfg: bindings::regulator_config, + data: T, + regmap: Option>, +} + +impl Config { + /// Create a [`Regulator`] config. + pub fn new(dev: impl AsRef, data: T) -> Self { + Self { + cfg: bindings::regulator_config { + dev: dev.as_ref().as_raw(), + ..Default::default() + }, + data, + regmap: None, + } + } + + /// Assign a regmap device to the config + #[cfg(CONFIG_REGMAP)] + pub fn with_regmap(mut self, regmap: Arc) -> Self { + self.regmap = Some(regmap); + self + } +} + +/// Registration structure for Regulator drivers. +pub struct Registration(#[allow(dead_code)] Regulator); + +impl Registration { + /// register a Regulator driver + pub fn register( + dev: impl AsRef, + desc: &'static Desc, + cfg: Config, + ) -> Result { + Ok(Self(Regulator::register(dev, desc, cfg)?)) + } +} + +/// Regulator device +/// +/// Wraps the C structure `regulator_dev`. +/// +/// # Invariants +/// +/// * `self.rdev` is valid and non-null. +/// * [`Self`] has ownership of `self.rdev` memory allocation. +pub struct Regulator { + rdev: *mut bindings::regulator_dev, + _regmap: Option>, +} + +impl Regulator { + /// register a Regulator driver + fn register( + dev: impl AsRef, + desc: &'static Desc, + mut config: Config, + ) -> Result { + config.cfg.driver_data = config.data.into_foreign() as _; + + let regmap = config.regmap.take(); + if let Some(regmap) = ®map { + config.cfg.regmap = regmap.as_ptr(); + }; + + // SAFETY: By the type invariants, we know that `dev.as_ref().as_raw()` is always + // valid and non-null, and the descriptor and config are guaranteed to be valid values, + // hence it is safe to perform the FFI call. + let rdev = from_err_ptr(unsafe { + bindings::regulator_register(dev.as_ref().as_raw(), &desc.0, &config.cfg) + })?; + + if rdev.is_null() { + Err(EINVAL) + } else { + Ok(Self { + rdev, + _regmap: regmap, + }) + } + } + + /// List voltages when the regulator is using linear mapping + pub fn list_voltage_linear(&self, selector: u32) -> Result { + // SAFETY: By the type invariants, we know that `self.rdev` is always valid and non-null. + // The C function is safe to call with any selector values. + let ret = unsafe { bindings::regulator_list_voltage_linear(self.rdev, selector) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + Ok(ret) + } + + /// Get regulator's name + pub fn get_name(&self) -> &'static CStr { + // SAFETY: By the type invariants, we know that `self.rdev` is always valid and non-null. + // The C function is guaranteed to return a valid string. + unsafe { CStr::from_char_ptr(bindings::rdev_get_name(self.rdev)) } + } + + /// Get regulator's ID + pub fn get_id(&self) -> i32 { + // SAFETY: By the type invariants, we know that `self.rdev` is always valid and non-null. + unsafe { bindings::rdev_get_id(self.rdev) } + } + + /// Retrieve driver data associated to `self` + pub fn data(&self) -> T::Borrowed<'_> { + // SAFETY: By the type invariants, we know that `self.rdev` is always valid and non-null. + unsafe { T::borrow(bindings::rdev_get_drvdata(self.rdev)) } + } +} + +impl Drop for Regulator { + fn drop(&mut self) { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + let regmap = unsafe { bindings::rdev_get_regmap(self.rdev) }; + + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + unsafe { bindings::regulator_unregister(self.rdev) } + + if !regmap.is_null() { + drop(unsafe { Arc::from_raw(regmap) }); + } + } +} + +// SAFETY: `Regulator` has sole ownership of `self.rdev` and is never read outside of the C +// implementation. It is safe to use it from any thread. +unsafe impl Send for Regulator {} + +mod sealed { + pub trait Sealed {} +} + +impl sealed::Sealed for Regulator {} + +/// Helper functions to implement some of the [`Driver`] trait methods using [`Regmap`]. +/// +/// This trait is implemented by [`Regulator`] and is Sealed to prevent +/// to be implemented by anyone else. +#[cfg(CONFIG_REGMAP)] +pub trait RegmapHelpers: sealed::Sealed { + /// Implementation of [`Driver::get_voltage_sel`] using [`Regmap`]. + fn get_voltage_sel_regmap(&self) -> Result; + /// Implementation of [`Driver::set_voltage_sel`] using [`Regmap`]. + fn set_voltage_sel_regmap(&self, sel: u32) -> Result; + + /// Implementation of [`Driver::is_enabled`] using [`Regmap`]. + /// + /// [`Desc::with_enable`] or [`Desc::with_inverted_enable`] must have been called + /// to setup the fields required by regmap. + fn is_enabled_regmap(&self) -> Result; + + /// Implementation of [`Driver::enable`] using [`Regmap`]. + /// + /// [`Desc::with_enable`] or [`Desc::with_inverted_enable`] must have been called + /// to setup the fields required by regmap. + fn enable_regmap(&self) -> Result; + + /// Implementation of [`Driver::disable`] using [`Regmap`]. + /// + /// [`Desc::with_enable`] or [`Desc::with_inverted_enable`] must have been called + /// to setup the fields required by regmap. + fn disable_regmap(&self) -> Result; + + /// Implementation of [`Driver::set_active_discharge`] using [`Regmap`]. + /// + /// [`Desc::with_active_discharge`] must have been called to setup the fields required + /// by regmap. + fn set_active_discharge_regmap(&self, enable: bool) -> Result; + /// Implementation of [`Driver::set_current_limit`] using [`Regmap`]. + fn set_current_limit_regmap(&self, min_ua: i32, max_ua: i32) -> Result; + /// Implementation of [`Driver::get_current_limit`] using [`Regmap`]. + fn get_current_limit_regmap(&self) -> Result; +} + +#[cfg(CONFIG_REGMAP)] +impl RegmapHelpers for Regulator { + fn get_voltage_sel_regmap(&self) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + let ret = unsafe { bindings::regulator_get_voltage_sel_regmap(self.rdev) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + Ok(ret) + } + + fn set_voltage_sel_regmap(&self, sel: u32) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + to_result(unsafe { bindings::regulator_set_voltage_sel_regmap(self.rdev, sel) }) + } + + fn is_enabled_regmap(&self) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + let ret = unsafe { bindings::regulator_is_enabled_regmap(self.rdev) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + Ok(ret > 0) + } + + fn enable_regmap(&self) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + to_result(unsafe { bindings::regulator_enable_regmap(self.rdev) }) + } + + fn disable_regmap(&self) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + to_result(unsafe { bindings::regulator_disable_regmap(self.rdev) }) + } + + fn set_active_discharge_regmap(&self, enable: bool) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + to_result(unsafe { bindings::regulator_set_active_discharge_regmap(self.rdev, enable) }) + } + + fn set_current_limit_regmap(&self, min_ua: i32, max_ua: i32) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + to_result(unsafe { + bindings::regulator_set_current_limit_regmap(self.rdev, min_ua, max_ua) + }) + } + + fn get_current_limit_regmap(&self) -> Result { + // SAFETY: The type invariants guarantee that `self.rdev` is valid and non-null, + // so it is safe to perform the FFI call. + let ret = unsafe { bindings::regulator_get_current_limit_regmap(self.rdev) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + Ok(ret) + } +} + +/// [`Regulator`] type +pub enum Type { + /// Voltage regulator + Voltage, + /// Current regulator + Current, +} + +pub(crate) struct Adapter(PhantomData); + +impl Adapter { + unsafe extern "C" fn list_voltage_callback( + rdev: *mut bindings::regulator_dev, + selector: core::ffi::c_uint, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| T::list_voltage(&rdev, selector)) + } + + unsafe extern "C" fn set_voltage_callback( + rdev: *mut bindings::regulator_dev, + min_uv: core::ffi::c_int, + max_uv: core::ffi::c_int, + selector: *mut core::ffi::c_uint, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + match T::set_voltage(&rdev, min_uv, max_uv) { + Ok(v) => { + unsafe { *selector = v as _ }; + 0 + } + Err(e) => e.to_errno(), + } + } + + unsafe extern "C" fn map_voltage_callback( + rdev: *mut bindings::regulator_dev, + min_uv: core::ffi::c_int, + max_uv: core::ffi::c_int, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| T::map_voltage(&rdev, min_uv, max_uv)) + } + + unsafe extern "C" fn set_voltage_sel_callback( + rdev: *mut bindings::regulator_dev, + selector: core::ffi::c_uint, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| { + T::set_voltage_sel(&rdev, selector)?; + Ok(0) + }) + } + + unsafe extern "C" fn get_voltage_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| T::get_voltage(&rdev)) + } + + unsafe extern "C" fn get_voltage_sel_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| T::get_voltage_sel(&rdev)) + } + + unsafe extern "C" fn set_current_limit_callback( + rdev: *mut bindings::regulator_dev, + min_ua: core::ffi::c_int, + max_ua: core::ffi::c_int, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| { + T::set_current_limit(&rdev, min_ua, max_ua)?; + Ok(0) + }) + } + + unsafe extern "C" fn get_current_limit_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| T::get_current_limit(&rdev)) + } + + unsafe extern "C" fn set_active_discharge_callback( + rdev: *mut bindings::regulator_dev, + enable: bool, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| { + T::set_active_discharge(&rdev, enable)?; + Ok(0) + }) + } + + unsafe extern "C" fn enable_callback(rdev: *mut bindings::regulator_dev) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| { + T::enable(&rdev)?; + Ok(0) + }) + } + + unsafe extern "C" fn disable_callback(rdev: *mut bindings::regulator_dev) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| { + T::disable(&rdev)?; + Ok(0) + }) + } + + unsafe extern "C" fn is_enabled_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| { + T::is_enabled(&rdev)?; + Ok(0) + }) + } + + unsafe extern "C" fn set_mode_callback( + rdev: *mut bindings::regulator_dev, + mode: core::ffi::c_uint, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| { + let mode = Mode::try_from(mode).unwrap_or(Mode::Invalid); + T::set_mode(&rdev, mode)?; + Ok(0) + }) + } + + unsafe extern "C" fn get_mode_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_uint { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + T::get_mode(&rdev) as _ + } + + unsafe extern "C" fn get_status_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| Ok(T::get_status(&rdev)? as _)) + } + + unsafe extern "C" fn set_suspend_voltage_callback( + rdev: *mut bindings::regulator_dev, + uv: core::ffi::c_int, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| { + T::set_suspend_voltage(&rdev, uv)?; + Ok(0) + }) + } + + unsafe extern "C" fn set_suspend_enable_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| { + T::set_suspend_enable(&rdev)?; + Ok(0) + }) + } + + unsafe extern "C" fn set_suspend_disable_callback( + rdev: *mut bindings::regulator_dev, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| { + T::set_suspend_disable(&rdev)?; + Ok(0) + }) + } + + unsafe extern "C" fn set_suspend_mode_callback( + rdev: *mut bindings::regulator_dev, + mode: core::ffi::c_uint, + ) -> core::ffi::c_int { + let rdev = ManuallyDrop::new(Regulator { + rdev, + _regmap: None, + }); + from_result(|| { + let mode = Mode::try_from(mode).unwrap_or(Mode::Invalid); + T::set_suspend_mode(&rdev, mode)?; + Ok(0) + }) + } + + const VTABLE: bindings::regulator_ops = bindings::regulator_ops { + list_voltage: if T::HAS_LIST_VOLTAGE { + Some(Adapter::::list_voltage_callback) + } else { + None + }, + set_voltage: if T::HAS_SET_VOLTAGE { + Some(Adapter::::set_voltage_callback) + } else { + None + }, + map_voltage: if T::HAS_MAP_VOLTAGE { + Some(Adapter::::map_voltage_callback) + } else { + None + }, + set_voltage_sel: if T::HAS_SET_VOLTAGE_SEL { + Some(Adapter::::set_voltage_sel_callback) + } else { + None + }, + get_voltage: if T::HAS_GET_VOLTAGE { + Some(Adapter::::get_voltage_callback) + } else { + None + }, + get_voltage_sel: if T::HAS_GET_VOLTAGE_SEL { + Some(Adapter::::get_voltage_sel_callback) + } else { + None + }, + set_current_limit: if T::HAS_SET_CURRENT_LIMIT { + Some(Adapter::::set_current_limit_callback) + } else { + None + }, + get_current_limit: if T::HAS_GET_CURRENT_LIMIT { + Some(Adapter::::get_current_limit_callback) + } else { + None + }, + set_active_discharge: if T::HAS_SET_ACTIVE_DISCHARGE { + Some(Adapter::::set_active_discharge_callback) + } else { + None + }, + enable: if T::HAS_ENABLE { + Some(Adapter::::enable_callback) + } else { + None + }, + disable: if T::HAS_DISABLE { + Some(Adapter::::disable_callback) + } else { + None + }, + is_enabled: if T::HAS_IS_ENABLED { + Some(Adapter::::is_enabled_callback) + } else { + None + }, + set_mode: if T::HAS_SET_MODE { + Some(Adapter::::set_mode_callback) + } else { + None + }, + get_mode: if T::HAS_GET_MODE { + Some(Adapter::::get_mode_callback) + } else { + None + }, + get_status: if T::HAS_GET_STATUS { + Some(Adapter::::get_status_callback) + } else { + None + }, + set_suspend_voltage: if T::HAS_SET_SUSPEND_VOLTAGE { + Some(Adapter::::set_suspend_voltage_callback) + } else { + None + }, + set_suspend_enable: if T::HAS_SET_SUSPEND_ENABLE { + Some(Adapter::::set_suspend_enable_callback) + } else { + None + }, + set_suspend_disable: if T::HAS_SET_SUSPEND_DISABLE { + Some(Adapter::::set_suspend_disable_callback) + } else { + None + }, + set_suspend_mode: if T::HAS_SET_SUSPEND_MODE { + Some(Adapter::::set_suspend_mode_callback) + } else { + None + }, + // SAFETY: The rest is zeroed out to initialize `struct regulator_ops`, + // sets `Option<&F>` to be `None`. + ..unsafe { core::mem::MaybeUninit::::zeroed().assume_init() } + }; + + const fn build() -> &'static bindings::regulator_ops { + &Self::VTABLE + } +} diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs new file mode 100644 index 00000000000000..83455558d7957f --- /dev/null +++ b/rust/kernel/revocable.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Revocable objects. +//! +//! The [`Revocable`] type wraps other types and allows access to them to be revoked. The existence +//! of a [`RevocableGuard`] ensures that objects remain valid. + +use crate::{ + bindings, + init::{self}, + prelude::*, + sync::rcu, +}; +use core::{ + cell::UnsafeCell, + marker::PhantomData, + mem::MaybeUninit, + ops::Deref, + ptr::drop_in_place, + sync::atomic::{AtomicBool, Ordering}, +}; + +/// An object that can become inaccessible at runtime. +/// +/// Once access is revoked and all concurrent users complete (i.e., all existing instances of +/// [`RevocableGuard`] are dropped), the wrapped object is also dropped. +/// +/// # Examples +/// +/// ``` +/// # use kernel::revocable::Revocable; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn add_two(v: &Revocable) -> Option { +/// let guard = v.try_access()?; +/// Some(guard.a + guard.b) +/// } +/// +/// let v = KBox::pin_init(Revocable::new(Example { a: 10, b: 20 }), GFP_KERNEL).unwrap(); +/// assert_eq!(add_two(&v), Some(30)); +/// v.revoke(); +/// assert_eq!(add_two(&v), None); +/// ``` +/// +/// Sample example as above, but explicitly using the rcu read side lock. +/// +/// ``` +/// # use kernel::revocable::Revocable; +/// use kernel::sync::rcu; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn add_two(v: &Revocable) -> Option { +/// let guard = rcu::read_lock(); +/// let e = v.try_access_with_guard(&guard)?; +/// Some(e.a + e.b) +/// } +/// +/// let v = KBox::pin_init(Revocable::new(Example { a: 10, b: 20 }), GFP_KERNEL).unwrap(); +/// assert_eq!(add_two(&v), Some(30)); +/// v.revoke(); +/// assert_eq!(add_two(&v), None); +/// ``` +#[pin_data(PinnedDrop)] +pub struct Revocable { + is_available: AtomicBool, + #[pin] + data: MaybeUninit>, +} + +// SAFETY: `Revocable` is `Send` if the wrapped object is also `Send`. This is because while the +// functionality exposed by `Revocable` can be accessed from any thread/CPU, it is possible that +// this isn't supported by the wrapped object. +unsafe impl Send for Revocable {} + +// SAFETY: `Revocable` is `Sync` if the wrapped object is both `Send` and `Sync`. We require `Send` +// from the wrapped object as well because of `Revocable::revoke`, which can trigger the `Drop` +// implementation of the wrapped object from an arbitrary thread. +unsafe impl Sync for Revocable {} + +impl Revocable { + /// Creates a new revocable instance of the given data. + pub fn new(data: impl PinInit) -> impl PinInit { + pin_init!(Self { + is_available: AtomicBool::new(true), + // SAFETY: The closure only returns `Ok(())` if `slot` is fully initialized; on error + // `slot` is not partially initialized and does not need to be dropped. + data <- unsafe { + init::pin_init_from_closure(move |slot: *mut MaybeUninit>| { + init::PinInit::::__pinned_init(data, + slot as *mut T)?; + Ok::<(), core::convert::Infallible>(()) + }) + }, + }) + } + + /// Tries to access the \[revocable\] wrapped object. + /// + /// Returns `None` if the object has been revoked and is therefore no longer accessible. + /// + /// Returns a guard that gives access to the object otherwise; the object is guaranteed to + /// remain accessible while the guard is alive. In such cases, callers are not allowed to sleep + /// because another CPU may be waiting to complete the revocation of this object. + pub fn try_access(&self) -> Option> { + let guard = rcu::read_lock(); + if self.is_available.load(Ordering::Relaxed) { + // SAFETY: Since `self.is_available` is true, data is initialised and has to remain + // valid because the RCU read side lock prevents it from being dropped. + Some(unsafe { RevocableGuard::new(self.data.assume_init_ref().get(), guard) }) + } else { + None + } + } + + /// Tries to access the \[revocable\] wrapped object. + /// + /// Returns `None` if the object has been revoked and is therefore no longer accessible. + /// + /// Returns a shared reference to the object otherwise; the object is guaranteed to + /// remain accessible while the rcu read side guard is alive. In such cases, callers are not + /// allowed to sleep because another CPU may be waiting to complete the revocation of this + /// object. + pub fn try_access_with_guard<'a>(&'a self, _guard: &'a rcu::Guard) -> Option<&'a T> { + if self.is_available.load(Ordering::Relaxed) { + // SAFETY: Since `self.is_available` is true, data is initialised and has to remain + // valid because the RCU read side lock prevents it from being dropped. + Some(unsafe { &*self.data.assume_init_ref().get() }) + } else { + None + } + } + + /// Revokes access to and drops the wrapped object. + /// + /// Access to the object is revoked immediately to new callers of [`Revocable::try_access`]. If + /// there are concurrent users of the object (i.e., ones that called [`Revocable::try_access`] + /// beforehand and still haven't dropped the returned guard), this function waits for the + /// concurrent access to complete before dropping the wrapped object. + pub fn revoke(&self) { + if self + .is_available + .compare_exchange(true, false, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + // SAFETY: Just an FFI call, there are no further requirements. + unsafe { bindings::synchronize_rcu() }; + + // SAFETY: We know `self.data` is valid because only one CPU can succeed the + // `compare_exchange` above that takes `is_available` from `true` to `false`. + unsafe { drop_in_place(self.data.assume_init_ref().get()) }; + } + } +} + +#[pinned_drop] +impl PinnedDrop for Revocable { + fn drop(self: Pin<&mut Self>) { + // Drop only if the data hasn't been revoked yet (in which case it has already been + // dropped). + // SAFETY: We are not moving out of `p`, only dropping in place + let p = unsafe { self.get_unchecked_mut() }; + if *p.is_available.get_mut() { + // SAFETY: We know `self.data` is valid because no other CPU has changed + // `is_available` to `false` yet, and no other CPU can do it anymore because this CPU + // holds the only reference (mutable) to `self` now. + unsafe { drop_in_place(p.data.assume_init_ref().get()) }; + } + } +} + +/// A guard that allows access to a revocable object and keeps it alive. +/// +/// CPUs may not sleep while holding on to [`RevocableGuard`] because it's in atomic context +/// holding the RCU read-side lock. +/// +/// # Invariants +/// +/// The RCU read-side lock is held while the guard is alive. +pub struct RevocableGuard<'a, T> { + data_ref: *const T, + _rcu_guard: rcu::Guard, + _p: PhantomData<&'a ()>, +} + +impl RevocableGuard<'_, T> { + fn new(data_ref: *const T, rcu_guard: rcu::Guard) -> Self { + Self { + data_ref, + _rcu_guard: rcu_guard, + _p: PhantomData, + } + } +} + +impl Deref for RevocableGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: By the type invariants, we hold the rcu read-side lock, so the object is + // guaranteed to remain valid. + unsafe { &*self.data_ref } + } +} diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index 66d4527f6c6f0b..6ae301f64cedc8 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -2,8 +2,7 @@ //! String representations. -use crate::alloc::{flags::*, vec_ext::VecExt, AllocError}; -use alloc::vec::Vec; +use crate::alloc::{flags::*, AllocError, KVec}; use core::fmt::{self, Write}; use core::ops::{self, Deref, DerefMut, Index}; @@ -32,6 +31,12 @@ impl BStr { // SAFETY: `BStr` is transparent to `[u8]`. unsafe { &*(bytes as *const [u8] as *const BStr) } } + + /// Returns a reference to the inner [u8]. + #[inline] + pub const fn deref_const(&self) -> &[u8] { + &self.0 + } } impl fmt::Display for BStr { @@ -103,7 +108,7 @@ impl Deref for BStr { #[inline] fn deref(&self) -> &Self::Target { - &self.0 + self.deref_const() } } @@ -525,7 +530,28 @@ macro_rules! c_str { #[cfg(test)] mod tests { use super::*; - use alloc::format; + + struct String(CString); + + impl String { + fn from_fmt(args: fmt::Arguments<'_>) -> Self { + String(CString::try_from_fmt(args).unwrap()) + } + } + + impl Deref for String { + type Target = str; + + fn deref(&self) -> &str { + self.0.to_str().unwrap() + } + } + + macro_rules! format { + ($($f:tt)*) => ({ + &*String::from_fmt(kernel::fmt!($($f)*)) + }) + } const ALL_ASCII_CHARS: &'static str = "\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\x09\\x0a\\x0b\\x0c\\x0d\\x0e\\x0f\ @@ -791,7 +817,7 @@ impl fmt::Write for Formatter { /// assert_eq!(s.is_ok(), false); /// ``` pub struct CString { - buf: Vec, + buf: KVec, } impl CString { @@ -804,7 +830,7 @@ impl CString { let size = f.bytes_written(); // Allocate a vector with the required number of bytes, and write to it. - let mut buf = as VecExt<_>>::with_capacity(size, GFP_KERNEL)?; + let mut buf = KVec::with_capacity(size, GFP_KERNEL)?; // SAFETY: The buffer stored in `buf` is at least of size `size` and is valid for writes. let mut f = unsafe { Formatter::from_buffer(buf.as_mut_ptr(), size) }; f.write_fmt(args)?; @@ -851,10 +877,9 @@ impl<'a> TryFrom<&'a CStr> for CString { type Error = AllocError; fn try_from(cstr: &'a CStr) -> Result { - let mut buf = Vec::new(); + let mut buf = KVec::new(); - as VecExt<_>>::extend_from_slice(&mut buf, cstr.as_bytes_with_nul(), GFP_KERNEL) - .map_err(|_| AllocError)?; + buf.extend_from_slice(cstr.as_bytes_with_nul(), GFP_KERNEL)?; // INVARIANT: The `CStr` and `CString` types have the same invariants for // the string data, and we copied it over without changes. diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 0ab20975a3b5db..1806767359fec2 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -11,6 +11,7 @@ mod arc; mod condvar; pub mod lock; mod locked_by; +pub mod rcu; pub use arc::{Arc, ArcBorrow, UniqueArc}; pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 9325cc5a16a4c4..db9da352d588f6 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -17,13 +17,12 @@ //! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html use crate::{ - alloc::{box_ext::BoxExt, AllocError, Flags}, + alloc::{AllocError, Flags, KBox}, bindings, init::{self, InPlaceInit, Init, PinInit}, try_init, types::{ForeignOwnable, Opaque}, }; -use alloc::boxed::Box; use core::{ alloc::Layout, fmt, @@ -201,11 +200,11 @@ impl Arc { data: contents, }; - let inner = as BoxExt<_>>::new(value, flags)?; + let inner = KBox::new(value, flags)?; // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new // `Arc` object. - Ok(unsafe { Self::from_inner(Box::leak(inner).into()) }) + Ok(unsafe { Self::from_inner(KBox::leak(inner).into()) }) } } @@ -398,8 +397,8 @@ impl Drop for Arc { if is_zero { // The count reached zero, we must free the memory. // - // SAFETY: The pointer was initialised from the result of `Box::leak`. - unsafe { drop(Box::from_raw(self.ptr.as_ptr())) }; + // SAFETY: The pointer was initialised from the result of `KBox::leak`. + unsafe { drop(KBox::from_raw(self.ptr.as_ptr())) }; } } } @@ -641,7 +640,7 @@ impl UniqueArc { /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet. pub fn new_uninit(flags: Flags) -> Result>, AllocError> { // INVARIANT: The refcount is initialised to a non-zero value. - let inner = Box::try_init::( + let inner = KBox::try_init::( try_init!(ArcInner { // SAFETY: There are no safety requirements for this FFI call. refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), @@ -651,8 +650,8 @@ impl UniqueArc { )?; Ok(UniqueArc { // INVARIANT: The newly-created object has a refcount of 1. - // SAFETY: The pointer from the `Box` is valid. - inner: unsafe { Arc::from_inner(Box::leak(inner).into()) }, + // SAFETY: The pointer from the `KBox` is valid. + inner: unsafe { Arc::from_inner(KBox::leak(inner).into()) }, }) } } diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs index dec2e5ffc91939..a1a29c0bdb3aa2 100644 --- a/rust/kernel/sync/condvar.rs +++ b/rust/kernel/sync/condvar.rs @@ -70,8 +70,8 @@ pub use new_condvar; /// } /// /// /// Allocates a new boxed `Example`. -/// fn new_example() -> Result>> { -/// Box::pin_init(pin_init!(Example { +/// fn new_example() -> Result>> { +/// KBox::pin_init(pin_init!(Example { /// value <- new_mutex!(0), /// value_changed <- new_condvar!(), /// }), GFP_KERNEL) diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index c4f3b6cbfe48de..9ce43ccb451583 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -58,7 +58,7 @@ pub use new_mutex; /// } /// /// // Allocate a boxed `Example`. -/// let e = Box::pin_init(Example::new(), GFP_KERNEL)?; +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; /// assert_eq!(e.c, 10); /// assert_eq!(e.d.lock().a, 20); /// assert_eq!(e.d.lock().b, 30); diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index c900ae23db76fe..040dc16975a68a 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -56,7 +56,7 @@ pub use new_spinlock; /// } /// /// // Allocate a boxed `Example`. -/// let e = Box::pin_init(Example::new(), GFP_KERNEL)?; +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; /// assert_eq!(e.c, 10); /// assert_eq!(e.d.lock().a, 20); /// assert_eq!(e.d.lock().b, 30); diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs index ce2ee8d8786587..a7b244675c2b96 100644 --- a/rust/kernel/sync/locked_by.rs +++ b/rust/kernel/sync/locked_by.rs @@ -43,7 +43,7 @@ use core::{cell::UnsafeCell, mem::size_of, ptr}; /// struct InnerDirectory { /// /// The sum of the bytes used by all files. /// bytes_used: u64, -/// _files: Vec, +/// _files: KVec, /// } /// /// struct Directory { diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs new file mode 100644 index 00000000000000..5a35495f69a46b --- /dev/null +++ b/rust/kernel/sync/rcu.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! RCU support. +//! +//! C header: [`include/linux/rcupdate.h`](srctree/include/linux/rcupdate.h) + +use crate::bindings; +use core::marker::PhantomData; + +/// Evidence that the RCU read side lock is held on the current thread/CPU. +/// +/// The type is explicitly not `Send` because this property is per-thread/CPU. +/// +/// # Invariants +/// +/// The RCU read side lock is actually held while instances of this guard exist. +pub struct Guard { + _not_send: PhantomData<*mut ()>, +} + +impl Guard { + /// Acquires the RCU read side lock and returns a guard. + pub fn new() -> Self { + // SAFETY: An FFI call with no additional requirements. + unsafe { bindings::rcu_read_lock() }; + // INVARIANT: The RCU read side lock was just acquired above. + Self { + _not_send: PhantomData, + } + } + + /// Explicitly releases the RCU read side lock. + pub fn unlock(self) {} +} + +impl Default for Guard { + fn default() -> Self { + Self::new() + } +} + +impl Drop for Guard { + fn drop(&mut self) { + // SAFETY: By the type invariants, the rcu read side is locked, so it is ok to unlock it. + unsafe { bindings::rcu_read_unlock() }; + } +} + +/// Acquires the RCU read side lock. +pub fn read_lock() -> Guard { + Guard::new() +} diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 085e8076f07883..236e8de5844b51 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -3,13 +3,11 @@ //! Kernel types. use crate::init::{self, PinInit}; -use alloc::boxed::Box; use core::{ cell::UnsafeCell, marker::{PhantomData, PhantomPinned}, mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, - pin::Pin, ptr::NonNull, }; @@ -71,54 +69,6 @@ pub trait ForeignOwnable: Sized { } } -impl ForeignOwnable for Box { - type Borrowed<'a> = &'a T; - - fn into_foreign(self) -> *const core::ffi::c_void { - Box::into_raw(self) as _ - } - - unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> &'a T { - // SAFETY: The safety requirements for this function ensure that the object is still alive, - // so it is safe to dereference the raw pointer. - // The safety requirements of `from_foreign` also ensure that the object remains alive for - // the lifetime of the returned value. - unsafe { &*ptr.cast() } - } - - unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { - // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous - // call to `Self::into_foreign`. - unsafe { Box::from_raw(ptr as _) } - } -} - -impl ForeignOwnable for Pin> { - type Borrowed<'a> = Pin<&'a T>; - - fn into_foreign(self) -> *const core::ffi::c_void { - // SAFETY: We are still treating the box as pinned. - Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }) as _ - } - - unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> Pin<&'a T> { - // SAFETY: The safety requirements for this function ensure that the object is still alive, - // so it is safe to dereference the raw pointer. - // The safety requirements of `from_foreign` also ensure that the object remains alive for - // the lifetime of the returned value. - let r = unsafe { &*ptr.cast() }; - - // SAFETY: This pointer originates from a `Pin>`. - unsafe { Pin::new_unchecked(r) } - } - - unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { - // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous - // call to `Self::into_foreign`. - unsafe { Pin::new_unchecked(Box::from_raw(ptr as _)) } - } -} - impl ForeignOwnable for () { type Borrowed<'a> = (); @@ -185,7 +135,7 @@ impl ForeignOwnable for () { /// # use kernel::types::ScopeGuard; /// fn example3(arg: bool) -> Result { /// let mut vec = -/// ScopeGuard::new_with_data(Vec::new(), |v| pr_info!("vec had {} elements\n", v.len())); +/// ScopeGuard::new_with_data(KVec::new(), |v| pr_info!("vec had {} elements\n", v.len())); /// /// vec.push(10u8, GFP_KERNEL)?; /// if arg { @@ -289,14 +239,22 @@ impl Opaque { /// uninitialized. Additionally, access to the inner `T` requires `unsafe`, so the caller needs /// to verify at that point that the inner value is valid. pub fn ffi_init(init_func: impl FnOnce(*mut T)) -> impl PinInit { + Self::try_ffi_init(move |slot| { + init_func(slot); + Ok(()) + }) + } + + /// Similar to [`Self::ffi_init`], except that the closure can fail. + /// + /// To avoid leaks on failure, the closure must drop any fields it has initialised before the + /// failure. + pub fn try_ffi_init( + init_func: impl FnOnce(*mut T) -> Result<(), E>, + ) -> impl PinInit { // SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully // initialize the `T`. - unsafe { - init::pin_init_from_closure::<_, ::core::convert::Infallible>(move |slot| { - init_func(Self::raw_get(slot)); - Ok(()) - }) - } + unsafe { init::pin_init_from_closure(|slot| init_func(Self::raw_get(slot))) } } /// Returns a raw pointer to the opaque data. diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index 0af243482eac9a..2c953ba53c771c 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -11,7 +11,6 @@ use crate::{ prelude::*, transmute::{AsBytes, FromBytes}, }; -use alloc::vec::Vec; use core::ffi::{c_ulong, c_void}; use core::mem::{size_of, MaybeUninit}; @@ -46,7 +45,6 @@ pub type UserPtr = usize; /// every byte in the region. /// /// ```no_run -/// use alloc::vec::Vec; /// use core::ffi::c_void; /// use kernel::error::Result; /// use kernel::uaccess::{UserPtr, UserSlice}; @@ -54,7 +52,7 @@ pub type UserPtr = usize; /// fn bytes_add_one(uptr: UserPtr, len: usize) -> Result<()> { /// let (read, mut write) = UserSlice::new(uptr, len).reader_writer(); /// -/// let mut buf = Vec::new(); +/// let mut buf = KVec::new(); /// read.read_all(&mut buf, GFP_KERNEL)?; /// /// for b in &mut buf { @@ -69,7 +67,6 @@ pub type UserPtr = usize; /// Example illustrating a TOCTOU (time-of-check to time-of-use) bug. /// /// ```no_run -/// use alloc::vec::Vec; /// use core::ffi::c_void; /// use kernel::error::{code::EINVAL, Result}; /// use kernel::uaccess::{UserPtr, UserSlice}; @@ -78,21 +75,21 @@ pub type UserPtr = usize; /// fn is_valid(uptr: UserPtr, len: usize) -> Result { /// let read = UserSlice::new(uptr, len).reader(); /// -/// let mut buf = Vec::new(); +/// let mut buf = KVec::new(); /// read.read_all(&mut buf, GFP_KERNEL)?; /// /// todo!() /// } /// /// /// Returns the bytes behind this user pointer if they are valid. -/// fn get_bytes_if_valid(uptr: UserPtr, len: usize) -> Result> { +/// fn get_bytes_if_valid(uptr: UserPtr, len: usize) -> Result> { /// if !is_valid(uptr, len)? { /// return Err(EINVAL); /// } /// /// let read = UserSlice::new(uptr, len).reader(); /// -/// let mut buf = Vec::new(); +/// let mut buf = KVec::new(); /// read.read_all(&mut buf, GFP_KERNEL)?; /// /// // THIS IS A BUG! The bytes could have changed since we checked them. @@ -130,7 +127,7 @@ impl UserSlice { /// Reads the entirety of the user slice, appending it to the end of the provided buffer. /// /// Fails with [`EFAULT`] if the read happens on a bad address. - pub fn read_all(self, buf: &mut Vec, flags: Flags) -> Result { + pub fn read_all(self, buf: &mut KVec, flags: Flags) -> Result { self.reader().read_all(buf, flags) } @@ -291,9 +288,9 @@ impl UserSliceReader { /// Reads the entirety of the user slice, appending it to the end of the provided buffer. /// /// Fails with [`EFAULT`] if the read happens on a bad address. - pub fn read_all(mut self, buf: &mut Vec, flags: Flags) -> Result { + pub fn read_all(mut self, buf: &mut KVec, flags: Flags) -> Result { let len = self.length; - VecExt::::reserve(buf, len, flags)?; + buf.reserve(len, flags)?; // The call to `try_reserve` was successful, so the spare capacity is at least `len` bytes // long. diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 10d2bc62e2cff7..4d1d2062f6eba5 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -216,7 +216,7 @@ impl Queue { func: Some(func), }); - self.enqueue(Box::pin_init(init, flags).map_err(|_| AllocError)?); + self.enqueue(KBox::pin_init(init, flags).map_err(|_| AllocError)?); Ok(()) } } @@ -239,9 +239,9 @@ impl ClosureWork { } impl WorkItem for ClosureWork { - type Pointer = Pin>; + type Pointer = Pin>; - fn run(mut this: Pin>) { + fn run(mut this: Pin>) { if let Some(func) = this.as_mut().project().take() { (func)() } @@ -297,7 +297,7 @@ pub unsafe trait RawWorkItem { /// Defines the method that should be called directly when a work item is executed. /// -/// This trait is implemented by `Pin>` and [`Arc`], and is mainly intended to be +/// This trait is implemented by `Pin>` and [`Arc`], and is mainly intended to be /// implemented for smart pointer types. For your own structs, you would implement [`WorkItem`] /// instead. The [`run`] method on this trait will usually just perform the appropriate /// `container_of` translation and then call into the [`run`][WorkItem::run] method from the @@ -329,7 +329,7 @@ pub unsafe trait WorkItemPointer: RawWorkItem { /// This trait is used when the `work_struct` field is defined using the [`Work`] helper. pub trait WorkItem { /// The pointer type that this struct is wrapped in. This will typically be `Arc` or - /// `Pin>`. + /// `Pin>`. type Pointer: WorkItemPointer; /// The method that should be called when this work item is executed. @@ -567,7 +567,7 @@ where } // SAFETY: TODO. -unsafe impl WorkItemPointer for Pin> +unsafe impl WorkItemPointer for Pin> where T: WorkItem, T: HasWork, @@ -578,7 +578,7 @@ where // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. let ptr = unsafe { T::work_container_of(ptr) }; // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. - let boxed = unsafe { Box::from_raw(ptr) }; + let boxed = unsafe { KBox::from_raw(ptr) }; // SAFETY: The box was already pinned when it was enqueued. let pinned = unsafe { Pin::new_unchecked(boxed) }; @@ -587,7 +587,7 @@ where } // SAFETY: TODO. -unsafe impl RawWorkItem for Pin> +unsafe impl RawWorkItem for Pin> where T: WorkItem, T: HasWork, @@ -601,9 +601,9 @@ where // SAFETY: We're not going to move `self` or any of its fields, so its okay to temporarily // remove the `Pin` wrapper. let boxed = unsafe { Pin::into_inner_unchecked(self) }; - let ptr = Box::into_raw(boxed); + let ptr = KBox::into_raw(boxed); - // SAFETY: Pointers into a `Box` point at a valid value. + // SAFETY: Pointers into a `KBox` point at a valid value. let work_ptr = unsafe { T::raw_get_work(ptr) }; // SAFETY: `raw_get_work` returns a pointer to a valid value. let work_ptr = unsafe { Work::raw_get(work_ptr) }; diff --git a/rust/macros/foreach.rs b/rust/macros/foreach.rs new file mode 100644 index 00000000000000..a92280f229d1de --- /dev/null +++ b/rust/macros/foreach.rs @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0 + +use proc_macro::{ + Delimiter, TokenStream, + TokenTree::{self, Group, Ident, Literal}, +}; +use std::ops::Range; + +fn process_group(var: &str, i: isize, tokens: impl Iterator) -> Vec { + let mut tt = Vec::::new(); + let mut tokens = tokens.peekable(); + + while let Some(token) = tokens.next() { + match token { + Group(ref group) => { + let group_tokens = process_group(var, i, group.stream().into_iter()); + let stream = FromIterator::from_iter(group_tokens.into_iter()); + let new_group = proc_macro::Group::new(group.delimiter(), stream); + tt.push(TokenTree::Group(new_group)); + } + TokenTree::Punct(ref punct) => { + if punct.to_string() == "$" { + if let Some(TokenTree::Ident(ident)) = tokens.peek() { + if ident.to_string() == var { + tt.push(TokenTree::Literal(proc_macro::Literal::isize_unsuffixed(i))); + tokens.next(); + continue; + } + } + } + + tt.push(token); + } + _ => tt.push(token), + } + } + + tt +} + +pub(crate) fn expand(input: TokenStream) -> TokenStream { + let mut tokens = input.into_iter().peekable(); + + let var = if let Some(Ident(i)) = tokens.next() { + i.to_string() + } else { + panic!("foreach! first token should be an identifier"); + }; + + let token = tokens.next().expect("missing token, expecting ident 'in'"); + assert!(matches!(token, TokenTree::Ident(x) if x.to_string() == "in")); + + let token = tokens + .next() + .expect("foreach!: missing token, expecting range"); + let token = if let Group(group) = token { + group + .stream() + .into_iter() + .next() + .expect("foreach: missing token, expecting integer") + } else { + token + }; + + let start = if let Literal(lit) = token { + lit.to_string() + .parse::() + .expect("Failed to convert literal to isize") + } else { + panic!("foreach!: unexpected token '{token}'"); + }; + + let token = tokens + .next() + .expect("foreach!: missing token, expecting '.'"); + assert!(matches!(token, TokenTree::Punct(x) if x == '.')); + let token = tokens + .next() + .expect("foreach!: missing token, expecting '.'"); + assert!(matches!(token, TokenTree::Punct(x) if x == '.')); + + let is_inclusive_range = if let Some(TokenTree::Punct(p)) = tokens.peek() { + if p.as_char() == '=' { + tokens.next(); + true + } else { + false + } + } else { + false + }; + + let token = tokens + .next() + .expect("foreach!: missing token, expecting integer"); + let token = if let Group(group) = token { + group + .stream() + .into_iter() + .next() + .expect("foreach: missing token, expecting integer") + } else { + token + }; + + let end = if let Literal(lit) = token { + lit.to_string() + .parse::() + .expect("Failed to convert literal to isize") + } else { + panic!("foreach!: unexpected token '{token}'"); + }; + let range = Range { + start, + end: end + if is_inclusive_range { 1 } else { 0 }, + }; + + let tokens = if let Some(Group(group)) = tokens.next() { + if group.delimiter() != Delimiter::Brace { + panic!("foreach! expected brace"); + } + + group.stream().into_iter() + } else { + panic!("foreach! missing opening brace"); + }; + + let tokens: Vec = tokens.collect(); + let mut tt = Vec::::new(); + + for i in range { + tt.extend_from_slice(&process_group(&var, i, tokens.clone().into_iter())); + } + + FromIterator::from_iter(tt) +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index a626b1145e5c4f..0b076f9173ef9d 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -9,6 +9,7 @@ #[macro_use] mod quote; mod concat_idents; +mod foreach; mod helpers; mod module; mod paste; @@ -242,8 +243,8 @@ pub fn concat_idents(ts: TokenStream) -> TokenStream { /// #[pin_data] /// struct DriverData { /// #[pin] -/// queue: Mutex>, -/// buf: Box<[u8; 1024 * 1024]>, +/// queue: Mutex>, +/// buf: KBox<[u8; 1024 * 1024]>, /// } /// ``` /// @@ -251,8 +252,8 @@ pub fn concat_idents(ts: TokenStream) -> TokenStream { /// #[pin_data(PinnedDrop)] /// struct DriverData { /// #[pin] -/// queue: Mutex>, -/// buf: Box<[u8; 1024 * 1024]>, +/// queue: Mutex>, +/// buf: KBox<[u8; 1024 * 1024]>, /// raw_info: *mut Info, /// } /// @@ -281,8 +282,8 @@ pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream { /// #[pin_data(PinnedDrop)] /// struct DriverData { /// #[pin] -/// queue: Mutex>, -/// buf: Box<[u8; 1024 * 1024]>, +/// queue: Mutex>, +/// buf: KBox<[u8; 1024 * 1024]>, /// raw_info: *mut Info, /// } /// @@ -412,6 +413,30 @@ pub fn paste(input: TokenStream) -> TokenStream { tokens.into_iter().collect() } +/// Repeat a fragment of code and provide a numerical index for the current repetition +/// +/// # Examples +/// +/// ```rust,ignore +/// foreach!(i in 0..10) { +/// paste! { +/// fn []() { +/// } +/// } +/// } +/// +/// foreach!(i in 8..=15) { +/// paste! { +/// struct []() { +/// } +/// } +/// } +/// ``` +#[proc_macro] +pub fn foreach(input: TokenStream) -> TokenStream { + foreach::expand(input) +} + /// Derives the [`Zeroable`] trait for the given struct. /// /// This can only be used for structs where every field implements the [`Zeroable`] trait. diff --git a/rust/macros/module.rs b/rust/macros/module.rs index aef3b132f32b33..471b427bb8b6cc 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -232,6 +232,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { mod __module_init {{ mod __module_init {{ use super::super::{type_}; + use kernel::init::PinInit; /// The \"Rust loadable module\" mark. // @@ -242,7 +243,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { #[used] static __IS_RUST_MODULE: () = (); - static mut __MOD: Option<{type_}> = None; + static mut __MOD: core::mem::MaybeUninit<{type_}> = core::mem::MaybeUninit::uninit(); // Loadable modules need to export the `{{init,cleanup}}_module` identifiers. /// # Safety @@ -331,20 +332,14 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { /// /// This function must only be called once. unsafe fn __init() -> core::ffi::c_int {{ - match <{type_} as kernel::Module>::init(&super::super::THIS_MODULE) {{ - Ok(m) => {{ - // SAFETY: No data race, since `__MOD` can only be accessed by this - // module and there only `__init` and `__exit` access it. These - // functions are only called once and `__exit` cannot be called - // before or during `__init`. - unsafe {{ - __MOD = Some(m); - }} - return 0; - }} - Err(e) => {{ - return e.to_errno(); - }} + let initer = <{type_} as kernel::InPlaceModule>::init(kernel::c_str!(\"{name}\"), + &super::super::THIS_MODULE); + // SAFETY: No data race, since `__MOD` can only be accessed by this module + // and there only `__init` and `__exit` access it. These functions are only + // called once and `__exit` cannot be called before or during `__init`. + match unsafe {{ initer.__pinned_init(__MOD.as_mut_ptr()) }} {{ + Ok(m) => 0, + Err(e) => e.to_errno(), }} }} @@ -359,7 +354,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { // called once and `__init` was already called. unsafe {{ // Invokes `drop()` on `__MOD`, which should be used for cleanup. - __MOD = None; + __MOD.assume_init_drop(); }} }} diff --git a/rust/macros/paste.rs b/rust/macros/paste.rs index f40d42b35b5869..409f5bb7dfb4be 100644 --- a/rust/macros/paste.rs +++ b/rust/macros/paste.rs @@ -2,7 +2,7 @@ use proc_macro::{Delimiter, Group, Ident, Spacing, Span, TokenTree}; -fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree { +fn process_tokens_into_segments(tokens: &[TokenTree]) -> (Vec<(String, Span)>, Option) { let mut tokens = tokens.iter(); let mut segments = Vec::new(); let mut span = None; @@ -46,10 +46,29 @@ fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree { }; segments.push((value, sp)); } + Some(TokenTree::Group(group)) if group.delimiter() == Delimiter::None => { + let group_tokens: Vec<_> = group.stream().into_iter().collect(); + let (group_segments, group_span) = process_tokens_into_segments(&group_tokens); + segments.extend(group_segments); + + assert!( + !(span.is_some() && group_span.is_some()), + "span modifier should only appear at most once" + ); + + if group_span.is_some() { + span = group_span; + } + } _ => panic!("unexpected token in paste segments"), }; } + (segments, span) +} + +fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree { + let (segments, span) = process_tokens_into_segments(tokens); let pasted: String = segments.into_iter().map(|x| x.0).collect(); TokenTree::Ident(Ident::new(&pasted, span.unwrap_or(group_span))) } diff --git a/samples/rust/rust_minimal.rs b/samples/rust/rust_minimal.rs index 2a9eaab62d1ca7..1577dc34e563d4 100644 --- a/samples/rust/rust_minimal.rs +++ b/samples/rust/rust_minimal.rs @@ -13,15 +13,15 @@ module! { } struct RustMinimal { - numbers: Vec, + numbers: KVec, } impl kernel::Module for RustMinimal { - fn init(_module: &'static ThisModule) -> Result { + fn init(_name: &'static CStr, _module: &'static ThisModule) -> Result { pr_info!("Rust minimal sample (init)\n"); pr_info!("Am I built-in? {}\n", !cfg!(MODULE)); - let mut numbers = Vec::new(); + let mut numbers = KVec::new(); numbers.push(72, GFP_KERNEL)?; numbers.push(108, GFP_KERNEL)?; numbers.push(200, GFP_KERNEL)?; diff --git a/samples/rust/rust_print.rs b/samples/rust/rust_print.rs index ba1606bdbd7543..73763ea2dc09d2 100644 --- a/samples/rust/rust_print.rs +++ b/samples/rust/rust_print.rs @@ -41,7 +41,7 @@ fn arc_print() -> Result { } impl kernel::Module for RustPrint { - fn init(_module: &'static ThisModule) -> Result { + fn init(_name: &'static CStr, _module: &'static ThisModule) -> Result { pr_info!("Rust printing macros sample (init)\n"); pr_emerg!("Emergency message (level 0) without args\n"); diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 64518c2f3d9c3a..2bba59e790b8a4 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -248,7 +248,7 @@ $(obj)/%.lst: $(obj)/%.c FORCE # Compile Rust sources (.rs) # --------------------------------------------------------------------------- -rust_allowed_features := arbitrary_self_types,lint_reasons,new_uninit +rust_allowed_features := arbitrary_self_types,lint_reasons # `--out-dir` is required to avoid temporaries being created by `rustc` in the # current working directory, which may be not accessible in the out-of-tree @@ -258,7 +258,7 @@ rust_common_cmd = \ -Zallow-features=$(rust_allowed_features) \ -Zcrate-attr=no_std \ -Zcrate-attr='feature($(rust_allowed_features))' \ - -Zunstable-options --extern force:alloc --extern kernel \ + -Zunstable-options --extern kernel \ --crate-type rlib -L $(objtree)/rust/ \ --crate-name $(basename $(notdir $@)) \ --sysroot=/dev/null \ diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index d2bc63cde8c6a3..09e1d166d8d236 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -64,13 +64,6 @@ def append_crate(display_name, root_module, deps, cfg=[], is_workspace_member=Tr [], ) - append_crate( - "alloc", - sysroot_src / "alloc" / "src" / "lib.rs", - ["core", "compiler_builtins"], - cfg=crates_cfgs.get("alloc", []), - ) - append_crate( "macros", srctree / "rust" / "macros" / "lib.rs", @@ -96,7 +89,7 @@ def append_crate(display_name, root_module, deps, cfg=[], is_workspace_member=Tr append_crate( "kernel", srctree / "rust" / "kernel" / "lib.rs", - ["core", "alloc", "macros", "build_error", "bindings"], + ["core", "macros", "build_error", "bindings"], cfg=cfg, ) crates[-1]["source"] = { @@ -133,7 +126,7 @@ def is_root_crate(build_file, target): append_crate( name, path, - ["core", "alloc", "kernel"], + ["core", "kernel"], cfg=cfg, )