diff --git a/core/arch/arm/include/kernel/thread_arch.h b/core/arch/arm/include/kernel/thread_arch.h index 4d4765d5b39..9d359b774ae 100644 --- a/core/arch/arm/include/kernel/thread_arch.h +++ b/core/arch/arm/include/kernel/thread_arch.h @@ -437,6 +437,19 @@ static inline void thread_user_clear_vfp(struct user_mode_ctx *uctx __unused) } #endif +/* + * thread_user_clear_vector() - Releases a user context's vector state + * @uctx: the user context being torn down + * + * Only RISC-V has a vector register file to release, and only there is it + * heap allocated. The generic teardown paths call this next to + * thread_user_clear_vfp(), so every other architecture needs the empty + * definition. + */ +static inline void thread_user_clear_vector(struct user_mode_ctx *uctx __unused) +{ +} + #ifdef ARM64 /* * thread_get_saved_thread_sp() - Returns the saved sp of current thread diff --git a/core/arch/riscv/include/kernel/thread_arch.h b/core/arch/riscv/include/kernel/thread_arch.h index bd6a6a10fcd..0f9e1ab5c15 100644 --- a/core/arch/riscv/include/kernel/thread_arch.h +++ b/core/arch/riscv/include/kernel/thread_arch.h @@ -13,6 +13,7 @@ #include #include +#include /* * Each RISC-V platform must define their own values. @@ -167,6 +168,29 @@ struct thread_ctx_regs { struct user_mode_ctx; +#ifdef CFG_RISCV_WITH_VECTOR +struct thread_user_vector_state { + /* Allocated the first time the TA asks for the vector unit */ + struct riscv_vector_state *state; + /* True when @state holds a saved copy of the TA's vector registers */ + bool valid; +}; + +/* Returns false if a context could not be allocated for the TA */ +bool thread_user_enable_vector(struct thread_user_vector_state *uvect); +void thread_user_save_vector(void); +void thread_user_clear_vector(struct user_mode_ctx *uctx); +#else /*CFG_RISCV_WITH_VECTOR*/ +static inline void thread_user_save_vector(void) +{ +} + +static inline void thread_user_clear_vector(struct user_mode_ctx *uctx + __unused) +{ +} +#endif /*CFG_RISCV_WITH_VECTOR*/ + #ifdef CFG_WITH_VFP uint32_t thread_kernel_enable_vfp(void); void thread_kernel_disable_vfp(uint32_t state); diff --git a/core/arch/riscv/include/kernel/thread_private_arch.h b/core/arch/riscv/include/kernel/thread_private_arch.h index add1d34008b..78b89587d28 100644 --- a/core/arch/riscv/include/kernel/thread_private_arch.h +++ b/core/arch/riscv/include/kernel/thread_private_arch.h @@ -9,6 +9,7 @@ #ifndef __ASSEMBLER__ #include +#include #define STACK_TMP_OFFS 0 @@ -58,6 +59,40 @@ struct thread_user_mode_rec { unsigned long x[13]; }; +#ifdef CFG_RISCV_WITH_VECTOR +/* + * Which context the vector registers of this hart currently hold. + * + * OP-TEE owns xstatus.VS while it runs, so one owner per OP-TEE thread is + * enough to describe the hardware: the registers hold either nothing worth + * preserving, the normal world context, or the context of the TA the + * thread is running. + */ +enum riscv_vector_owner { + RISCV_VECTOR_OWNER_NONE = 0, + RISCV_VECTOR_OWNER_NS, + RISCV_VECTOR_OWNER_USER, +}; + +/* + * struct thread_vector_state - per OP-TEE thread vector bookkeeping + * @ns: saved normal world vector context, allocated at boot + * @uvect: vector context of the TA this thread is running, if it has one + * @owner: context the vector registers currently hold + * @ns_vs: xstatus.VS the normal world had on entry to OP-TEE + * @ns_valid: @ns holds a saved normal world context + * @sec_used: secure code has written the vector registers since entry + */ +struct thread_vector_state { + struct riscv_vector_state *ns; + struct thread_user_vector_state *uvect; + enum riscv_vector_owner owner; + unsigned long ns_vs; + bool ns_valid; + bool sec_used; +}; +#endif /*CFG_RISCV_WITH_VECTOR*/ + extern long thread_user_kcode_offset; void thread_native_interrupt_handler(struct thread_ctx_regs *regs, diff --git a/core/arch/riscv/include/riscv.h b/core/arch/riscv/include/riscv.h index d7d60036972..45568bc047d 100644 --- a/core/arch/riscv/include/riscv.h +++ b/core/arch/riscv/include/riscv.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-2-Clause */ /* * Copyright 2022-2023 NXP + * Copyright (c) 2026, RISCStar Solutions Limited */ #ifndef __RISCV_H @@ -75,6 +76,19 @@ #define CSR_XSTATUS_SUM BIT(18) #define CSR_XSTATUS_MXR BIT(19) +/* + * xstatus.VS holds the state of the vector unit. The field is two bits wide + * and always resides in the low 32 bits of xstatus, on RV32 as well as on + * RV64. + */ +#define CSR_XSTATUS_VS_SHIFT 9 +#define CSR_XSTATUS_VS_MASK SHIFT_U32(3, CSR_XSTATUS_VS_SHIFT) + +#define CSR_XSTATUS_VS_OFF 0 +#define CSR_XSTATUS_VS_INITIAL 1 +#define CSR_XSTATUS_VS_CLEAN 2 +#define CSR_XSTATUS_VS_DIRTY 3 + #define CSR_XCAUSE_INTR_FLAG BIT64(__riscv_xlen - 1) #ifndef __ASSEMBLER__ diff --git a/core/arch/riscv/include/riscv_macros.S b/core/arch/riscv/include/riscv_macros.S index ff40b872306..fea79ff1a1e 100644 --- a/core/arch/riscv/include/riscv_macros.S +++ b/core/arch/riscv/include/riscv_macros.S @@ -2,6 +2,7 @@ /* * Copyright 2022-2023 NXP * Copyright (c) 2015, Linaro Limited + * Copyright (c) 2026, RISCStar Solutions Limited */ .altmacro @@ -49,6 +50,48 @@ \from_regnum, \to_regnum .endm + /* + * This helper macro transfers the whole vector register file, eight + * registers at a time. + * + * The whole-register forms vs8r.v and vl8r.v are used because they + * move a fixed amount of state regardless of the vtype and vl in + * effect, so the context can be moved without first having to + * reconstruct the configuration it was taken under. + * + * base_reg is advanced by 8 * vlenb between chunks and is left + * pointing past the end. tmp_reg is clobbered. The vector ISA is + * enabled for this block alone, so the file assembles for a core + * that is otherwise built without it. + */ + .macro _do_vregs instr, base_reg, vlenb_reg, tmp_reg + .option push + .option arch, +v + slli \tmp_reg, \vlenb_reg, 3 + \instr v0, (\base_reg) + add \base_reg, \base_reg, \tmp_reg + \instr v8, (\base_reg) + add \base_reg, \base_reg, \tmp_reg + \instr v16, (\base_reg) + add \base_reg, \base_reg, \tmp_reg + \instr v24, (\base_reg) + .option pop + .endm + + /* + * Stores registers v0..v31 at [base_reg], vlenb bytes each + */ + .macro store_vregs base_reg, vlenb_reg, tmp_reg + _do_vregs vs8r.v, \base_reg, \vlenb_reg, \tmp_reg + .endm + + /* + * Loads registers v0..v31 from [base_reg], vlenb bytes each + */ + .macro load_vregs base_reg, vlenb_reg, tmp_reg + _do_vregs vl8r.v, \base_reg, \vlenb_reg, \tmp_reg + .endm + /* * Multiplication macro for RISC-V harts without M extension. */ diff --git a/core/arch/riscv/include/riscv_vector.h b/core/arch/riscv/include/riscv_vector.h new file mode 100644 index 00000000000..abc9d95a908 --- /dev/null +++ b/core/arch/riscv/include/riscv_vector.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2026, RISCStar Solutions Limited + */ + +#ifndef __RISCV_VECTOR_H +#define __RISCV_VECTOR_H + +#ifndef __ASSEMBLER__ + +#include +#include +#include + +/* + * The width of a vector register is discovered at run time from vlenb and + * can be anything from 8 bytes upwards, so a context cannot be sized from + * the ISA the core was built for. The register file is a flexible array and + * a context is allocated for the hart it will run on. + */ +#define RISCV_VECTOR_NUM_REGS 32 + +/* + * struct riscv_vector_state - vector register context + * @vstart: index the next vector instruction would resume from + * @vtype: current vector type, read only, put back through vsetvl + * @vl: current vector length, likewise + * @vcsr: vector rounding mode and saturation flag + * @vregs: v0..v31, each vlenb bytes wide and laid out back to back + * + * The layout is shared with riscv_vector.S, see the RISCV_VECTOR_*_OFF + * defines. + */ +struct riscv_vector_state { + unsigned long vstart; + unsigned long vtype; + unsigned long vl; + unsigned long vcsr; + uint8_t vregs[]; +}; + +/* + * Bytes one context occupies on this hart, the header plus 32 * vlenb. + * Reads vlenb, so it enables the vector unit for the read the same way the + * save and restore routines do. + */ +size_t riscv_vector_state_size(void); + +void riscv_vector_save_state(struct riscv_vector_state *state); +void riscv_vector_restore_state(struct riscv_vector_state *state); + +/* + * xstatus.VS holds the state of the vector unit and takes the same four + * values as the FS field does for floating point: + * + * Off the unit is disabled, a vector instruction or an access to a + * vector CSR traps as an illegal instruction + * Initial enabled, the registers hold their initial value + * Clean enabled, the registers match the copy held in memory + * Dirty enabled, the registers have been written since they were last + * saved + * + * The helpers come in two flavours: those taking an xstatus value operate + * on a saved context, since xstatus is part of the register frame restored + * on the way back to a trapped context, while riscv_vector_write_vs() and + * friends act on the live CSR of this hart. + */ + +static inline unsigned long riscv_vector_get_vs(unsigned long xstatus) +{ +#ifdef RV32 + return get_field_u32(xstatus, CSR_XSTATUS_VS_MASK); +#else + return get_field_u64(xstatus, CSR_XSTATUS_VS_MASK); +#endif +} + +static inline unsigned long riscv_vector_set_vs(unsigned long xstatus, + unsigned long vs) +{ +#ifdef RV32 + return set_field_u32(xstatus, CSR_XSTATUS_VS_MASK, vs); +#else + return set_field_u64(xstatus, CSR_XSTATUS_VS_MASK, vs); +#endif +} + +/* Returns true if @xstatus describes a context with the vector unit on */ +static inline bool riscv_vector_state_is_enabled(unsigned long xstatus) +{ + return riscv_vector_get_vs(xstatus) != CSR_XSTATUS_VS_OFF; +} + +static inline void riscv_vector_write_vs(unsigned long vs) +{ + write_csr(CSR_XSTATUS, + riscv_vector_set_vs(read_csr(CSR_XSTATUS), vs)); +} + +static inline unsigned long riscv_vector_read_vs(void) +{ + return riscv_vector_get_vs(read_csr(CSR_XSTATUS)); +} + +static inline void riscv_vector_disable(void) +{ + clear_csr(CSR_XSTATUS, CSR_XSTATUS_VS_MASK); +} + +static inline bool riscv_vector_is_enabled(void) +{ + return riscv_vector_read_vs() != CSR_XSTATUS_VS_OFF; +} + +/* + * vlenb is only readable with the unit enabled, so this leaves xstatus.VS + * as it found it the same way the save and restore routines do. + */ +static inline unsigned long riscv_vector_vlenb(void) +{ + unsigned long xstatus = read_csr(CSR_XSTATUS); + unsigned long vlenb = 0; + + riscv_vector_write_vs(CSR_XSTATUS_VS_INITIAL); + vlenb = read_csr(CSR_VLENB); + write_csr(CSR_XSTATUS, xstatus); + + return vlenb; +} + +#endif /* !__ASSEMBLER__ */ +#endif /* __RISCV_VECTOR_H */ diff --git a/core/arch/riscv/kernel/abort.c b/core/arch/riscv/kernel/abort.c index d5af8b190ff..426f25c3867 100644 --- a/core/arch/riscv/kernel/abort.c +++ b/core/arch/riscv/kernel/abort.c @@ -2,6 +2,7 @@ /* * Copyright 2022-2023 NXP * Copyright (c) 2015-2022, Linaro Limited + * Copyright (c) 2026, RISCStar Solutions Limited */ #include @@ -14,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -21,6 +23,7 @@ enum fault_type { FAULT_TYPE_USER_MODE_PANIC, FAULT_TYPE_USER_MODE_VFP, + FAULT_TYPE_USER_MODE_VECTOR, FAULT_TYPE_PAGE_FAULT, FAULT_TYPE_IGNORE, }; @@ -277,11 +280,141 @@ static bool is_vfp_fault(struct abort_info *ai __unused) } #endif /*CFG_WITH_VFP && CFG_WITH_USER_TA*/ +#if defined(CFG_RISCV_WITH_VECTOR) && defined(CFG_WITH_USER_TA) +#define OPCODE_MASK 0x7f +#define OPCODE_LOAD_FP 0x07 /* also the vector loads */ +#define OPCODE_STORE_FP 0x27 /* also the vector stores */ +#define OPCODE_OP_V 0x57 /* vadd, vsetvli, vsetvl, ... */ +#define OPCODE_SYSTEM 0x73 /* csrrw and friends */ + +#define INSN_FUNCT3_SHIFT 12 +#define INSN_FUNCT3_MASK 0x7 +#define INSN_CSR_SHIFT 20 +#define INSN_CSR_MASK 0xfff + +/* + * The vector loads and stores share their major opcodes with the + * floating-point ones and are told apart by the width field, which holds + * the three reserved-for-vector encodings rather than one of the + * floating-point widths. + */ +#define WIDTH_VECTOR_8 0 +#define WIDTH_VECTOR_16 5 +#define WIDTH_VECTOR_32 6 +#define WIDTH_VECTOR_64 7 + +static bool is_vector_ldst_insn(uint32_t insn) +{ + uint32_t width = (insn >> INSN_FUNCT3_SHIFT) & INSN_FUNCT3_MASK; + + switch (width) { + case WIDTH_VECTOR_8: + case WIDTH_VECTOR_16: + case WIDTH_VECTOR_32: + case WIDTH_VECTOR_64: + return true; + default: + return false; + } +} + +/* + * With VS == Off a hart traps reads and writes of the vector CSRs exactly + * as it traps the vector instructions themselves, so a TA that asks for + * vlenb or sets a rounding mode before it issues any vector instruction + * has to be given a context just the same. + */ +static bool is_vector_csr_insn(uint32_t insn) +{ + uint32_t csr = (insn >> INSN_CSR_SHIFT) & INSN_CSR_MASK; + + /* funct3 zero is ecall, ebreak and friends, not a CSR access */ + if (!((insn >> INSN_FUNCT3_SHIFT) & INSN_FUNCT3_MASK)) + return false; + + switch (csr) { + case CSR_VSTART: + case CSR_VXSAT: + case CSR_VXRM: + case CSR_VCSR: + case CSR_VL: + case CSR_VTYPE: + case CSR_VLENB: + return true; + default: + return false; + } +} + +static bool is_vector_insn(uint32_t insn) +{ + switch (insn & OPCODE_MASK) { + case OPCODE_OP_V: + return true; + case OPCODE_LOAD_FP: + case OPCODE_STORE_FP: + return is_vector_ldst_insn(insn); + case OPCODE_SYSTEM: + return is_vector_csr_insn(insn); + default: + return false; + } +} + +/* + * A vector instruction executed with xstatus.VS == Off traps as an illegal + * instruction. Recognising that case is what turns the first vector use by + * a TA into a request for a context instead of a panic. + * + * The decision is made on the faulting instruction, which the hart reports + * in xtval, so that a TA executing a genuinely illegal instruction still + * panics rather than being resumed on it forever. Vector instructions are + * always 32 bits wide, there are no compressed forms. + */ +static bool is_vector_fault(struct abort_info *ai) +{ + if (ai->regs->cause != CAUSE_ILLEGAL_INSTRUCTION) + return false; + + /* Only a context that had vector disabled can be asking for it */ + if (riscv_vector_state_is_enabled(ai->regs->status)) + return false; + + return is_vector_insn(ai->regs->tval); +} + +static bool handle_user_mode_vector(struct abort_info *ai) +{ + struct ts_session *s = ts_get_current_session(); + + if (!thread_user_enable_vector(&to_user_mode_ctx(s->ctx)->vector)) + return false; + + /* + * xstatus is restored from the saved context on the way back to the + * TA, so handing it the vector unit means updating VS there and not + * only in the live CSR. Without this the TA would resume with VS + * still Off and trap on the very same instruction again. + */ + ai->regs->status = riscv_vector_set_vs(ai->regs->status, + riscv_vector_read_vs()); + + return true; +} +#else /*CFG_RISCV_WITH_VECTOR && CFG_WITH_USER_TA*/ +static bool is_vector_fault(struct abort_info *ai __unused) +{ + return false; +} +#endif /*CFG_RISCV_WITH_VECTOR && CFG_WITH_USER_TA*/ + static enum fault_type get_fault_type(struct abort_info *ai) { if (abort_is_user_exception(ai)) { if (is_vfp_fault(ai)) return FAULT_TYPE_USER_MODE_VFP; + if (is_vector_fault(ai)) + return FAULT_TYPE_USER_MODE_VECTOR; return FAULT_TYPE_USER_MODE_PANIC; } @@ -366,6 +499,15 @@ void abort_handler(uint32_t abort_type, struct thread_abort_regs *regs) case FAULT_TYPE_USER_MODE_VFP: handle_user_mode_vfp(); break; +#endif +#if defined(CFG_RISCV_WITH_VECTOR) && defined(CFG_WITH_USER_TA) + case FAULT_TYPE_USER_MODE_VECTOR: + if (!handle_user_mode_vector(&ai)) { + EMSG("Out of memory for a TA vector context"); + save_abort_info_in_tsd(&ai); + handle_user_mode_panic(&ai); + } + break; #endif case FAULT_TYPE_PAGE_FAULT: default: diff --git a/core/arch/riscv/kernel/asm-defines.c b/core/arch/riscv/kernel/asm-defines.c index dad8cb3950b..75772ea5b42 100644 --- a/core/arch/riscv/kernel/asm-defines.c +++ b/core/arch/riscv/kernel/asm-defines.c @@ -10,6 +10,7 @@ #include #include #include +#include #include DEFINES @@ -109,4 +110,18 @@ DEFINES /* struct thread_abi_args */ DEFINE(THREAD_ABI_ARGS_A0, offsetof(struct thread_abi_args, a0)); DEFINE(THREAD_ABI_ARGS_SIZE, sizeof(struct thread_abi_args)); + +#ifdef CFG_RISCV_WITH_VECTOR + /* struct riscv_vector_state */ + DEFINE(RISCV_VECTOR_VSTART_OFF, + offsetof(struct riscv_vector_state, vstart)); + DEFINE(RISCV_VECTOR_VTYPE_OFF, + offsetof(struct riscv_vector_state, vtype)); + DEFINE(RISCV_VECTOR_VL_OFF, offsetof(struct riscv_vector_state, vl)); + DEFINE(RISCV_VECTOR_VCSR_OFF, + offsetof(struct riscv_vector_state, vcsr)); + DEFINE(RISCV_VECTOR_VREGS_OFF, + offsetof(struct riscv_vector_state, vregs)); + DEFINE(RISCV_VECTOR_NREGS, RISCV_VECTOR_NUM_REGS); +#endif } diff --git a/core/arch/riscv/kernel/riscv_vector.S b/core/arch/riscv/kernel/riscv_vector.S new file mode 100644 index 00000000000..f80367cdb6d --- /dev/null +++ b/core/arch/riscv/kernel/riscv_vector.S @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2026, RISCStar Solutions Limited + */ + +#include +#include +#include +#include + +/* + * size_t riscv_vector_state_size(void) + * + * Returns the header plus 32 * vlenb. vlenb is only readable with the + * vector unit enabled, so it is enabled for the read and xstatus is put + * back untouched, as in the two routines below. + */ +FUNC riscv_vector_state_size , : + csrr t1, CSR_XSTATUS + li t0, CSR_XSTATUS_VS_MASK + csrs CSR_XSTATUS, t0 + + csrr a0, CSR_VLENB + + csrw CSR_XSTATUS, t1 + + li t0, RISCV_VECTOR_NREGS + mul a0, a0, t0 + addi a0, a0, RISCV_VECTOR_VREGS_OFF + ret +END_FUNC riscv_vector_state_size + +/* + * void riscv_vector_save_state(struct riscv_vector_state *state) + * + * Saves v0..v31 and the vector CSRs. The vector unit is enabled for the + * duration of the transfer and xstatus is written back unmodified before + * returning, so the caller's VS state is preserved. + */ +FUNC riscv_vector_save_state , : + csrr t1, CSR_XSTATUS + li t0, CSR_XSTATUS_VS_MASK + csrs CSR_XSTATUS, t0 + + /* Take the CSRs before vstart is cleared below */ + csrr t2, CSR_VSTART + STR t2, RISCV_VECTOR_VSTART_OFF(a0) + csrr t2, CSR_VTYPE + STR t2, RISCV_VECTOR_VTYPE_OFF(a0) + csrr t2, CSR_VL + STR t2, RISCV_VECTOR_VL_OFF(a0) + csrr t2, CSR_VCSR + STR t2, RISCV_VECTOR_VCSR_OFF(a0) + + /* + * The whole-register stores below honour vstart and would skip the + * elements below it, so start them from zero. + */ + csrw CSR_VSTART, zero + + csrr t2, CSR_VLENB + addi a1, a0, RISCV_VECTOR_VREGS_OFF + store_vregs a1, t2, t3 + + csrw CSR_XSTATUS, t1 + ret +END_FUNC riscv_vector_save_state + +/* + * void riscv_vector_restore_state(struct riscv_vector_state *state) + * + * Loads v0..v31 and the vector CSRs. Enables and restores xstatus as above. + */ +FUNC riscv_vector_restore_state , : + csrr t1, CSR_XSTATUS + li t0, CSR_XSTATUS_VS_MASK + csrs CSR_XSTATUS, t0 + + /* As above, the whole-register loads must start from vstart zero */ + csrw CSR_VSTART, zero + + csrr t2, CSR_VLENB + addi a1, a0, RISCV_VECTOR_VREGS_OFF + load_vregs a1, t2, t3 + + /* + * vl and vtype are read only, the pair is put back by re-running the + * vsetvl that produced them. + */ + LDR t2, RISCV_VECTOR_VL_OFF(a0) + LDR t3, RISCV_VECTOR_VTYPE_OFF(a0) + .option push + .option arch, +v + vsetvl zero, t2, t3 + .option pop + + LDR t2, RISCV_VECTOR_VSTART_OFF(a0) + csrw CSR_VSTART, t2 + LDR t2, RISCV_VECTOR_VCSR_OFF(a0) + csrw CSR_VCSR, t2 + + csrw CSR_XSTATUS, t1 + ret +END_FUNC riscv_vector_restore_state diff --git a/core/arch/riscv/kernel/sub.mk b/core/arch/riscv/kernel/sub.mk index 942e4d0eec9..b4fadc01e86 100644 --- a/core/arch/riscv/kernel/sub.mk +++ b/core/arch/riscv/kernel/sub.mk @@ -18,6 +18,7 @@ srcs-$(CFG_UNWIND) += unwind_rv.c srcs-$(CFG_SEMIHOSTING) += semihosting_rv.S srcs-y += thread_optee_abi.c srcs-y += thread_optee_abi_rv.S +srcs-$(CFG_RISCV_WITH_VECTOR) += riscv_vector.S asm-defines-y += asm-defines.c ifeq ($(CFG_SYSCALL_FTRACE),y) diff --git a/core/arch/riscv/kernel/thread_arch.c b/core/arch/riscv/kernel/thread_arch.c index 58b141d9b37..73a50f67920 100644 --- a/core/arch/riscv/kernel/thread_arch.c +++ b/core/arch/riscv/kernel/thread_arch.c @@ -29,7 +29,11 @@ #include #include #include +#include +#include #include +#include +#include #include #include @@ -96,6 +100,158 @@ static void thread_lazy_restore_ns_vfp(void) static_assert(!IS_ENABLED(CFG_WITH_VFP)); } +#ifdef CFG_RISCV_WITH_VECTOR +/* + * The normal world vector context is switched eagerly at the domain + * boundary: the registers are saved on the way in and the unit is + * disabled, so nothing of the normal world is reachable from secure code, + * and whatever secure code puts in the registers is overwritten before the + * normal world resumes. + */ +static void thread_vector_release(struct thread_ctx *thr) +{ + switch (thr->vector_state.owner) { + case RISCV_VECTOR_OWNER_NONE: + return; + case RISCV_VECTOR_OWNER_NS: + riscv_vector_save_state(thr->vector_state.ns); + thr->vector_state.ns_valid = true; + break; + case RISCV_VECTOR_OWNER_USER: + assert(thr->vector_state.uvect && + thr->vector_state.uvect->state); + riscv_vector_save_state(thr->vector_state.uvect->state); + thr->vector_state.uvect->valid = true; + break; + default: + panic(); + } + + thr->vector_state.owner = RISCV_VECTOR_OWNER_NONE; + riscv_vector_disable(); +} + +/* + * A context is sized from the vlenb of the hart it will run on, so the + * normal world contexts are allocated once here rather than in the domain + * switch, which then has no allocation and so no failure path in it. + */ +static TEE_Result riscv_vector_init(void) +{ + size_t size = riscv_vector_state_size(); + size_t n = 0; + + for (n = 0; n < CFG_NUM_THREADS; n++) { + threads[n].vector_state.ns = memalign(__alignof__(long), size); + if (!threads[n].vector_state.ns) { + EMSG("Failed to allocate %zu bytes of vector context", + size); + panic(); + } + memset(threads[n].vector_state.ns, 0, size); + } + + DMSG("Vector context switching enabled, vlenb %lu, %zu bytes a context", + riscv_vector_vlenb(), size); + + return TEE_SUCCESS; +} +service_init(riscv_vector_init); +#endif /*CFG_RISCV_WITH_VECTOR*/ + +static void init_vector_state(struct thread_ctx *thread __maybe_unused) +{ +#ifdef CFG_RISCV_WITH_VECTOR + /* + * The thread slot may have been used before, so start from a clean + * state, keeping the buffer allocated for it at boot. The vector + * registers hold the normal world context at this point, which + * thread_save_ns_vector() takes care of below. + */ + struct riscv_vector_state *ns = thread->vector_state.ns; + + memset(&thread->vector_state, 0, sizeof(thread->vector_state)); + thread->vector_state.ns = ns; + thread->vector_state.owner = RISCV_VECTOR_OWNER_NS; +#endif /*CFG_RISCV_WITH_VECTOR*/ +} + +/* + * A TA runs with the vector unit disabled until it executes a vector + * instruction and traps, so whenever its context is saved and released the + * saved xstatus has to go back to VS == Off. VS is part of the register + * frame restored on the way back to user mode, so it is the saved copy + * that decides whether the TA traps again. + */ +static unsigned long user_status_disable_vector(unsigned long status) +{ + if (IS_ENABLED(CFG_RISCV_WITH_VECTOR)) + return riscv_vector_set_vs(status, CSR_XSTATUS_VS_OFF); + + return status; +} + +static void thread_save_ns_vector(void) +{ +#ifdef CFG_RISCV_WITH_VECTOR + struct thread_ctx *thr = threads + thread_get_id(); + + assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); + assert(thr->vector_state.owner == RISCV_VECTOR_OWNER_NS); + + /* + * The save cannot be made conditional on the normal world VS. A + * monitor that gives each domain its own S-mode CSRs, as OpenSBI + * does for the domain it runs OP-TEE in, swaps xstatus across the + * domain switch while leaving the vector registers shared. The VS + * read here is therefore OP-TEE's own and says nothing about + * whether the normal world has a live vector context. + * + * VS is saved and put back all the same, so that a monitor which + * does share xstatus between the domains gets the normal world VS + * it had on entry rather than the Off that OP-TEE runs with. + */ + assert(thr->vector_state.ns); + thr->vector_state.ns_vs = riscv_vector_read_vs(); + riscv_vector_save_state(thr->vector_state.ns); + thr->vector_state.ns_valid = true; + thr->vector_state.sec_used = false; + + thr->vector_state.owner = RISCV_VECTOR_OWNER_NONE; + riscv_vector_disable(); +#endif /*CFG_RISCV_WITH_VECTOR*/ +} + +static void thread_restore_ns_vector(void) +{ +#ifdef CFG_RISCV_WITH_VECTOR + struct thread_ctx *thr = threads + thread_get_id(); + + assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); + + /* Release the registers from whatever secure context holds them */ + thread_vector_release(thr); + + /* + * This is where the VS tracking pays off. Secure code only reaches + * the vector registers by taking VS out of Off, so if that never + * happened they still hold exactly what the normal world left in + * them and the restore can be skipped. A thread running a TA that + * never touches vector, which is most of them, costs one save per + * call and no restore. + */ + if (thr->vector_state.sec_used) { + assert(thr->vector_state.ns_valid); + riscv_vector_restore_state(thr->vector_state.ns); + } + + thr->vector_state.ns_valid = false; + thr->vector_state.sec_used = false; + riscv_vector_write_vs(thr->vector_state.ns_vs); + thr->vector_state.owner = RISCV_VECTOR_OWNER_NS; +#endif /*CFG_RISCV_WITH_VECTOR*/ +} + static void setup_unwind_user_mode(struct thread_scall_regs *regs) { regs->epc = (uintptr_t)thread_unwind_user_mode; @@ -128,6 +284,8 @@ void thread_scall_handler(struct thread_scall_regs *regs) thread_unmask_exceptions(state & ~THREAD_EXCP_NATIVE_INTR); thread_user_save_vfp(); + thread_user_save_vector(); + regs->status = user_status_disable_vector(regs->status); sess = ts_get_current_session(); @@ -255,8 +413,10 @@ static void __thread_alloc_and_run(uint32_t a0, uint32_t a1, uint32_t a2, threads[n].flags = 0; init_regs(threads + n, a0, a1, a2, a3, a4, a5, a6, a7, pc); + init_vector_state(threads + n); thread_lazy_save_ns_vfp(); + thread_save_ns_vector(); l->flags &= ~THREAD_CLF_TMP; @@ -382,6 +542,7 @@ void thread_resume_from_rpc(uint32_t thread_id, uint32_t a0, uint32_t a1, } thread_lazy_save_ns_vfp(); + thread_save_ns_vector(); if (threads[n].have_user_map) ftrace_resume(); @@ -400,6 +561,7 @@ void thread_state_free(void) assert(ct != THREAD_ID_INVALID); thread_lazy_restore_ns_vfp(); + thread_restore_ns_vector(); thread_lock_global(); @@ -427,10 +589,13 @@ int thread_state_suspend(uint32_t flags, unsigned long status, vaddr_t pc) if (is_from_user(status)) { thread_user_save_vfp(); + thread_user_save_vector(); + status = user_status_disable_vector(status); tee_ta_update_session_utime_suspend(); tee_ta_gprof_sample_pc(pc); } thread_lazy_restore_ns_vfp(); + thread_restore_ns_vector(); thread_lock_global(); @@ -542,7 +707,8 @@ uint32_t thread_enter_user_mode(unsigned long a0, unsigned long a1, */ exceptions = thread_mask_exceptions(THREAD_EXCP_ALL); regs = thread_get_ctx_regs(); - status = xstatus_for_xret(true, PRV_U); + /* A TA starts with vector disabled and is given it on first use */ + status = user_status_disable_vector(xstatus_for_xret(true, PRV_U)); set_ctx_regs(regs, a0, a1, a2, a3, user_sp, entry_func, status, ie, NULL); rc = __thread_enter_user_mode(regs, exit_status0, exit_status1); @@ -555,3 +721,104 @@ void __thread_rpc(uint32_t rv[THREAD_RPC_NUM_ARGS]) { thread_rpc_xstatus(rv, xstatus_for_xret(false, PRV_S)); } + +#ifdef CFG_RISCV_WITH_VECTOR +static void thread_vector_clear_regs(struct riscv_vector_state *state) +{ + memset(state, 0, riscv_vector_state_size()); + riscv_vector_restore_state(state); +} + +/* + * A TA is given the vector unit the first time it executes a vector + * instruction, rather than on every entry to user mode. Within the TEE, + * OP-TEE alone owns VS and no other context can look at the vector + * registers behind our back, so there is nothing to defend against here, + * and most TAs never touch vector. Enabling on first use keeps those TAs + * from paying for a context switch they have no use for, which for vector + * is a good deal larger than it is for floating point. + */ +bool thread_user_enable_vector(struct thread_user_vector_state *uvect) +{ + struct thread_ctx *thr = threads + thread_get_id(); + uint32_t exceptions = 0; + + assert(uvect); + + /* + * The TA's context is allocated the first time it asks for the + * vector unit, so a TA that never uses vector costs nothing beyond + * the flag. Do it before masking, allocation may sleep on a mutex. + */ + if (!uvect->state) { + uvect->state = memalign(__alignof__(long), + riscv_vector_state_size()); + if (!uvect->state) + return false; + uvect->valid = false; + } + + exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + + /* Take the vector registers away from whoever holds them */ + thread_vector_release(thr); + + if (uvect->valid) { + riscv_vector_restore_state(uvect->state); + riscv_vector_write_vs(CSR_XSTATUS_VS_CLEAN); + } else { + /* + * This TA has no vector context yet. Clear the registers + * before handing them over: VS == Initial says the context + * is new, but enabling the unit does not architecturally + * clear the register file, so the TA would otherwise start + * out seeing what the previous owner left there. + */ + thread_vector_clear_regs(uvect->state); + riscv_vector_write_vs(CSR_XSTATUS_VS_INITIAL); + } + + thr->vector_state.uvect = uvect; + thr->vector_state.owner = RISCV_VECTOR_OWNER_USER; + thr->vector_state.sec_used = true; + + thread_set_exceptions(exceptions); + + return true; +} + +void thread_user_save_vector(void) +{ + struct thread_ctx *thr = threads + thread_get_id(); + + assert(thread_get_exceptions() & THREAD_EXCP_FOREIGN_INTR); + + /* The TA either never used vector or has already been saved */ + if (thr->vector_state.owner != RISCV_VECTOR_OWNER_USER) + return; + + thread_vector_release(thr); +} + +void thread_user_clear_vector(struct user_mode_ctx *uctx) +{ + struct thread_user_vector_state *uvect = &uctx->vector; + struct thread_ctx *thr = threads + thread_get_id(); + uint32_t exceptions = thread_mask_exceptions(THREAD_EXCP_FOREIGN_INTR); + + if (uvect == thr->vector_state.uvect) { + if (thr->vector_state.owner == RISCV_VECTOR_OWNER_USER) { + thr->vector_state.owner = RISCV_VECTOR_OWNER_NONE; + riscv_vector_disable(); + } + thr->vector_state.uvect = NULL; + } + + uvect->valid = false; + + thread_set_exceptions(exceptions); + + free(uvect->state); + uvect->state = NULL; +} +#endif /*CFG_RISCV_WITH_VECTOR*/ diff --git a/core/arch/riscv/plat-virt/conf.mk b/core/arch/riscv/plat-virt/conf.mk index 0cc1b7ea451..0288d1c3dff 100644 --- a/core/arch/riscv/plat-virt/conf.mk +++ b/core/arch/riscv/plat-virt/conf.mk @@ -3,6 +3,7 @@ $(call force,CFG_RV64_core,y) # ISA extension flags $(call force,CFG_RISCV_ISA_C,y) $(call force,CFG_RISCV_FPU,y) +CFG_RISCV_VEC ?= y $(call force,CFG_CORE_LARGE_PHYS_ADDR,y) $(call force,CFG_CORE_RESERVED_SHM,n) diff --git a/core/arch/riscv/riscv.mk b/core/arch/riscv/riscv.mk index 47af503c067..14e653f949f 100644 --- a/core/arch/riscv/riscv.mk +++ b/core/arch/riscv/riscv.mk @@ -99,10 +99,26 @@ $(call force,CFG_PAGED_USER_TA,n) $(call force,CFG_WITH_PAGER,n) $(call force,CFG_GIC,n) $(call force,CFG_ARM_GICV3,n) -$(call force,CFG_WITH_VFP,n) $(call force,CFG_WITH_STMM_SP,n) $(call force,CFG_TA_BTI,n) +# CFG_WITH_VFP asks OP-TEE to context switch the extended register state. +# Which register files that covers follows the extensions the platform says +# the hart has, so the vector registers are switched when CFG_RISCV_VEC=y +# and are left alone otherwise. +# +# Floating-point context switching is a separate series. The generic +# CFG_WITH_VFP machinery in core stays off here because it would pull in +# floating-point entry points this series does not implement, so the flag is +# consumed for the vector decision and then cleared. +CFG_RISCV_VEC ?= n + +CFG_RISCV_WITH_VECTOR := n +ifeq ($(CFG_WITH_VFP)-$(CFG_RISCV_VEC),y-y) +CFG_RISCV_WITH_VECTOR := y +endif +override CFG_WITH_VFP := n + # Enable generic timer $(call force,CFG_CORE_HAS_GENERIC_TIMER,y) @@ -124,6 +140,9 @@ ifeq ($(CFG_RISCV_FPU),y) ISA_D = fd ABI_D = d endif +ifeq ($(CFG_RISCV_WITH_VECTOR),y) +TA_ISA_V = v +endif ifeq ($(CFG_RISCV_ISA_C),y) ISA_C = c endif @@ -134,6 +153,13 @@ endif riscv-isa = $(ISA_BASE)$(ISA_D)$(ISA_C)$(ISA_ZBB)_zicsr_zifencei riscv-abi = $(ABI_BASE)$(ABI_D) +# The core is deliberately not built for the vector ISA even when vector +# context switching is on. riscv_vector.S turns it on for the two save and +# restore routines with .option arch, and leaving it off everywhere else +# means the compiler cannot put vector instructions into core code, which +# would otherwise trap against the VS == Off that core runs with. +riscv-ta-isa = $(ISA_BASE)$(ISA_D)$(ISA_C)$(TA_ISA_V)$(ISA_ZBB)_zicsr_zifencei + rv64-platform-cflags += -mcmodel=$(riscv-platform-mcmodel) rv64-platform-cflags += -march=$(riscv-isa) -mabi=$(riscv-abi) rv64-platform-cflags += -Wno-missing-include-dirs @@ -260,9 +286,14 @@ ta_rv64-platform-cflags += $(rv64-platform-cflags-hard-float) else ta_rv64-platform-cflags += $(rv64-platform-cflags-no-hard-float) endif +# TAs are built for the vector ISA when the context is being switched, even +# though the core is not, so this has to come after the core flags were +# inherited above. +ta_rv64-platform-cflags += -march=$(riscv-ta-isa) ta_rv64-platform-aflags += $(platform-aflags-generic) ta_rv64-platform-aflags += $(platform-aflags-debug-info) ta_rv64-platform-aflags += $(rv64-platform-aflags) +ta_rv64-platform-aflags += -march=$(riscv-ta-isa) ta_rv64-platform-cxxflags += -fpic ta_rv64-platform-cxxflags += $(platform-cflags-optimization) diff --git a/core/include/kernel/thread_private.h b/core/include/kernel/thread_private.h index 1c5a50582f5..d501dc8086e 100644 --- a/core/include/kernel/thread_private.h +++ b/core/include/kernel/thread_private.h @@ -47,6 +47,9 @@ struct thread_ctx { #endif #ifdef CFG_WITH_VFP struct thread_vfp_state vfp_state; +#endif +#ifdef CFG_RISCV_WITH_VECTOR + struct thread_vector_state vector_state; #endif void *rpc_arg; struct mobj *rpc_mobj; diff --git a/core/include/kernel/user_mode_ctx_struct.h b/core/include/kernel/user_mode_ctx_struct.h index c8812470ddb..2b6e014a261 100644 --- a/core/include/kernel/user_mode_ctx_struct.h +++ b/core/include/kernel/user_mode_ctx_struct.h @@ -38,6 +38,9 @@ struct user_mode_ctx { #if defined(CFG_WITH_VFP) struct thread_user_vfp_state vfp; #endif +#if defined(CFG_RISCV_WITH_VECTOR) + struct thread_user_vector_state vector; +#endif #if defined(CFG_TA_PAUTH) struct thread_pauth_keys keys; #endif diff --git a/core/kernel/ldelf_loader.c b/core/kernel/ldelf_loader.c index 40091729d0e..6baaa291cbe 100644 --- a/core/kernel/ldelf_loader.c +++ b/core/kernel/ldelf_loader.c @@ -141,6 +141,7 @@ TEE_Result ldelf_init_with_ldelf(struct ts_session *sess, sess->handle_scall = sess->ctx->ops->handle_scall; thread_user_clear_vfp(uctx); + thread_user_clear_vector(uctx); ldelf_sess_cleanup(sess); if (panicked) { @@ -303,6 +304,7 @@ TEE_Result ldelf_dump_state(struct user_mode_ctx *uctx) sess->handle_scall = sess->ctx->ops->handle_scall; thread_user_clear_vfp(uctx); + thread_user_clear_vector(uctx); ldelf_sess_cleanup(sess); if (panicked) { @@ -371,6 +373,7 @@ TEE_Result ldelf_dump_ftrace(struct user_mode_ctx *uctx, sess->fbuf = saved_fbuf; sess->handle_scall = sess->ctx->ops->handle_scall; thread_user_clear_vfp(uctx); + thread_user_clear_vector(uctx); ldelf_sess_cleanup(sess); if (panicked) { diff --git a/core/kernel/user_ta.c b/core/kernel/user_ta.c index e394a485abf..91906e67af5 100644 --- a/core/kernel/user_ta.c +++ b/core/kernel/user_ta.c @@ -190,6 +190,7 @@ static TEE_Result user_ta_enter(struct ts_session *session, &utc->ta_ctx.panic_code); thread_user_clear_vfp(&utc->uctx); + thread_user_clear_vector(&utc->uctx); if (utc->ta_ctx.panicked) { abort_print_current_ts();