console: share the Uart by reference and add iprint

aarch64/src/devcons.rs

@@ -3,7 +3,7 @@
 use crate::uartmini::MiniUart;
 use core::cell::SyncUnsafeCell;
 use core::mem::MaybeUninit;
-use port::devcons::Console;
+use port::devcons::{Console, IprintOps, Uart};
 use port::fdt::DeviceTree;
 #[cfg(not(test))]
 use port::println;
@@ -25,6 +25,19 @@ use port::println;
 // - UART2 PL011
 // - UART3 PL011
 
+static UART: SyncUnsafeCell<MaybeUninit<MiniUart>> = SyncUnsafeCell::new(MaybeUninit::uninit());
+
+static IPRINT_OPS: IprintOps = IprintOps { putb: iputb };
+
+/// Direct polled write for iprint, bypassing the console lock.
+/// `MiniUart::putb` needs only a shared reference, so this can safely
+/// alias the reference held by the console.
+fn iputb(b: u8) {
+    // Safety: IPRINT_OPS is only registered once UART is initialised.
+    let uart = unsafe { (*UART.get()).assume_init_ref() };
+    uart.putb(b);
+}
+
 pub fn init(dt: &DeviceTree) {
     Console::set_uart(|| {
         let uart = MiniUart::new_with_map_ranges(dt);
@@ -35,12 +48,11 @@ pub fn init(dt: &DeviceTree) {
             Ok(uart) => {
                 uart.init();
 
-                static UART: SyncUnsafeCell<MaybeUninit<MiniUart>> =
-                    SyncUnsafeCell::new(MaybeUninit::uninit());
                 unsafe {
                     let cons = &mut *UART.get();
                     cons.write(uart);
-                    Ok(cons.assume_init_mut())
+                    port::devcons::set_iprint_ops(&IPRINT_OPS);
+                    Ok(cons.assume_init_ref())
                 }
             }
             Err(msg) => {

aarch64/src/irq.rs

@@ -0,0 +1,36 @@
+//! DAIF-based implementation of the portable interrupt masking hooks.
+
+use port::irq::IrqOps;
+
+static IRQ_OPS: IrqOps = IrqOps { mask: mask_irqs, restore: restore_irqs };
+
+/// Register DAIF masking with `port::irq`.  Must be called before
+/// interrupts are enabled.
+pub fn init() {
+    port::irq::set_ops(&IRQ_OPS);
+}
+
+/// Mask IRQs on this core, returning the previous DAIF state.
+fn mask_irqs() -> u64 {
+    let daif: u64;
+    unsafe {
+        core::arch::asm!(
+            "mrs {daif}, daif",
+            "msr daifset, #2",
+            daif = out(reg) daif,
+            options(nostack, preserves_flags)
+        );
+    }
+    daif
+}
+
+/// Restore a DAIF state previously returned by `mask_irqs`.
+fn restore_irqs(daif: u64) {
+    unsafe {
+        core::arch::asm!(
+            "msr daif, {daif}",
+            daif = in(reg) daif,
+            options(nostack, preserves_flags)
+        );
+    }
+}

aarch64/src/mailbox.rs

@@ -38,21 +38,23 @@ pub fn init(dt: &DeviceTree) {
 /// https://developer.arm.com/documentation/ddi0306/b/CHDGHAIG
 /// https://github.com/raspberrypi/firmware/wiki/Mailbox-property-interface
 struct Mailbox {
-    pub mbox_virtrange: VirtRange,
-    req_buffer_va: VirtRange,
-    req_buffer_pa: PhysRange,
+    mbox_virtrange: VirtRange,
+    req_buf_virtrange: VirtRange,
+    req_buf_physrange: PhysRange,
 }
 
 impl Mailbox {
     fn new(dt: &DeviceTree) -> Result<Self> {
         // Allocate a page of device memory for the mailbox request/response buffer
         // TODO Split this into multiple buffers to allow parallel requests.
-        let (req_buffer_va, req_buffer_pa) =
+        let (req_buf_virtrange, req_buf_physrange) =
             deviceutil::alloc_device_page("mailboxbuf", vm::PageSize::Page4K)?;
 
         let mbox_physrange = Self::find_mbox_physrange(dt)?;
         let mbox = match map_device_register("mailbox", mbox_physrange, vm::PageSize::Page4K) {
-            Ok(mbox_virtrange) => Ok(Mailbox { mbox_virtrange, req_buffer_va, req_buffer_pa }),
+            Ok(mbox_virtrange) => {
+                Ok(Mailbox { mbox_virtrange, req_buf_virtrange, req_buf_physrange })
+            }
             Err(msg) => {
                 println!("can't map mailbox {:?}", msg);
                 Err("can't create mailbox")
@@ -78,7 +80,7 @@ impl Mailbox {
 
         // Write the request address combined with the channel to the write register
         let channel = ChannelId::ArmToVc as u32;
-        let uart_mbox_u32 = self.req_buffer_pa.start.addr() as u32;
+        let uart_mbox_u32 = self.req_buf_physrange.start.addr() as u32;
         let r = (uart_mbox_u32 & !0xF) | channel;
         write_reg(&self.mbox_virtrange, MBOX_WRITE, r);
 
@@ -146,7 +148,7 @@ where
         .as_mut()
         .map(|mb| {
             let msg = unsafe {
-                let page_va_ptr = mb.req_buffer_va.start as u64 as *mut MessageWithTags<T, U>;
+                let page_va_ptr = mb.req_buf_virtrange.start as u64 as *mut MessageWithTags<T, U>;
                 core::intrinsics::volatile_set_memory(page_va_ptr, 0, 1);
                 let msg = NonNull::new_unchecked(page_va_ptr).as_mut();
                 msg.request.size = size;

aarch64/src/main.rs

@@ -12,11 +12,13 @@ mod allocator;
 mod devcons;
 mod deviceutil;
 mod io;
+mod irq;
 mod kmem;
 mod mailbox;
 mod pagealloc;
 mod param;
 mod pre_mmu;
+mod reg;
 mod registers;
 mod swtch;
 mod trap;
@@ -33,6 +35,7 @@ use param::KZERO;
 use port::fdt::DeviceTree;
 use port::mem::{PhysAddr, PhysRange, VirtRange};
 use port::println;
+use reg::midr_el1::MidrEl1;
 use vm::{Entry, RootPageTableType, VaMapping};
 
 use crate::kmem::{
@@ -110,6 +113,7 @@ fn print_stacks() {
 /// assumed to be dtb_va-KZERO.
 #[unsafe(no_mangle)]
 pub extern "C" fn main9(dtb_va: usize) {
+    irq::init();
     trap::init();
 
     // Parse the DTB before we set up memory so we can correctly map it
@@ -134,7 +138,7 @@ pub extern "C" fn main9(dtb_va: usize) {
     println!();
     println!("r9 from the Internet");
     println!("DTB found at: {:#x}", dtb_va);
-    println!("midr_el1: {:?}", registers::MidrEl1::read());
+    println!("midr_el1: {:?}", MidrEl1::read());
 
     print_stacks();
 

aarch64/src/reg/esr_el1.rs

@@ -0,0 +1,161 @@
+use core::fmt;
+
+use bitstruct::bitstruct;
+use num_enum::TryFromPrimitive;
+
+bitstruct! {
+    #[derive(Copy, Clone)]
+    pub struct EsrEl1(pub u64) {
+        pub iss: u32 = 0..25;
+        pub il: bool = 25;
+        pub ec: u8 = 26..32;
+        pub iss2: u8 = 32..37;
+    }
+}
+
+impl EsrEl1 {
+    /// Try to convert the error into an ExceptionClass enum, or return the original number
+    /// as the error.
+    pub fn exception_class_enum(&self) -> Result<ExceptionClass, u8> {
+        ExceptionClass::try_from(self.ec()).map_err(|e| e.number)
+    }
+}
+
+impl fmt::Debug for EsrEl1 {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("EsrEl1")
+            .field("iss", &format_args!("{:#010x}", self.iss()))
+            .field("il", &format_args!("{}", self.il()))
+            .field("ec", &format_args!("{:?}", self.exception_class_enum()))
+            .field("iss2", &format_args!("{:#04x}", self.iss2()))
+            .finish()
+    }
+}
+
+/// Exception class maps to ESR_EL1 EC bits[31:26]. We skip aarch32 exceptions.
+#[derive(Debug, Eq, PartialEq, TryFromPrimitive)]
+#[repr(u8)]
+pub enum ExceptionClass {
+    Unknown = 0,
+    WaitFor = 1,
+    FloatSimd = 7,
+    Ls64 = 10,
+    BranchTargetException = 13,
+    IllegalExecutionState = 14,
+    MsrMrsSystem = 24,
+    Sve = 25,
+    Tstart = 27,
+    PointerAuthFailure = 28,
+    Sme = 29,
+    GranuleProtectionCheck = 30,
+    InstructionAbortLowerEl = 32,
+    InstructionAbortSameEl = 33,
+    PcAlignmentFault = 34,
+    DataAbortLowerEl = 36,
+    DataAbortSameEl = 37,
+    SpAlignmentFault = 38,
+    MemoryOperationException = 39,
+    TrappedFloatingPointException = 44,
+    SError = 47,
+    BreakpointLowerEl = 48,
+    BreakpointSameEl = 49,
+    SoftwareStepLowerEl = 50,
+    SoftwareStepSameEl = 51,
+    WatchpointLowerEl = 52,
+    WatchpointSameEl = 53,
+    Brk = 60,
+}
+
+bitstruct! {
+    #[derive(Copy, Clone)]
+    pub struct EsrEl1IssInstructionAbort(pub u32) {
+        ifsc: u8 = 0..6;
+        s1ptw: bool = 7;
+        ea: bool = 9;
+        fnv: bool = 10;
+        set: u8 = 11..13;
+    }
+}
+
+#[allow(dead_code)]
+impl EsrEl1IssInstructionAbort {
+    pub fn from_esr_el1(r: EsrEl1) -> Option<EsrEl1IssInstructionAbort> {
+        r.exception_class_enum()
+            .ok()
+            .filter(|ec| *ec == ExceptionClass::InstructionAbortSameEl)
+            .map(|_| EsrEl1IssInstructionAbort(r.iss()))
+    }
+
+    pub fn instruction_fault(&self) -> Result<InstructionFaultStatusCode, u8> {
+        InstructionFaultStatusCode::try_from(self.ifsc()).map_err(|e| e.number)
+    }
+}
+
+#[derive(Debug, Eq, PartialEq, TryFromPrimitive)]
+#[repr(u8)]
+pub enum InstructionFaultStatusCode {
+    AddressSizeFaultLevel0 = 0,
+    AddressSizeFaultLevel1 = 1,
+    AddressSizeFaultLevel2 = 2,
+    AddressSizeFaultLevel3 = 3,
+    TranslationFaultLevel0 = 4,
+    TranslationFaultLevel1 = 5,
+    TranslationFaultLevel2 = 6,
+    TranslationFaultLevel3 = 7,
+    AccessFlagFaultLevel0 = 8,
+    AccessFlagFaultLevel1 = 9,
+    AccessFlagFaultLevel2 = 10,
+    AccessFlagFaultLevel3 = 11,
+    PermissionFaultLevel0 = 12,
+    PermissionFaultLevel1 = 13,
+    PermissionFaultLevel2 = 14,
+    PermissionFaultLevel3 = 15,
+    SyncExtAbortNotOnWalkOrUpdate = 16,
+    SyncExtAbortOnWalkOrUpdateLevelNeg1 = 19,
+    SyncExtAbortOnWalkOrUpdateLevel0 = 20,
+    SyncExtAbortOnWalkOrUpdateLevel1 = 21,
+    SyncExtAbortOnWalkOrUpdateLevel2 = 22,
+    SyncExtAbortOnWalkOrUpdateLevel3 = 23,
+    SyncParityOrEccErrOnMemAccessNotOnWalk = 24,
+    SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevelNeg1 = 27,
+    SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevel0 = 28,
+    SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevel1 = 29,
+    SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevel2 = 30,
+    SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevel3 = 31,
+    GranuleProtectFaultOnWalkOrUpdateLevelNeg1 = 35,
+    GranuleProtectFaultOnWalkOrUpdateLevel0 = 36,
+    GranuleProtectFaultOnWalkOrUpdateLevel1 = 37,
+    GranuleProtectFaultOnWalkOrUpdateLevel2 = 38,
+    GranuleProtectFaultOnWalkOrUpdateLevel3 = 39,
+    GranuleProtectFaultNotOnWalkOrUpdateLevel = 40,
+    AddressSizeFaultLevelNeg1 = 41,
+    TranslationFaultLevelNeg1 = 43,
+    TlbConflictAbort = 48,
+    UnsupportedAtomicHardwareUpdateFault = 49,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // This test is useful for making sense of early-stage exceptions.  Qemu
+    // will report an exception of the form below.  Copy the ESR value into
+    // this test to break it down.
+    //
+    // Exception return from AArch64 EL2 to AArch64 EL1 PC 0x8006c
+    // Taking exception 3 [Prefetch Abort] on CPU 0
+    // ...from EL1 to EL1
+    // ...with ESR 0x21/0x86000004
+    // ...with FAR 0x80090
+    // ...with ELR 0x80090
+    // ...to EL1 PC 0x200 PSTATE 0x3c5
+    #[test]
+    fn test_parse_esr_el1() {
+        let r = EsrEl1(0x86000004);
+        assert_eq!(r.exception_class_enum().unwrap(), ExceptionClass::InstructionAbortSameEl);
+        assert_eq!(
+            EsrEl1IssInstructionAbort::from_esr_el1(r).unwrap().instruction_fault().unwrap(),
+            InstructionFaultStatusCode::TranslationFaultLevel0
+        );
+    }
+}

aarch64/src/reg/midr_el1.rs

@@ -0,0 +1,52 @@
+use core::fmt;
+
+use aarch64_cpu::registers::{MIDR_EL1, Readable};
+use bitstruct::bitstruct;
+use num_enum::TryFromPrimitive;
+
+bitstruct! {
+    #[derive(Copy, Clone)]
+    pub struct MidrEl1(pub u64) {
+        revision: u8 = 0..4;
+        partnum: u16 = 4..16;
+        architecture: u8 = 16..20;
+        variant: u8 = 20..24;
+        implementer: u16 = 24..32;
+    }
+}
+
+impl MidrEl1 {
+    pub fn read() -> Self {
+        Self(if cfg!(test) { 0 } else { MIDR_EL1.extract().into() })
+    }
+
+    pub fn partnum_enum(&self) -> Result<PartNum, u16> {
+        PartNum::try_from(self.partnum()).map_err(|e| e.number)
+    }
+}
+
+impl fmt::Debug for MidrEl1 {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("MidrEl1")
+            .field("revision", &format_args!("{:#x}", self.revision()))
+            .field(
+                "partnum",
+                &format_args!("{:?}", self.partnum_enum().unwrap_or(PartNum::Unknown)),
+            )
+            .field("architecture", &format_args!("{:#x}", self.architecture()))
+            .field("variant", &format_args!("{:#x}", self.variant()))
+            .field("implementer", &format_args!("{:#x}", self.implementer()))
+            .finish()
+    }
+}
+
+/// Known IDs for midr_el1's partnum
+#[derive(Debug, Eq, PartialEq, TryFromPrimitive)]
+#[repr(u16)]
+pub enum PartNum {
+    Unknown = 0,
+    RaspberryPi1 = 0xb76,
+    RaspberryPi2 = 0xc07,
+    RaspberryPi3 = 0xd03,
+    RaspberryPi4 = 0xd08,
+}

aarch64/src/reg/mod.rs

@@ -0,0 +1,2 @@
+pub mod esr_el1;
+pub mod midr_el1;