arm64_addrenv: Add support for 4 level MMU translations

arch/arm64/include/arch.h

@@ -45,9 +45,9 @@
 #  error Only pages sizes of 4096 are currently supported (CONFIG_ARCH_ADDRENV)
 #endif
 
-/* All implementations have 3 levels of page tables */
+/* All implementations have 4 levels of page tables */
 
-#define ARCH_PGT_MAX_LEVELS (3)
+#define ARCH_PGT_MAX_LEVELS (4)
 #define ARCH_SPGTS          (ARCH_PGT_MAX_LEVELS - 1)
 
 #endif /* CONFIG_ARCH_ADDRENV */

arch/arm64/src/common/arm64_addrenv.c

@@ -115,24 +115,28 @@ static void map_spgtables(arch_addrenv_t *addrenv, uintptr_t vaddr)
 {
   int       i;
   uintptr_t prev;
+  uintptr_t l0;
 
-  /* Start from L1, and connect until max level - 1 */
+  /* Get the base page table level and the page table associated with it */
 
-  prev = arm64_pgvaddr(addrenv->spgtables[0]);
+  l0   = mmu_get_base_pgt_level();
+  prev = arm64_pgvaddr(addrenv->spgtables[l0]);
 
-  /* Check if the mapping already exists */
+  /* Start from the base level, and connect until max level - 1 */
 
-  if (mmu_ln_getentry(1, prev, vaddr) != 0)
+  for (i = l0; i < (ARCH_SPGTS - 1); i++)
     {
-      return;
-    }
+      uintptr_t next = addrenv->spgtables[i + 1];
 
-  /* No mapping yet, create it */
+      /* Check if the mapping already exists */
+
+      if (mmu_ln_getentry(i, prev, vaddr) == 0)
+        {
+          /* No mapping yet, create it */
+
+          mmu_ln_setentry(i, prev, next, vaddr, MMU_UPGT_FLAGS);
+        }
 
-  for (i = 0; i < (ARCH_SPGTS - 1); i++)
-    {
-      uintptr_t next = addrenv->spgtables[i + 1];
-      mmu_ln_setentry(i + 1, prev, next, vaddr, MMU_UPGT_FLAGS);
       prev = arm64_pgvaddr(next);
     }
 }
@@ -157,7 +161,7 @@ static int create_spgtables(arch_addrenv_t *addrenv)
   int       i;
   uintptr_t paddr;
 
-  for (i = 0; i < ARCH_SPGTS; i++)
+  for (i = mmu_get_base_pgt_level(); i < ARCH_SPGTS; i++)
     {
       paddr = mm_pgalloc(1);
       if (!paddr)
@@ -197,20 +201,37 @@ static int create_spgtables(arch_addrenv_t *addrenv)
 
 static int copy_kernel_mappings(arch_addrenv_t *addrenv)
 {
-  uintptr_t user_mappings = arm64_pgvaddr(addrenv->spgtables[0]);
+  uintptr_t kpgt;
+  uintptr_t upgt;
+  uintptr_t l0;
 
-  /* Copy the L1 references */
+  /* Determine the base page table level */
 
-  if (user_mappings == 0)
+  l0   = mmu_get_base_pgt_level();
+  kpgt = g_kernel_mappings;
+
+  /* Don't copy L0 references, as those encompass 512GB each */
+
+  if (l0 == 0)
+    {
+      upgt = arm64_pgvaddr(addrenv->spgtables[1]);
+      kpgt = arm64_pgvaddr(mmu_pte_to_paddr(((uintptr_t *)kpgt)[0]));
+    }
+  else
+    {
+      upgt = arm64_pgvaddr(addrenv->spgtables[l0]);
+    }
+
+  if (upgt == 0 || kpgt == 0)
     {
       return -EINVAL;
     }
 
-  memcpy((void *)user_mappings, (void *)g_kernel_mappings, MMU_PAGE_SIZE);
+  memcpy((void *)upgt, (void *)kpgt, MMU_PAGE_SIZE);
 
   /* Update with memory by flushing the cache */
 
-  up_flush_dcache(user_mappings, user_mappings + MMU_PAGE_SIZE);
+  up_flush_dcache(upgt, upgt + MMU_PAGE_SIZE);
 
   return OK;
 }
@@ -248,8 +269,8 @@ static int create_region(arch_addrenv_t *addrenv, uintptr_t vaddr,
 
   nmapped   = 0;
   npages    = MM_NPAGES(size);
-  ptprev    = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]);
-  ptlevel   = ARCH_SPGTS;
+  ptlevel   = MMU_PGT_LEVEL_MAX - 1;
+  ptprev    = arm64_pgvaddr(addrenv->spgtables[ptlevel]);
 
   /* Create mappings for the lower level tables */
 
@@ -384,6 +405,7 @@ int up_addrenv_create(size_t textsize, size_t datasize, size_t heapsize,
   uintptr_t textbase;
   uintptr_t database;
   uintptr_t heapbase;
+  uintptr_t l0;
 
   DEBUGASSERT(addrenv);
   DEBUGASSERT(MM_ISALIGNED(ARCH_ADDRENV_VBASE));
@@ -485,7 +507,8 @@ int up_addrenv_create(size_t textsize, size_t datasize, size_t heapsize,
 
   /* Provide the ttbr0 value for context switch */
 
-  addrenv->ttbr0 = mmu_ttbr_reg(addrenv->spgtables[0], 0);
+  l0 = mmu_get_base_pgt_level();
+  addrenv->ttbr0 = mmu_ttbr_reg(addrenv->spgtables[l0], 0);
 
   /* Synchronize data and instruction pipelines */
 
@@ -532,17 +555,14 @@ int up_addrenv_destroy(arch_addrenv_t *addrenv)
   /* Things start from the beginning of the user virtual memory */
 
   vaddr  = ARCH_ADDRENV_VBASE;
-  pgsize = mmu_get_region_size(ARCH_SPGTS);
+  pgsize = mmu_get_region_size(MMU_PGT_LEVEL_MAX - 1);
 
   /* First destroy the allocated memory and the final level page table */
 
   ptprev = (uintptr_t *)arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]);
   if (ptprev)
     {
-      /* walk user space only */
-
-      i = (ARCH_SPGTS < 2) ? vaddr / pgsize : 0;
-      for (; i < ENTRIES_PER_PGT; i++, vaddr += pgsize)
+      for (i = 0; i < ENTRIES_PER_PGT; i++, vaddr += pgsize)
         {
           ptlast = (uintptr_t *)arm64_pgvaddr(mmu_pte_to_paddr(ptprev[i]));
           if (ptlast)
@@ -570,7 +590,7 @@ int up_addrenv_destroy(arch_addrenv_t *addrenv)
 
   /* Then destroy the static tables */
 
-  for (i = 0; i < ARCH_SPGTS; i++)
+  for (i = mmu_get_base_pgt_level(); i < ARCH_SPGTS; i++)
     {
       paddr = addrenv->spgtables[i];
       if (paddr)

arch/arm64/src/common/arm64_addrenv_perms.c

@@ -70,8 +70,8 @@ static int modify_region(uintptr_t vstart, uintptr_t vend, uintptr_t setmask)
 
   for (vaddr = vstart; vaddr < vend; vaddr += MM_PGSIZE)
     {
-      for (ptlevel = 1, lnvaddr = l1vaddr;
-           ptlevel < MMU_PGT_LEVELS;
+      for (ptlevel = mmu_get_base_pgt_level(), lnvaddr = l1vaddr;
+           ptlevel < MMU_PGT_LEVEL_MAX;
            ptlevel++)
         {
           paddr = mmu_pte_to_paddr(mmu_ln_getentry(ptlevel, lnvaddr, vaddr));

arch/arm64/src/common/arm64_addrenv_pgmap.c

@@ -90,7 +90,9 @@ uintptr_t up_addrenv_find_page(arch_addrenv_t *addrenv, uintptr_t vaddr)
 
   /* Make table walk to find the page */
 
-  for (ptlevel = 1, lnvaddr = pgdir; ptlevel < MMU_PGT_LEVELS; ptlevel++)
+  for (ptlevel = mmu_get_base_pgt_level(), lnvaddr = pgdir;
+       ptlevel < MMU_PGT_LEVEL_MAX;
+       ptlevel++)
     {
       paddr = mmu_pte_to_paddr(mmu_ln_getentry(ptlevel, lnvaddr, vaddr));
       lnvaddr = arm64_pgvaddr(paddr);
@@ -189,26 +191,28 @@ int up_addrenv_kmap_init(void)
   struct arch_addrenv_s *addrenv;
   uintptr_t              next;
   uintptr_t              vaddr;
+  uintptr_t              l0;
   int                    i;
 
   /* Populate the static page tables one by one */
 
   addrenv = &g_kernel_addrenv;
   next    =  g_kernel_pgt_pbase;
   vaddr   =  CONFIG_ARCH_KMAP_VBASE;
+  l0      =  mmu_get_base_pgt_level();
 
-  for (i = 0; i < ARCH_SPGTS; i++)
+  for (i = l0; i < ARCH_SPGTS; i++)
     {
       /* Connect the static page tables */
 
       uintptr_t lnvaddr = arm64_pgvaddr(next);
       addrenv->spgtables[i] = next;
-      next = mmu_pte_to_paddr(mmu_ln_getentry(i + 1, lnvaddr, vaddr));
+      next = mmu_pte_to_paddr(mmu_ln_getentry(i, lnvaddr, vaddr));
     }
 
   /* Set the page directory root */
 
-  addrenv->ttbr0 = mmu_ttbr_reg(g_kernel_pgt_pbase, 0);
+  addrenv->ttbr0 = mmu_ttbr_reg(addrenv->spgtables[l0], 0);
 
   /* When all is set and done, flush the data caches */
 

arch/arm64/src/common/arm64_addrenv_utils.c

@@ -66,8 +66,8 @@ uintptr_t arm64_get_pgtable(arch_addrenv_t *addrenv, uintptr_t vaddr)
 
   /* Get the current level MAX_LEVELS-1 entry corresponding to this vaddr */
 
-  ptlevel = ARCH_SPGTS;
-  ptprev  = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]);
+  ptlevel = MMU_PGT_LEVEL_MAX - 1;
+  ptprev  = arm64_pgvaddr(addrenv->spgtables[ptlevel]);
   if (!ptprev)
     {
       /* Something is very wrong */
@@ -134,7 +134,7 @@ int arm64_map_pages(arch_addrenv_t *addrenv, uintptr_t *pages,
   uintptr_t ptlevel;
   uintptr_t paddr;
 
-  ptlevel = MMU_PGT_LEVELS;
+  ptlevel = MMU_PGT_LEVEL_MAX;
 
   /* Add the references to pages[] into the caller's address environment */
 
@@ -184,16 +184,17 @@ int arm64_unmap_pages(arch_addrenv_t *addrenv, uintptr_t vaddr,
   uintptr_t ptlevel;
   uintptr_t paddr;
 
-  ptprev  = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]);
+  /* Get the current level MAX_LEVELS-1 entry corresponding to this vaddr */
+
+  ptlevel = MMU_PGT_LEVEL_MAX - 1;
+  ptprev  = arm64_pgvaddr(addrenv->spgtables[ptlevel]);
   if (!ptprev)
     {
       /* Something is very wrong */
 
       return -EFAULT;
     }
 
-  ptlevel = ARCH_SPGTS;
-
   /* Remove the references from the caller's address environment */
 
   for (; npages > 0; npages--)

arch/arm64/src/common/arm64_mmu.c

@@ -82,7 +82,7 @@
 #define XLAT_TABLE_SIZE                 (1U << XLAT_TABLE_SIZE_SHIFT)
 
 #define XLAT_TABLE_ENTRY_SIZE_SHIFT     3U /* Each table entry is 8 bytes */
-#define XLAT_TABLE_LEVEL_MAX            MMU_PGT_LEVELS
+#define XLAT_TABLE_LEVEL_MAX            MMU_PGT_LEVEL_MAX
 
 #define XLAT_TABLE_ENTRIES_SHIFT \
   (XLAT_TABLE_SIZE_SHIFT - XLAT_TABLE_ENTRY_SIZE_SHIFT)
@@ -207,6 +207,7 @@ static const struct arm_mmu_config g_mmu_nxrt_config =
 
 static const size_t g_pgt_sizes[] =
 {
+  MMU_L0_PAGE_SIZE,
   MMU_L1_PAGE_SIZE,
   MMU_L2_PAGE_SIZE,
   MMU_L3_PAGE_SIZE
@@ -709,7 +710,8 @@ void mmu_ln_setentry(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t paddr,
   uintptr_t *lntable = (uintptr_t *)lnvaddr;
   uint32_t   index;
 
-  DEBUGASSERT(ptlevel > 0 && ptlevel <= XLAT_TABLE_LEVEL_MAX);
+  DEBUGASSERT(ptlevel >= XLAT_TABLE_BASE_LEVEL &&
+              ptlevel <= XLAT_TABLE_LEVEL_MAX);
 
   /* Calculate index for lntable */
 
@@ -735,7 +737,8 @@ uintptr_t mmu_ln_getentry(uint32_t ptlevel, uintptr_t lnvaddr,
   uintptr_t *lntable = (uintptr_t *)lnvaddr;
   uint32_t  index;
 
-  DEBUGASSERT(ptlevel > 0 && ptlevel <= XLAT_TABLE_LEVEL_MAX);
+  DEBUGASSERT(ptlevel >= XLAT_TABLE_BASE_LEVEL &&
+              ptlevel <= XLAT_TABLE_LEVEL_MAX);
 
   index = XLAT_TABLE_VA_IDX(vaddr, ptlevel);
 
@@ -753,7 +756,8 @@ void mmu_ln_restore(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t vaddr,
   uintptr_t *lntable = (uintptr_t *)lnvaddr;
   uint32_t  index;
 
-  DEBUGASSERT(ptlevel > 0 && ptlevel <= XLAT_TABLE_LEVEL_MAX);
+  DEBUGASSERT(ptlevel >= XLAT_TABLE_BASE_LEVEL &&
+              ptlevel <= XLAT_TABLE_LEVEL_MAX);
 
   index = XLAT_TABLE_VA_IDX(vaddr, ptlevel);
 
@@ -771,7 +775,13 @@ void mmu_ln_restore(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t vaddr,
 
 size_t mmu_get_region_size(uint32_t ptlevel)
 {
-  DEBUGASSERT(ptlevel > 0 && ptlevel <= XLAT_TABLE_LEVEL_MAX);
+  DEBUGASSERT(ptlevel >= XLAT_TABLE_BASE_LEVEL &&
+              ptlevel <= XLAT_TABLE_LEVEL_MAX);
 
-  return g_pgt_sizes[ptlevel - 1];
+  return g_pgt_sizes[ptlevel];
+}
+
+uintptr_t mmu_get_base_pgt_level(void)
+{
+  return XLAT_TABLE_BASE_LEVEL;
 }

arch/arm64/src/common/arm64_mmu.h

@@ -233,13 +233,15 @@
 
 /* Amount of page table levels */
 
-#define MMU_PGT_LEVELS              (3U)
+#define MMU_PGT_LEVELS              (4U)
+#define MMU_PGT_LEVEL_MAX           (3U) /* Levels go from 0-3 */
 
 /* Page sizes per page table level */
 
-#define MMU_L1_PAGE_SIZE            (0x40000000) /* 1G */
-#define MMU_L2_PAGE_SIZE            (0x200000)   /* 2M */
-#define MMU_L3_PAGE_SIZE            (0x1000)     /* 4K */
+#define MMU_L0_PAGE_SIZE            (0x8000000000) /* 512G */
+#define MMU_L1_PAGE_SIZE            (0x40000000)   /* 1G */
+#define MMU_L2_PAGE_SIZE            (0x200000)     /* 2M */
+#define MMU_L3_PAGE_SIZE            (0x1000)       /* 4K */
 
 /* Flags for user page tables */
 
@@ -620,6 +622,28 @@ void mmu_ln_restore(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t vaddr,
 
 size_t mmu_get_region_size(uint32_t ptlevel);
 
+/****************************************************************************
+ * Name: mmu_get_base_pgt_level
+ *
+ * Description:
+ *   Get the base translation table level. The ARM64 MMU implementation
+ *   optimizes the amount of translation table levels in use, based on the
+ *   configured virtual address range (CONFIG_ARM64_VA_BITS).
+ *
+ *   Table indices range from 0...3 and the lowest table indices are dropped
+ *   as needed. If CONFIG_ARM64_VA_BITS >= 40, all 4 translation table levels
+ *   are needed.
+ *
+ * Input Parameters:
+ *   None.
+ *
+ * Returned Value:
+ *   The base translation table level.
+ *
+ ****************************************************************************/
+
+uintptr_t mmu_get_base_pgt_level(void);
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* __ARCH_ARM64_SRC_COMMON_ARM64_MMU_H */

arch/arm64/src/common/arm64_pgalloc.c

@@ -93,6 +93,7 @@ uintptr_t pgalloc(uintptr_t brkaddr, unsigned int npages)
   struct tcb_s          *tcb = this_task();
   struct arch_addrenv_s *addrenv;
   uintptr_t              ptlast;
+  uintptr_t              ptlevel;
   uintptr_t              paddr;
   uintptr_t              vaddr;
 
@@ -114,7 +115,8 @@ uintptr_t pgalloc(uintptr_t brkaddr, unsigned int npages)
 
   /* Start mapping from the old heap break address */
 
-  vaddr = brkaddr;
+  vaddr   = brkaddr;
+  ptlevel = MMU_PGT_LEVEL_MAX;
 
   /* Sanity checks */
 
@@ -145,7 +147,7 @@ uintptr_t pgalloc(uintptr_t brkaddr, unsigned int npages)
 
       /* Then add the reference */
 
-      mmu_ln_setentry(MMU_PGT_LEVELS, ptlast, paddr, vaddr, MMU_UDATA_FLAGS);
+      mmu_ln_setentry(ptlevel, ptlast, paddr, vaddr, MMU_UDATA_FLAGS);
       vaddr += MM_PGSIZE;
     }