[components/mm] support for scalable memory management (#7277)

* [mm/page] multi-list page manager
[mm/page] page debugger
[libcpu/aarch64] hugepage support

* [quality] remove void-arith

* [format] remove kasan codes

4 files changed, 164 insertions, 41 deletions

diff --git a/libcpu/aarch64/common/cache.h b/libcpu/aarch64/common/cache.h
index e94e4b65da..06a2805c48 100644
--- a/libcpu/aarch64/common/cache.h
+++ b/libcpu/aarch64/common/cache.h
@@ -23,7 +23,10 @@ void rt_hw_cpu_dcache_invalidate(void *start_addr, unsigned long size);
 
 static inline void rt_hw_icache_invalidate_all(void)
 {
-    __asm_invalidate_icache_all();
+    /* wait for any modification complete */
+    __asm__ volatile ("dsb ishst");
+    __asm__ volatile ("ic iallu");
+    __asm__ volatile ("isb");
 }
 
 void rt_hw_cpu_icache_invalidate(void *addr, rt_size_t size);
diff --git a/libcpu/aarch64/common/cpu.c b/libcpu/aarch64/common/cpu.c
index f5f96a5d90..8f7f2843a9 100644
--- a/libcpu/aarch64/common/cpu.c
+++ b/libcpu/aarch64/common/cpu.c
@@ -132,7 +132,10 @@ static rt_uint64_t _read_be_number(void *start, int size)
 {
     rt_uint64_t buf = 0;
     for (; size > 0; size--)
-        buf = (buf << 32) | fdt32_to_cpu(*(uint32_t *)start++);
+    {
+        buf = (buf << 32) | fdt32_to_cpu(*(uint32_t *)start);
+        start = (uint32_t *)start + 1;
+    }
     return buf;
 }
 
diff --git a/libcpu/aarch64/common/mmu.c b/libcpu/aarch64/common/mmu.c
index 1e94fdca97..a08f3fca53 100644
--- a/libcpu/aarch64/common/mmu.c
+++ b/libcpu/aarch64/common/mmu.c
@@ -1,13 +1,14 @@
 /*
- * Copyright (c) 2006-2018, RT-Thread Development Team
+ * Copyright (c) 2006-2023, RT-Thread Development Team
  *
  * SPDX-License-Identifier: Apache-2.0
  *
  * Change Logs:
  * Date           Author       Notes
  * 2012-01-10     bernard      porting to AM1808
+ * 2021-11-28     GuEe-GUI     first version
+ * 2022-12-10     WangXiaoyao  porting to MM
  */
-
 #include <board.h>
 #include <rthw.h>
 #include <rtthread.h>
@@ -79,6 +80,7 @@ static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr)
         {
             break;
         }
+        /* next table entry in current level */
         level_info[level].pos = cur_lv_tbl + off;
         cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
         cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
@@ -119,8 +121,7 @@ static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr)
     return;
 }
 
-static int _kenrel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr,
-                          unsigned long attr)
+static int _kernel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr)
 {
     int ret = 0;
     int level;
@@ -145,7 +146,7 @@ static int _kenrel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr,
         off &= MMU_LEVEL_MASK;
         if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
         {
-            page = (unsigned long)rt_pages_alloc(0);
+            page = (unsigned long)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
             if (!page)
             {
                 ret = MMU_MAP_ERROR_NOPAGE;
@@ -188,19 +189,104 @@ err:
     return ret;
 }
 
+static int _kernel_map_2M(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr)
+{
+    int ret = 0;
+    int level;
+    unsigned long *cur_lv_tbl = lv0_tbl;
+    unsigned long page;
+    unsigned long off;
+    unsigned long va = (unsigned long)vaddr;
+    unsigned long pa = (unsigned long)paddr;
+
+    int level_shift = MMU_ADDRESS_BITS;
+
+    if (va & ARCH_SECTION_MASK)
+    {
+        return MMU_MAP_ERROR_VANOTALIGN;
+    }
+    if (pa & ARCH_SECTION_MASK)
+    {
+        return MMU_MAP_ERROR_PANOTALIGN;
+    }
+    for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++)
+    {
+        off = (va >> level_shift);
+        off &= MMU_LEVEL_MASK;
+        if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
+        {
+            page = (unsigned long)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
+            if (!page)
+            {
+                ret = MMU_MAP_ERROR_NOPAGE;
+                goto err;
+            }
+            rt_memset((char *)page, 0, ARCH_PAGE_SIZE);
+            rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE);
+            cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE;
+            rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
+        }
+        else
+        {
+            page = cur_lv_tbl[off];
+            page &= MMU_ADDRESS_MASK;
+            /* page to va */
+            page -= PV_OFFSET;
+            rt_page_ref_inc((void *)page, 0);
+        }
+        page = cur_lv_tbl[off];
+        if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
+        {
+            /* is block! error! */
+            ret = MMU_MAP_ERROR_CONFLICT;
+            goto err;
+        }
+        cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
+        cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
+        level_shift -= MMU_LEVEL_SHIFT;
+    }
+    /* now is level page */
+    attr &= MMU_ATTRIB_MASK;
+    pa |= (attr | MMU_TYPE_BLOCK); /* block */
+    off = (va >> ARCH_SECTION_SHIFT);
+    off &= MMU_LEVEL_MASK;
+    cur_lv_tbl[off] = pa;
+    rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
+    return ret;
+err:
+    _kenrel_unmap_4K(lv0_tbl, (void *)va);
+    return ret;
+}
+
 void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
                     size_t attr)
 {
     int ret = -1;
 
     void *unmap_va = v_addr;
-    size_t npages = size >> ARCH_PAGE_SHIFT;
+    size_t npages;
+    size_t stride;
+    int (*mapper)(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr);
+
+    if (((rt_ubase_t)v_addr & ARCH_SECTION_MASK) || (size & ARCH_SECTION_MASK))
+    {
+        /* legacy 4k mapping */
+        npages = size >> ARCH_PAGE_SHIFT;
+        stride = ARCH_PAGE_SIZE;
+        mapper = _kernel_map_4K;
+    }
+    else
+    {
+        /* 2m huge page */
+        npages = size >> ARCH_SECTION_SHIFT;
+        stride = ARCH_SECTION_SIZE;
+        mapper = _kernel_map_2M;
+    }
 
-    // TODO trying with HUGEPAGE here
     while (npages--)
     {
         MM_PGTBL_LOCK(aspace);
-        ret = _kenrel_map_4K(aspace->page_table, v_addr, p_addr, attr);
+        ret = mapper(aspace->page_table, v_addr, p_addr, attr);
         MM_PGTBL_UNLOCK(aspace);
 
         if (ret != 0)
@@ -213,12 +299,12 @@ void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
                 MM_PGTBL_LOCK(aspace);
                 _kenrel_unmap_4K(aspace->page_table, (void *)unmap_va);
                 MM_PGTBL_UNLOCK(aspace);
-                unmap_va += ARCH_PAGE_SIZE;
+                unmap_va = (char *)unmap_va + stride;
             }
             break;
         }
-        v_addr += ARCH_PAGE_SIZE;
-        p_addr += ARCH_PAGE_SIZE;
+        v_addr = (char *)v_addr + stride;
+        p_addr = (char *)p_addr + stride;
     }
 
     if (ret == 0)
@@ -244,7 +330,7 @@ void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size)
         MM_PGTBL_LOCK(aspace);
         _kenrel_unmap_4K(aspace->page_table, v_addr);
         MM_PGTBL_UNLOCK(aspace);
-        v_addr += ARCH_PAGE_SIZE;
+        v_addr = (char *)v_addr + ARCH_PAGE_SIZE;
     }
 }
 
@@ -254,7 +340,7 @@ void rt_hw_aspace_switch(rt_aspace_t aspace)
     {
         void *pgtbl = aspace->page_table;
         pgtbl = rt_kmem_v2p(pgtbl);
-        uintptr_t tcr;
+        rt_ubase_t tcr;
 
         __asm__ volatile("msr ttbr0_el1, %0" ::"r"(pgtbl) : "memory");
 
@@ -337,20 +423,19 @@ void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr)
     rt_page_cleanup();
 }
 
-
 #ifdef RT_USING_SMART
-static inline void _init_region(void *vaddr, size_t size)
+static void _init_region(void *vaddr, size_t size)
 {
     rt_ioremap_start = vaddr;
     rt_ioremap_size = size;
-    rt_mpr_start = rt_ioremap_start - rt_mpr_size;
+    rt_mpr_start = (char *)rt_ioremap_start - rt_mpr_size;
 }
 #else
 
-#define RTOS_VEND ((void *)0xfffffffff000UL)
+#define RTOS_VEND (0xfffffffff000UL)
 static inline void _init_region(void *vaddr, size_t size)
 {
-    rt_mpr_start = RTOS_VEND - rt_mpr_size;
+    rt_mpr_start = (void *)(RTOS_VEND - rt_mpr_size);
 }
 #endif
 
@@ -395,7 +480,7 @@ int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, size_t size,
     rt_aspace_init(aspace, (void *)KERNEL_VADDR_START, 0 - KERNEL_VADDR_START,
                    vtable);
 #else
-    rt_aspace_init(aspace, (void *)0x1000, RTOS_VEND - (void *)0x1000, vtable);
+    rt_aspace_init(aspace, (void *)0x1000, RTOS_VEND - 0x1000ul, vtable);
 #endif
 
     _init_region(v_address, size);
@@ -586,26 +671,35 @@ void *rt_hw_mmu_v2p(rt_aspace_t aspace, void *v_addr)
 {
     int level_shift;
     unsigned long paddr;
-    unsigned long *pte = _query(aspace, v_addr, &level_shift);
 
-    if (pte)
+    if (aspace == &rt_kernel_space)
     {
-        paddr = *pte & MMU_ADDRESS_MASK;
-        paddr |= (uintptr_t)v_addr & ((1ul << level_shift) - 1);
+        paddr = (unsigned long)rt_hw_mmu_kernel_v2p(v_addr);
     }
     else
     {
-        paddr = (unsigned long)ARCH_MAP_FAILED;
+        unsigned long *pte = _query(aspace, v_addr, &level_shift);
+
+        if (pte)
+        {
+            paddr = *pte & MMU_ADDRESS_MASK;
+            paddr |= (rt_ubase_t)v_addr & ((1ul << level_shift) - 1);
+        }
+        else
+        {
+            paddr = (unsigned long)ARCH_MAP_FAILED;
+        }
     }
+
     return (void *)paddr;
 }
 
-static int _noncache(uintptr_t *pte)
+static int _noncache(rt_ubase_t *pte)
 {
     int err = 0;
-    const uintptr_t idx_shift = 2;
-    const uintptr_t idx_mask = 0x7 << idx_shift;
-    uintptr_t entry = *pte;
+    const rt_ubase_t idx_shift = 2;
+    const rt_ubase_t idx_mask = 0x7 << idx_shift;
+    rt_ubase_t entry = *pte;
     if ((entry & idx_mask) == (NORMAL_MEM << idx_shift))
     {
         *pte = (entry & ~idx_mask) | (NORMAL_NOCACHE_MEM << idx_shift);
@@ -618,12 +712,12 @@ static int _noncache(uintptr_t *pte)
     return err;
 }
 
-static int _cache(uintptr_t *pte)
+static int _cache(rt_ubase_t *pte)
 {
     int err = 0;
-    const uintptr_t idx_shift = 2;
-    const uintptr_t idx_mask = 0x7 << idx_shift;
-    uintptr_t entry = *pte;
+    const rt_ubase_t idx_shift = 2;
+    const rt_ubase_t idx_mask = 0x7 << idx_shift;
+    rt_ubase_t entry = *pte;
     if ((entry & idx_mask) == (NORMAL_NOCACHE_MEM << idx_shift))
     {
         *pte = (entry & ~idx_mask) | (NORMAL_MEM << idx_shift);
@@ -636,7 +730,7 @@ static int _cache(uintptr_t *pte)
     return err;
 }
 
-static int (*control_handler[MMU_CNTL_DUMMY_END])(uintptr_t *pte) = {
+static int (*control_handler[MMU_CNTL_DUMMY_END])(rt_ubase_t *pte) = {
     [MMU_CNTL_CACHE] = _cache,
     [MMU_CNTL_NONCACHE] = _noncache,
 };
@@ -646,17 +740,18 @@ int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
 {
     int level_shift;
     int err = -RT_EINVAL;
-    void *vend = vaddr + size;
+    rt_ubase_t vstart = (rt_ubase_t)vaddr;
+    rt_ubase_t vend = vstart + size;
 
-    int (*handler)(uintptr_t * pte);
+    int (*handler)(rt_ubase_t * pte);
     if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END)
     {
         handler = control_handler[cmd];
 
-        while (vaddr < vend)
+        while (vstart < vend)
         {
-            uintptr_t *pte = _query(aspace, vaddr, &level_shift);
-            void *range_end = vaddr + (1ul << level_shift);
+            rt_ubase_t *pte = _query(aspace, (void *)vstart, &level_shift);
+            rt_ubase_t range_end = vstart + (1ul << level_shift);
             RT_ASSERT(range_end <= vend);
 
             if (pte)
@@ -664,7 +759,7 @@ int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
                 err = handler(pte);
                 RT_ASSERT(err == RT_EOK);
             }
-            vaddr = range_end;
+            vstart = range_end;
         }
     }
     else
diff --git a/libcpu/aarch64/common/mmu.h b/libcpu/aarch64/common/mmu.h
index af9505ea6a..d85f79e792 100644
--- a/libcpu/aarch64/common/mmu.h
+++ b/libcpu/aarch64/common/mmu.h
@@ -97,6 +97,28 @@ static inline void *rt_hw_mmu_tbl_get()
     return (void *)(tbl & ((1ul << 48) - 2));
 }
 
+static inline void *rt_hw_mmu_kernel_v2p(void *v_addr)
+{
+    rt_ubase_t par;
+    void *paddr;
+    asm volatile("at s1e1w, %0"::"r"(v_addr):"memory");
+    asm volatile("mrs %0, par_el1":"=r"(par)::"memory");
+
+    if (par & 0x1)
+    {
+        paddr = ARCH_MAP_FAILED;
+    }
+    else
+    {
+        #define MMU_ADDRESS_MASK 0x0000fffffffff000UL
+        par &= MMU_ADDRESS_MASK;
+        par |= (rt_ubase_t)v_addr & ARCH_PAGE_MASK;
+        paddr =  (void *)par;
+    }
+
+    return paddr;
+}
+
 int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
                       enum rt_mmu_cntl cmd);