[fixup] add cache maintenance ops;

fix bugs on cache maintenance when starting user app

9 files changed, 60 insertions, 43 deletions

diff --git a/libcpu/aarch64/common/cache.S b/libcpu/aarch64/common/cache.S
index bd8504e9ff..59fad1659e 100644
--- a/libcpu/aarch64/common/cache.S
+++ b/libcpu/aarch64/common/cache.S
@@ -193,8 +193,9 @@ __asm_invalidate_icache_range:
  */
 .globl __asm_invalidate_icache_all
 __asm_invalidate_icache_all:
-    ic    ialluis
-    isb    sy
+    dsb     sy
+    ic      ialluis
+    isb     sy
     ret
 
 .globl __asm_flush_l3_cache
diff --git a/libcpu/aarch64/common/cache.h b/libcpu/aarch64/common/cache.h
index 35b9893802..dec4e91bb5 100644
--- a/libcpu/aarch64/common/cache.h
+++ b/libcpu/aarch64/common/cache.h
@@ -11,13 +11,19 @@
 #ifndef __CACHE_H__
 #define __CACHE_H__
 
+void __asm_invalidate_icache_all(void);
+
 void rt_hw_dcache_flush_all(void);
 void rt_hw_dcache_invalidate_all(void);
 void rt_hw_dcache_flush_range(unsigned long start_addr, unsigned long size);
-void rt_hw_cpu_dcache_clean(void *addr, int size);
-void rt_hw_cpu_dcache_invalidate(unsigned long start_addr,unsigned long size);
+void rt_hw_cpu_dcache_clean(void *addr, unsigned long size);
+void rt_hw_cpu_dcache_invalidate(void *start_addr, unsigned long size);
+
+static inline void rt_hw_icache_invalidate_all(void)
+{
+    __asm_invalidate_icache_all();
+}
 
-void rt_hw_icache_invalidate_all();
 void rt_hw_icache_invalidate_range(unsigned long start_addr, int size);
 
 #endif /* __CACHE_H__ */
diff --git a/libcpu/aarch64/common/cache_ops.c b/libcpu/aarch64/common/cache_ops.c
index 018f0be4a4..3bfa9f2056 100644
--- a/libcpu/aarch64/common/cache_ops.c
+++ b/libcpu/aarch64/common/cache_ops.c
@@ -12,9 +12,9 @@
 
 void __asm_invalidate_icache_all(void);
 void __asm_flush_dcache_all(void);
-void __asm_flush_dcache_range(unsigned long start, unsigned long end);
-void __asm_invalidate_dcache_range(unsigned long start, unsigned long end);
-void __asm_invalidate_icache_range(unsigned long start, unsigned long end);
+void __asm_flush_dcache_range(rt_size_t start, rt_size_t end);
+void __asm_invalidate_dcache_range(rt_size_t start, rt_size_t end);
+void __asm_invalidate_icache_range(rt_size_t start, rt_size_t end);
 void __asm_invalidate_dcache_all(void);
 void __asm_invalidate_icache_all(void);
 
@@ -28,24 +28,24 @@ rt_inline rt_uint32_t rt_cpu_dcache_line_size(void)
     return 0;
 }
 
-void rt_hw_cpu_icache_invalidate(void *addr, int size)
+void rt_hw_cpu_icache_invalidate(void *addr, rt_size_t size)
 {
-    __asm_invalidate_icache_range((unsigned long)addr, (unsigned long)addr + size);
+    __asm_invalidate_icache_range((rt_size_t)addr, (rt_size_t)addr + size);
 }
 
-void rt_hw_cpu_dcache_invalidate(void *addr, int size)
+void rt_hw_cpu_dcache_invalidate(void *addr, rt_size_t size)
 {
-    __asm_invalidate_dcache_range((unsigned long)addr, (unsigned long)addr + size);
+    __asm_invalidate_dcache_range((rt_size_t)addr, (rt_size_t)addr + size);
 }
 
-void rt_hw_cpu_dcache_clean(void *addr, int size)
+void rt_hw_cpu_dcache_clean(void *addr, rt_size_t size)
 {
-    __asm_flush_dcache_range((unsigned long)addr, (unsigned long)addr + size);
+    __asm_flush_dcache_range((rt_size_t)addr, (rt_size_t)addr + size);
 }
 
-void rt_hw_cpu_dcache_clean_and_invalidate(void *addr, int size)
+void rt_hw_cpu_dcache_clean_and_invalidate(void *addr, rt_size_t size)
 {
-    __asm_flush_dcache_range((unsigned long)addr, (unsigned long)addr + size);
+    __asm_flush_dcache_range((rt_size_t)addr, (rt_size_t)addr + size);
 }
 
 void rt_hw_cpu_icache_ops(int ops, void *addr, int size)
diff --git a/libcpu/aarch64/common/cpuport.h b/libcpu/aarch64/common/cpuport.h
index cf767b9cfb..d20c1d5008 100644
--- a/libcpu/aarch64/common/cpuport.h
+++ b/libcpu/aarch64/common/cpuport.h
@@ -25,27 +25,27 @@ typedef union {
 
 rt_inline void rt_hw_isb(void)
 {
-    asm volatile ("isb":::"memory");
+    __asm__ volatile ("isb":::"memory");
 }
 
 rt_inline void rt_hw_dmb(void)
 {
-    asm volatile ("dmb ish":::"memory");
+    __asm__ volatile ("dmb ish":::"memory");
 }
 
 rt_inline void rt_hw_wmb(void)
 {
-    asm volatile ("dmb ishst":::"memory");
+    __asm__ volatile ("dmb ishst":::"memory");
 }
 
 rt_inline void rt_hw_rmb(void)
 {
-    asm volatile ("dmb ishld":::"memory");
+    __asm__ volatile ("dmb ishld":::"memory");
 }
 
 rt_inline void rt_hw_dsb(void)
 {
-    asm volatile ("dsb ish":::"memory");
+    __asm__ volatile ("dsb ish":::"memory");
 }
 
 #endif  /*CPUPORT_H__*/
diff --git a/libcpu/aarch64/common/exception.c b/libcpu/aarch64/common/exception.c
index c2ad726bdd..7da19c0e6e 100644
--- a/libcpu/aarch64/common/exception.c
+++ b/libcpu/aarch64/common/exception.c
@@ -1,3 +1,12 @@
+/*
+ * Copyright (c) 2006-2023, RT-Thread Development Team
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Change Logs:
+ * Date           Author       Notes
+ * 2023-02-08     RT-Thread    the first version
+ */
 #include "rtthread.h"
 
 static void data_abort(unsigned long far, unsigned long iss)
diff --git a/libcpu/aarch64/common/mmu.c b/libcpu/aarch64/common/mmu.c
index b175450dc7..e13045c773 100644
--- a/libcpu/aarch64/common/mmu.c
+++ b/libcpu/aarch64/common/mmu.c
@@ -148,11 +148,9 @@ static int _kenrel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr,
                 goto err;
             }
             rt_memset((void *)page, 0, ARCH_PAGE_SIZE);
-            rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page,
-                                 ARCH_PAGE_SIZE);
+            rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE);
             cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE;
-            rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off,
-                                 sizeof(void *));
+            rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
         }
         else
         {
@@ -197,14 +195,18 @@ void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
     // TODO trying with HUGEPAGE here
     while (npages--)
     {
+        MM_PGTBL_LOCK(aspace);
         ret = _kenrel_map_4K(aspace->page_table, v_addr, p_addr, attr);
+        MM_PGTBL_UNLOCK(aspace);
+
         if (ret != 0)
         {
             /* error, undo map */
             while (unmap_va != v_addr)
             {
+                MM_PGTBL_LOCK(aspace);
                 _kenrel_unmap_4K(aspace->page_table, (void *)unmap_va);
-                unmap_va += ARCH_PAGE_SIZE;
+                MM_PGTBL_UNLOCK(aspace);
             }
             break;
         }
@@ -232,7 +234,9 @@ void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size)
 
     while (npages--)
     {
+        MM_PGTBL_LOCK(aspace);
         _kenrel_unmap_4K(aspace->page_table, v_addr);
+        MM_PGTBL_UNLOCK(aspace);
         v_addr += ARCH_PAGE_SIZE;
     }
 }
@@ -242,7 +246,7 @@ void rt_hw_aspace_switch(rt_aspace_t aspace)
     if (aspace != &rt_kernel_space)
     {
         void *pgtbl = aspace->page_table;
-        pgtbl = _rt_kmem_v2p(pgtbl);
+        pgtbl = rt_kmem_v2p(pgtbl);
         uintptr_t tcr;
 
         __asm__ volatile("msr ttbr0_el1, %0" ::"r"(pgtbl) : "memory");
@@ -311,9 +315,14 @@ void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr)
 
         if (mdesc->paddr_start == (rt_size_t)ARCH_MAP_FAILED)
             mdesc->paddr_start = mdesc->vaddr_start + PV_OFFSET;
-
-        rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr,
+        int retval;
+        retval = rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr,
                                  mdesc->paddr_start >> MM_PAGE_SHIFT, &err);
+        if (retval)
+        {
+            LOG_E("%s: map failed with code %d", retval);
+            RT_ASSERT(0);
+        }
         mdesc++;
     }
 
diff --git a/libcpu/aarch64/common/mmu.h b/libcpu/aarch64/common/mmu.h
index 912ee6fdb9..af9505ea6a 100644
--- a/libcpu/aarch64/common/mmu.h
+++ b/libcpu/aarch64/common/mmu.h
@@ -97,19 +97,6 @@ static inline void *rt_hw_mmu_tbl_get()
     return (void *)(tbl & ((1ul << 48) - 2));
 }
 
-static inline void *_rt_kmem_v2p(void *vaddr)
-{
-    return rt_hw_mmu_v2p(&rt_kernel_space, vaddr);
-}
-
-static inline void *rt_kmem_v2p(void *vaddr)
-{
-    MM_PGTBL_LOCK(&rt_kernel_space);
-    void *paddr = _rt_kmem_v2p(vaddr);
-    MM_PGTBL_UNLOCK(&rt_kernel_space);
-    return paddr;
-}
-
 int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
                       enum rt_mmu_cntl cmd);
 
diff --git a/libcpu/aarch64/common/tlb.h b/libcpu/aarch64/common/tlb.h
index 088af488fc..e92735a6b3 100644
--- a/libcpu/aarch64/common/tlb.h
+++ b/libcpu/aarch64/common/tlb.h
@@ -67,7 +67,7 @@ static inline void rt_hw_tlb_invalidate_page(rt_aspace_t aspace, void *start)
 static inline void rt_hw_tlb_invalidate_range(rt_aspace_t aspace, void *start,
                                               size_t size, size_t stride)
 {
-    if (size < ARCH_PAGE_SIZE)
+    if (size <= ARCH_PAGE_SIZE)
     {
         rt_hw_tlb_invalidate_page(aspace, start);
     }
diff --git a/libcpu/aarch64/cortex-a/entry_point.S b/libcpu/aarch64/cortex-a/entry_point.S
index a73b420781..c1f14c28b0 100644
--- a/libcpu/aarch64/cortex-a/entry_point.S
+++ b/libcpu/aarch64/cortex-a/entry_point.S
@@ -89,6 +89,9 @@ __start:
     /* Set CPACR_EL1 (Architecture Feature Access Control Register) to avoid trap from SIMD or float point instruction */
     mov     x1, #0x00300000         /* Don't trap any SIMD/FP instructions in both EL0 and EL1 */
     msr     cpacr_el1, x1
+    /* applying context change */
+    dsb     ish
+    isb
 
     /* clear bss */
     GET_PHY x1, __bss_start
@@ -270,11 +273,13 @@ _secondary_cpu_entry:
     ret
 
 after_mmu_enable_cpux:
+#ifdef RT_USING_SMART
     mrs x0, tcr_el1          /* disable ttbr0, only using kernel space */
     orr x0, x0, #(1 << 7)
     msr tcr_el1, x0
     msr ttbr0_el1, xzr
     dsb sy
+#endif
 
     mov     x0, #1
     msr     spsel, x0