From 484a0d602e8b571a9d8186d38c8b136abc4ccfc9 Mon Sep 17 00:00:00 2001 From: wangxiaoyao Date: Mon, 20 Feb 2023 13:48:00 +0800 Subject: [fixup] add cache maintenance ops; fix bugs on cache maintenance when starting user app --- libcpu/aarch64/common/cache.S | 5 +++-- libcpu/aarch64/common/cache.h | 12 +++++++++--- libcpu/aarch64/common/cache_ops.c | 22 +++++++++++----------- libcpu/aarch64/common/cpuport.h | 10 +++++----- libcpu/aarch64/common/exception.c | 9 +++++++++ libcpu/aarch64/common/mmu.c | 25 +++++++++++++++++-------- libcpu/aarch64/common/mmu.h | 13 ------------- libcpu/aarch64/common/tlb.h | 2 +- libcpu/aarch64/cortex-a/entry_point.S | 5 +++++ 9 files changed, 60 insertions(+), 43 deletions(-) (limited to 'libcpu/aarch64') diff --git a/libcpu/aarch64/common/cache.S b/libcpu/aarch64/common/cache.S index bd8504e9ff..59fad1659e 100644 --- a/libcpu/aarch64/common/cache.S +++ b/libcpu/aarch64/common/cache.S @@ -193,8 +193,9 @@ __asm_invalidate_icache_range: */ .globl __asm_invalidate_icache_all __asm_invalidate_icache_all: - ic ialluis - isb sy + dsb sy + ic ialluis + isb sy ret .globl __asm_flush_l3_cache diff --git a/libcpu/aarch64/common/cache.h b/libcpu/aarch64/common/cache.h index 35b9893802..dec4e91bb5 100644 --- a/libcpu/aarch64/common/cache.h +++ b/libcpu/aarch64/common/cache.h @@ -11,13 +11,19 @@ #ifndef __CACHE_H__ #define __CACHE_H__ +void __asm_invalidate_icache_all(void); + void rt_hw_dcache_flush_all(void); void rt_hw_dcache_invalidate_all(void); void rt_hw_dcache_flush_range(unsigned long start_addr, unsigned long size); -void rt_hw_cpu_dcache_clean(void *addr, int size); -void rt_hw_cpu_dcache_invalidate(unsigned long start_addr,unsigned long size); +void rt_hw_cpu_dcache_clean(void *addr, unsigned long size); +void rt_hw_cpu_dcache_invalidate(void *start_addr, unsigned long size); + +static inline void rt_hw_icache_invalidate_all(void) +{ + __asm_invalidate_icache_all(); +} -void rt_hw_icache_invalidate_all(); void rt_hw_icache_invalidate_range(unsigned long start_addr, int size); #endif /* __CACHE_H__ */ diff --git a/libcpu/aarch64/common/cache_ops.c b/libcpu/aarch64/common/cache_ops.c index 018f0be4a4..3bfa9f2056 100644 --- a/libcpu/aarch64/common/cache_ops.c +++ b/libcpu/aarch64/common/cache_ops.c @@ -12,9 +12,9 @@ void __asm_invalidate_icache_all(void); void __asm_flush_dcache_all(void); -void __asm_flush_dcache_range(unsigned long start, unsigned long end); -void __asm_invalidate_dcache_range(unsigned long start, unsigned long end); -void __asm_invalidate_icache_range(unsigned long start, unsigned long end); +void __asm_flush_dcache_range(rt_size_t start, rt_size_t end); +void __asm_invalidate_dcache_range(rt_size_t start, rt_size_t end); +void __asm_invalidate_icache_range(rt_size_t start, rt_size_t end); void __asm_invalidate_dcache_all(void); void __asm_invalidate_icache_all(void); @@ -28,24 +28,24 @@ rt_inline rt_uint32_t rt_cpu_dcache_line_size(void) return 0; } -void rt_hw_cpu_icache_invalidate(void *addr, int size) +void rt_hw_cpu_icache_invalidate(void *addr, rt_size_t size) { - __asm_invalidate_icache_range((unsigned long)addr, (unsigned long)addr + size); + __asm_invalidate_icache_range((rt_size_t)addr, (rt_size_t)addr + size); } -void rt_hw_cpu_dcache_invalidate(void *addr, int size) +void rt_hw_cpu_dcache_invalidate(void *addr, rt_size_t size) { - __asm_invalidate_dcache_range((unsigned long)addr, (unsigned long)addr + size); + __asm_invalidate_dcache_range((rt_size_t)addr, (rt_size_t)addr + size); } -void rt_hw_cpu_dcache_clean(void *addr, int size) +void rt_hw_cpu_dcache_clean(void *addr, rt_size_t size) { - __asm_flush_dcache_range((unsigned long)addr, (unsigned long)addr + size); + __asm_flush_dcache_range((rt_size_t)addr, (rt_size_t)addr + size); } -void rt_hw_cpu_dcache_clean_and_invalidate(void *addr, int size) +void rt_hw_cpu_dcache_clean_and_invalidate(void *addr, rt_size_t size) { - __asm_flush_dcache_range((unsigned long)addr, (unsigned long)addr + size); + __asm_flush_dcache_range((rt_size_t)addr, (rt_size_t)addr + size); } void rt_hw_cpu_icache_ops(int ops, void *addr, int size) diff --git a/libcpu/aarch64/common/cpuport.h b/libcpu/aarch64/common/cpuport.h index cf767b9cfb..d20c1d5008 100644 --- a/libcpu/aarch64/common/cpuport.h +++ b/libcpu/aarch64/common/cpuport.h @@ -25,27 +25,27 @@ typedef union { rt_inline void rt_hw_isb(void) { - asm volatile ("isb":::"memory"); + __asm__ volatile ("isb":::"memory"); } rt_inline void rt_hw_dmb(void) { - asm volatile ("dmb ish":::"memory"); + __asm__ volatile ("dmb ish":::"memory"); } rt_inline void rt_hw_wmb(void) { - asm volatile ("dmb ishst":::"memory"); + __asm__ volatile ("dmb ishst":::"memory"); } rt_inline void rt_hw_rmb(void) { - asm volatile ("dmb ishld":::"memory"); + __asm__ volatile ("dmb ishld":::"memory"); } rt_inline void rt_hw_dsb(void) { - asm volatile ("dsb ish":::"memory"); + __asm__ volatile ("dsb ish":::"memory"); } #endif /*CPUPORT_H__*/ diff --git a/libcpu/aarch64/common/exception.c b/libcpu/aarch64/common/exception.c index c2ad726bdd..7da19c0e6e 100644 --- a/libcpu/aarch64/common/exception.c +++ b/libcpu/aarch64/common/exception.c @@ -1,3 +1,12 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-02-08 RT-Thread the first version + */ #include "rtthread.h" static void data_abort(unsigned long far, unsigned long iss) diff --git a/libcpu/aarch64/common/mmu.c b/libcpu/aarch64/common/mmu.c index b175450dc7..e13045c773 100644 --- a/libcpu/aarch64/common/mmu.c +++ b/libcpu/aarch64/common/mmu.c @@ -148,11 +148,9 @@ static int _kenrel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr, goto err; } rt_memset((void *)page, 0, ARCH_PAGE_SIZE); - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, - ARCH_PAGE_SIZE); + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE); cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE; - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, - sizeof(void *)); + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *)); } else { @@ -197,14 +195,18 @@ void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size, // TODO trying with HUGEPAGE here while (npages--) { + MM_PGTBL_LOCK(aspace); ret = _kenrel_map_4K(aspace->page_table, v_addr, p_addr, attr); + MM_PGTBL_UNLOCK(aspace); + if (ret != 0) { /* error, undo map */ while (unmap_va != v_addr) { + MM_PGTBL_LOCK(aspace); _kenrel_unmap_4K(aspace->page_table, (void *)unmap_va); - unmap_va += ARCH_PAGE_SIZE; + MM_PGTBL_UNLOCK(aspace); } break; } @@ -232,7 +234,9 @@ void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size) while (npages--) { + MM_PGTBL_LOCK(aspace); _kenrel_unmap_4K(aspace->page_table, v_addr); + MM_PGTBL_UNLOCK(aspace); v_addr += ARCH_PAGE_SIZE; } } @@ -242,7 +246,7 @@ void rt_hw_aspace_switch(rt_aspace_t aspace) if (aspace != &rt_kernel_space) { void *pgtbl = aspace->page_table; - pgtbl = _rt_kmem_v2p(pgtbl); + pgtbl = rt_kmem_v2p(pgtbl); uintptr_t tcr; __asm__ volatile("msr ttbr0_el1, %0" ::"r"(pgtbl) : "memory"); @@ -311,9 +315,14 @@ void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr) if (mdesc->paddr_start == (rt_size_t)ARCH_MAP_FAILED) mdesc->paddr_start = mdesc->vaddr_start + PV_OFFSET; - - rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr, + int retval; + retval = rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr, mdesc->paddr_start >> MM_PAGE_SHIFT, &err); + if (retval) + { + LOG_E("%s: map failed with code %d", retval); + RT_ASSERT(0); + } mdesc++; } diff --git a/libcpu/aarch64/common/mmu.h b/libcpu/aarch64/common/mmu.h index 912ee6fdb9..af9505ea6a 100644 --- a/libcpu/aarch64/common/mmu.h +++ b/libcpu/aarch64/common/mmu.h @@ -97,19 +97,6 @@ static inline void *rt_hw_mmu_tbl_get() return (void *)(tbl & ((1ul << 48) - 2)); } -static inline void *_rt_kmem_v2p(void *vaddr) -{ - return rt_hw_mmu_v2p(&rt_kernel_space, vaddr); -} - -static inline void *rt_kmem_v2p(void *vaddr) -{ - MM_PGTBL_LOCK(&rt_kernel_space); - void *paddr = _rt_kmem_v2p(vaddr); - MM_PGTBL_UNLOCK(&rt_kernel_space); - return paddr; -} - int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, enum rt_mmu_cntl cmd); diff --git a/libcpu/aarch64/common/tlb.h b/libcpu/aarch64/common/tlb.h index 088af488fc..e92735a6b3 100644 --- a/libcpu/aarch64/common/tlb.h +++ b/libcpu/aarch64/common/tlb.h @@ -67,7 +67,7 @@ static inline void rt_hw_tlb_invalidate_page(rt_aspace_t aspace, void *start) static inline void rt_hw_tlb_invalidate_range(rt_aspace_t aspace, void *start, size_t size, size_t stride) { - if (size < ARCH_PAGE_SIZE) + if (size <= ARCH_PAGE_SIZE) { rt_hw_tlb_invalidate_page(aspace, start); } diff --git a/libcpu/aarch64/cortex-a/entry_point.S b/libcpu/aarch64/cortex-a/entry_point.S index a73b420781..c1f14c28b0 100644 --- a/libcpu/aarch64/cortex-a/entry_point.S +++ b/libcpu/aarch64/cortex-a/entry_point.S @@ -89,6 +89,9 @@ __start: /* Set CPACR_EL1 (Architecture Feature Access Control Register) to avoid trap from SIMD or float point instruction */ mov x1, #0x00300000 /* Don't trap any SIMD/FP instructions in both EL0 and EL1 */ msr cpacr_el1, x1 + /* applying context change */ + dsb ish + isb /* clear bss */ GET_PHY x1, __bss_start @@ -270,11 +273,13 @@ _secondary_cpu_entry: ret after_mmu_enable_cpux: +#ifdef RT_USING_SMART mrs x0, tcr_el1 /* disable ttbr0, only using kernel space */ orr x0, x0, #(1 << 7) msr tcr_el1, x0 msr ttbr0_el1, xzr dsb sy +#endif mov x0, #1 msr spsel, x0 -- cgit v1.3.1