summaryrefslogtreecommitdiff
path: root/libcpu
diff options
context:
space:
mode:
Diffstat (limited to 'libcpu')
-rw-r--r--libcpu/aarch64/common/cache.h5
-rw-r--r--libcpu/aarch64/common/cpu.c5
-rw-r--r--libcpu/aarch64/common/mmu.c173
-rw-r--r--libcpu/aarch64/common/mmu.h22
4 files changed, 164 insertions, 41 deletions
diff --git a/libcpu/aarch64/common/cache.h b/libcpu/aarch64/common/cache.h
index e94e4b65da..06a2805c48 100644
--- a/libcpu/aarch64/common/cache.h
+++ b/libcpu/aarch64/common/cache.h
@@ -23,7 +23,10 @@ void rt_hw_cpu_dcache_invalidate(void *start_addr, unsigned long size);
static inline void rt_hw_icache_invalidate_all(void)
{
- __asm_invalidate_icache_all();
+ /* wait for any modification complete */
+ __asm__ volatile ("dsb ishst");
+ __asm__ volatile ("ic iallu");
+ __asm__ volatile ("isb");
}
void rt_hw_cpu_icache_invalidate(void *addr, rt_size_t size);
diff --git a/libcpu/aarch64/common/cpu.c b/libcpu/aarch64/common/cpu.c
index f5f96a5d90..8f7f2843a9 100644
--- a/libcpu/aarch64/common/cpu.c
+++ b/libcpu/aarch64/common/cpu.c
@@ -132,7 +132,10 @@ static rt_uint64_t _read_be_number(void *start, int size)
{
rt_uint64_t buf = 0;
for (; size > 0; size--)
- buf = (buf << 32) | fdt32_to_cpu(*(uint32_t *)start++);
+ {
+ buf = (buf << 32) | fdt32_to_cpu(*(uint32_t *)start);
+ start = (uint32_t *)start + 1;
+ }
return buf;
}
diff --git a/libcpu/aarch64/common/mmu.c b/libcpu/aarch64/common/mmu.c
index 1e94fdca97..a08f3fca53 100644
--- a/libcpu/aarch64/common/mmu.c
+++ b/libcpu/aarch64/common/mmu.c
@@ -1,13 +1,14 @@
/*
- * Copyright (c) 2006-2018, RT-Thread Development Team
+ * Copyright (c) 2006-2023, RT-Thread Development Team
*
* SPDX-License-Identifier: Apache-2.0
*
* Change Logs:
* Date Author Notes
* 2012-01-10 bernard porting to AM1808
+ * 2021-11-28 GuEe-GUI first version
+ * 2022-12-10 WangXiaoyao porting to MM
*/
-
#include <board.h>
#include <rthw.h>
#include <rtthread.h>
@@ -79,6 +80,7 @@ static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr)
{
break;
}
+ /* next table entry in current level */
level_info[level].pos = cur_lv_tbl + off;
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
@@ -119,8 +121,7 @@ static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr)
return;
}
-static int _kenrel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr,
- unsigned long attr)
+static int _kernel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr)
{
int ret = 0;
int level;
@@ -145,7 +146,7 @@ static int _kenrel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr,
off &= MMU_LEVEL_MASK;
if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
{
- page = (unsigned long)rt_pages_alloc(0);
+ page = (unsigned long)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
if (!page)
{
ret = MMU_MAP_ERROR_NOPAGE;
@@ -188,19 +189,104 @@ err:
return ret;
}
+static int _kernel_map_2M(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr)
+{
+ int ret = 0;
+ int level;
+ unsigned long *cur_lv_tbl = lv0_tbl;
+ unsigned long page;
+ unsigned long off;
+ unsigned long va = (unsigned long)vaddr;
+ unsigned long pa = (unsigned long)paddr;
+
+ int level_shift = MMU_ADDRESS_BITS;
+
+ if (va & ARCH_SECTION_MASK)
+ {
+ return MMU_MAP_ERROR_VANOTALIGN;
+ }
+ if (pa & ARCH_SECTION_MASK)
+ {
+ return MMU_MAP_ERROR_PANOTALIGN;
+ }
+ for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++)
+ {
+ off = (va >> level_shift);
+ off &= MMU_LEVEL_MASK;
+ if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
+ {
+ page = (unsigned long)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
+ if (!page)
+ {
+ ret = MMU_MAP_ERROR_NOPAGE;
+ goto err;
+ }
+ rt_memset((char *)page, 0, ARCH_PAGE_SIZE);
+ rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE);
+ cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE;
+ rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
+ }
+ else
+ {
+ page = cur_lv_tbl[off];
+ page &= MMU_ADDRESS_MASK;
+ /* page to va */
+ page -= PV_OFFSET;
+ rt_page_ref_inc((void *)page, 0);
+ }
+ page = cur_lv_tbl[off];
+ if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
+ {
+ /* is block! error! */
+ ret = MMU_MAP_ERROR_CONFLICT;
+ goto err;
+ }
+ cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
+ cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
+ level_shift -= MMU_LEVEL_SHIFT;
+ }
+ /* now is level page */
+ attr &= MMU_ATTRIB_MASK;
+ pa |= (attr | MMU_TYPE_BLOCK); /* block */
+ off = (va >> ARCH_SECTION_SHIFT);
+ off &= MMU_LEVEL_MASK;
+ cur_lv_tbl[off] = pa;
+ rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
+ return ret;
+err:
+ _kenrel_unmap_4K(lv0_tbl, (void *)va);
+ return ret;
+}
+
void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
size_t attr)
{
int ret = -1;
void *unmap_va = v_addr;
- size_t npages = size >> ARCH_PAGE_SHIFT;
+ size_t npages;
+ size_t stride;
+ int (*mapper)(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr);
+
+ if (((rt_ubase_t)v_addr & ARCH_SECTION_MASK) || (size & ARCH_SECTION_MASK))
+ {
+ /* legacy 4k mapping */
+ npages = size >> ARCH_PAGE_SHIFT;
+ stride = ARCH_PAGE_SIZE;
+ mapper = _kernel_map_4K;
+ }
+ else
+ {
+ /* 2m huge page */
+ npages = size >> ARCH_SECTION_SHIFT;
+ stride = ARCH_SECTION_SIZE;
+ mapper = _kernel_map_2M;
+ }
- // TODO trying with HUGEPAGE here
while (npages--)
{
MM_PGTBL_LOCK(aspace);
- ret = _kenrel_map_4K(aspace->page_table, v_addr, p_addr, attr);
+ ret = mapper(aspace->page_table, v_addr, p_addr, attr);
MM_PGTBL_UNLOCK(aspace);
if (ret != 0)
@@ -213,12 +299,12 @@ void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
MM_PGTBL_LOCK(aspace);
_kenrel_unmap_4K(aspace->page_table, (void *)unmap_va);
MM_PGTBL_UNLOCK(aspace);
- unmap_va += ARCH_PAGE_SIZE;
+ unmap_va = (char *)unmap_va + stride;
}
break;
}
- v_addr += ARCH_PAGE_SIZE;
- p_addr += ARCH_PAGE_SIZE;
+ v_addr = (char *)v_addr + stride;
+ p_addr = (char *)p_addr + stride;
}
if (ret == 0)
@@ -244,7 +330,7 @@ void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size)
MM_PGTBL_LOCK(aspace);
_kenrel_unmap_4K(aspace->page_table, v_addr);
MM_PGTBL_UNLOCK(aspace);
- v_addr += ARCH_PAGE_SIZE;
+ v_addr = (char *)v_addr + ARCH_PAGE_SIZE;
}
}
@@ -254,7 +340,7 @@ void rt_hw_aspace_switch(rt_aspace_t aspace)
{
void *pgtbl = aspace->page_table;
pgtbl = rt_kmem_v2p(pgtbl);
- uintptr_t tcr;
+ rt_ubase_t tcr;
__asm__ volatile("msr ttbr0_el1, %0" ::"r"(pgtbl) : "memory");
@@ -337,20 +423,19 @@ void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr)
rt_page_cleanup();
}
-
#ifdef RT_USING_SMART
-static inline void _init_region(void *vaddr, size_t size)
+static void _init_region(void *vaddr, size_t size)
{
rt_ioremap_start = vaddr;
rt_ioremap_size = size;
- rt_mpr_start = rt_ioremap_start - rt_mpr_size;
+ rt_mpr_start = (char *)rt_ioremap_start - rt_mpr_size;
}
#else
-#define RTOS_VEND ((void *)0xfffffffff000UL)
+#define RTOS_VEND (0xfffffffff000UL)
static inline void _init_region(void *vaddr, size_t size)
{
- rt_mpr_start = RTOS_VEND - rt_mpr_size;
+ rt_mpr_start = (void *)(RTOS_VEND - rt_mpr_size);
}
#endif
@@ -395,7 +480,7 @@ int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, size_t size,
rt_aspace_init(aspace, (void *)KERNEL_VADDR_START, 0 - KERNEL_VADDR_START,
vtable);
#else
- rt_aspace_init(aspace, (void *)0x1000, RTOS_VEND - (void *)0x1000, vtable);
+ rt_aspace_init(aspace, (void *)0x1000, RTOS_VEND - 0x1000ul, vtable);
#endif
_init_region(v_address, size);
@@ -586,26 +671,35 @@ void *rt_hw_mmu_v2p(rt_aspace_t aspace, void *v_addr)
{
int level_shift;
unsigned long paddr;
- unsigned long *pte = _query(aspace, v_addr, &level_shift);
- if (pte)
+ if (aspace == &rt_kernel_space)
{
- paddr = *pte & MMU_ADDRESS_MASK;
- paddr |= (uintptr_t)v_addr & ((1ul << level_shift) - 1);
+ paddr = (unsigned long)rt_hw_mmu_kernel_v2p(v_addr);
}
else
{
- paddr = (unsigned long)ARCH_MAP_FAILED;
+ unsigned long *pte = _query(aspace, v_addr, &level_shift);
+
+ if (pte)
+ {
+ paddr = *pte & MMU_ADDRESS_MASK;
+ paddr |= (rt_ubase_t)v_addr & ((1ul << level_shift) - 1);
+ }
+ else
+ {
+ paddr = (unsigned long)ARCH_MAP_FAILED;
+ }
}
+
return (void *)paddr;
}
-static int _noncache(uintptr_t *pte)
+static int _noncache(rt_ubase_t *pte)
{
int err = 0;
- const uintptr_t idx_shift = 2;
- const uintptr_t idx_mask = 0x7 << idx_shift;
- uintptr_t entry = *pte;
+ const rt_ubase_t idx_shift = 2;
+ const rt_ubase_t idx_mask = 0x7 << idx_shift;
+ rt_ubase_t entry = *pte;
if ((entry & idx_mask) == (NORMAL_MEM << idx_shift))
{
*pte = (entry & ~idx_mask) | (NORMAL_NOCACHE_MEM << idx_shift);
@@ -618,12 +712,12 @@ static int _noncache(uintptr_t *pte)
return err;
}
-static int _cache(uintptr_t *pte)
+static int _cache(rt_ubase_t *pte)
{
int err = 0;
- const uintptr_t idx_shift = 2;
- const uintptr_t idx_mask = 0x7 << idx_shift;
- uintptr_t entry = *pte;
+ const rt_ubase_t idx_shift = 2;
+ const rt_ubase_t idx_mask = 0x7 << idx_shift;
+ rt_ubase_t entry = *pte;
if ((entry & idx_mask) == (NORMAL_NOCACHE_MEM << idx_shift))
{
*pte = (entry & ~idx_mask) | (NORMAL_MEM << idx_shift);
@@ -636,7 +730,7 @@ static int _cache(uintptr_t *pte)
return err;
}
-static int (*control_handler[MMU_CNTL_DUMMY_END])(uintptr_t *pte) = {
+static int (*control_handler[MMU_CNTL_DUMMY_END])(rt_ubase_t *pte) = {
[MMU_CNTL_CACHE] = _cache,
[MMU_CNTL_NONCACHE] = _noncache,
};
@@ -646,17 +740,18 @@ int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
{
int level_shift;
int err = -RT_EINVAL;
- void *vend = vaddr + size;
+ rt_ubase_t vstart = (rt_ubase_t)vaddr;
+ rt_ubase_t vend = vstart + size;
- int (*handler)(uintptr_t * pte);
+ int (*handler)(rt_ubase_t * pte);
if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END)
{
handler = control_handler[cmd];
- while (vaddr < vend)
+ while (vstart < vend)
{
- uintptr_t *pte = _query(aspace, vaddr, &level_shift);
- void *range_end = vaddr + (1ul << level_shift);
+ rt_ubase_t *pte = _query(aspace, (void *)vstart, &level_shift);
+ rt_ubase_t range_end = vstart + (1ul << level_shift);
RT_ASSERT(range_end <= vend);
if (pte)
@@ -664,7 +759,7 @@ int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
err = handler(pte);
RT_ASSERT(err == RT_EOK);
}
- vaddr = range_end;
+ vstart = range_end;
}
}
else
diff --git a/libcpu/aarch64/common/mmu.h b/libcpu/aarch64/common/mmu.h
index af9505ea6a..d85f79e792 100644
--- a/libcpu/aarch64/common/mmu.h
+++ b/libcpu/aarch64/common/mmu.h
@@ -97,6 +97,28 @@ static inline void *rt_hw_mmu_tbl_get()
return (void *)(tbl & ((1ul << 48) - 2));
}
+static inline void *rt_hw_mmu_kernel_v2p(void *v_addr)
+{
+ rt_ubase_t par;
+ void *paddr;
+ asm volatile("at s1e1w, %0"::"r"(v_addr):"memory");
+ asm volatile("mrs %0, par_el1":"=r"(par)::"memory");
+
+ if (par & 0x1)
+ {
+ paddr = ARCH_MAP_FAILED;
+ }
+ else
+ {
+ #define MMU_ADDRESS_MASK 0x0000fffffffff000UL
+ par &= MMU_ADDRESS_MASK;
+ par |= (rt_ubase_t)v_addr & ARCH_PAGE_MASK;
+ paddr = (void *)par;
+ }
+
+ return paddr;
+}
+
int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
enum rt_mmu_cntl cmd);
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-30 22:05:05 +0000