diff options
| author | Shell <[email protected]> | 2023-01-09 10:08:55 +0800 |
|---|---|---|
| committer | GitHub <[email protected]> | 2023-01-08 21:08:55 -0500 |
| commit | 7450ef6c4d71f992444d27f4a26c16f813ed8e55 (patch) | |
| tree | 1d9b450d85c2c048cae2410e97ef9edfc2868323 /libcpu | |
| parent | 7f9ccd3c809afa6b54f0d453f2edcc8367a6aeb8 (diff) | |
[rt-smart] kernel virtual memory management layer (#6809)
synchronize virtual memory system works.
adding kernel virtual memory management layer for page-based MMU enabled architecture
porting libcpu MMU codes
porting lwp memory related codes
Diffstat (limited to 'libcpu')
44 files changed, 2095 insertions, 2558 deletions
diff --git a/libcpu/aarch64/common/context_gcc.S b/libcpu/aarch64/common/context_gcc.S index 344d7317c6..0a611cbf48 100644 --- a/libcpu/aarch64/common/context_gcc.S +++ b/libcpu/aarch64/common/context_gcc.S @@ -183,7 +183,7 @@ rt_hw_get_gtimer_frq: #ifdef RT_USING_LWP BL rt_thread_self MOV X19, X0 - BL lwp_mmu_switch + BL lwp_aspace_switch MOV X0, X19 BL lwp_user_setting_restore #endif @@ -514,28 +514,3 @@ vector_serror: STP X0, X1, [SP, #-0x10]! BL rt_hw_trap_serror b . - -.global rt_hw_mmu_switch -rt_hw_mmu_switch: - MSR TTBR0_EL1, X0 - MRS X1, TCR_EL1 - CMP X0, XZR - ORR X1, X1, #(1 << 7) - BEQ 1f - BIC X1, X1, #(1 << 7) -1: - MSR TCR_EL1, X1 - DSB SY - ISB - TLBI VMALLE1 - DSB SY - ISB - IC IALLUIS - DSB SY - ISB - RET - -.global rt_hw_mmu_tbl_get -rt_hw_mmu_tbl_get: - MRS X0, TTBR0_EL1 - RET diff --git a/libcpu/aarch64/common/cpu_ops_common.h b/libcpu/aarch64/common/cpu_ops_common.h index 3ac0a363c3..5e47584c7b 100644 --- a/libcpu/aarch64/common/cpu_ops_common.h +++ b/libcpu/aarch64/common/cpu_ops_common.h @@ -8,7 +8,7 @@ static inline rt_uint64_t get_secondary_entry_pa(void) { - rt_uint64_t secondary_entry_pa = (rt_uint64_t)rt_hw_mmu_v2p(&mmu_info, _secondary_cpu_entry); + rt_uint64_t secondary_entry_pa = (rt_uint64_t)rt_kmem_v2p(_secondary_cpu_entry); if (!secondary_entry_pa) { diff --git a/libcpu/aarch64/common/cpuport.h b/libcpu/aarch64/common/cpuport.h index 9f209069ac..cf767b9cfb 100644 --- a/libcpu/aarch64/common/cpuport.h +++ b/libcpu/aarch64/common/cpuport.h @@ -11,6 +11,7 @@ #define CPUPORT_H__ #include <armv8.h> +#include <rtdef.h> #ifdef RT_USING_SMP typedef union { @@ -29,12 +30,22 @@ rt_inline void rt_hw_isb(void) rt_inline void rt_hw_dmb(void) { - asm volatile ("dmb sy":::"memory"); + asm volatile ("dmb ish":::"memory"); +} + +rt_inline void rt_hw_wmb(void) +{ + asm volatile ("dmb ishst":::"memory"); +} + +rt_inline void rt_hw_rmb(void) +{ + asm volatile ("dmb ishld":::"memory"); } rt_inline void rt_hw_dsb(void) { - asm volatile ("dsb sy":::"memory"); + asm volatile ("dsb ish":::"memory"); } #endif /*CPUPORT_H__*/ diff --git a/libcpu/aarch64/common/interrupt.c b/libcpu/aarch64/common/interrupt.c index 3e527640b2..5a0b67160c 100644 --- a/libcpu/aarch64/common/interrupt.c +++ b/libcpu/aarch64/common/interrupt.c @@ -15,6 +15,12 @@ #include "gic.h" #include "gicv3.h" +#ifdef RT_USING_SMART +#include "ioremap.h" +#else +#define rt_ioremap(x, ...) (x) +#endif + /* exception and interrupt handler table */ struct rt_irq_desc isr_table[MAX_HANDLERS]; @@ -103,11 +109,11 @@ void rt_hw_interrupt_init(void) /* initialize ARM GIC */ #ifdef RT_USING_SMART - gic_dist_base = (rt_uint64_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_dist_base(), 0x2000, MMU_MAP_K_DEVICE); - gic_cpu_base = (rt_uint64_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_cpu_base(), 0x1000, MMU_MAP_K_DEVICE); + gic_dist_base = (rt_uint64_t)rt_ioremap((void*)platform_get_gic_dist_base(), 0x2000); + gic_cpu_base = (rt_uint64_t)rt_ioremap((void*)platform_get_gic_cpu_base(), 0x1000); #ifdef BSP_USING_GICV3 - gic_rdist_base = (rt_uint64_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_redist_base(), - RT_CPUS_NR * (2 << 16), MMU_MAP_K_DEVICE); + gic_rdist_base = (rt_uint64_t)rt_ioremap((void*)platform_get_gic_redist_base(), + RT_CPUS_NR * (2 << 16)); #endif #else gic_dist_base = platform_get_gic_dist_base(); diff --git a/libcpu/aarch64/common/mmu.c b/libcpu/aarch64/common/mmu.c index 0197e23f0f..d567d111d9 100644 --- a/libcpu/aarch64/common/mmu.c +++ b/libcpu/aarch64/common/mmu.c @@ -8,245 +8,44 @@ * 2012-01-10 bernard porting to AM1808 */ -#include <rtthread.h> -#include <rthw.h> #include <board.h> +#include <rthw.h> +#include <rtthread.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> +#include "mm_aspace.h" +#include "mm_page.h" #include "mmu.h" +#include "tlb.h" #ifdef RT_USING_SMART +#include "ioremap.h" #include <lwp_mm.h> -#include <page.h> #endif -#define MMU_LEVEL_MASK 0x1ffUL -#define MMU_LEVEL_SHIFT 9 -#define MMU_ADDRESS_BITS 39 -#define MMU_ADDRESS_MASK 0x0000fffffffff000UL -#define MMU_ATTRIB_MASK 0xfff0000000000ffcUL +#define DBG_TAG "hw.mmu" +#define DBG_LVL DBG_LOG +#include <rtdbg.h> + +#define MMU_LEVEL_MASK 0x1ffUL +#define MMU_LEVEL_SHIFT 9 +#define MMU_ADDRESS_BITS 39 +#define MMU_ADDRESS_MASK 0x0000fffffffff000UL +#define MMU_ATTRIB_MASK 0xfff0000000000ffcUL -#define MMU_TYPE_MASK 3UL -#define MMU_TYPE_USED 1UL -#define MMU_TYPE_BLOCK 1UL -#define MMU_TYPE_TABLE 3UL -#define MMU_TYPE_PAGE 3UL +#define MMU_TYPE_MASK 3UL +#define MMU_TYPE_USED 1UL +#define MMU_TYPE_BLOCK 1UL +#define MMU_TYPE_TABLE 3UL +#define MMU_TYPE_PAGE 3UL #define MMU_TBL_BLOCK_2M_LEVEL 2 #define MMU_TBL_PAGE_4k_LEVEL 3 #define MMU_TBL_LEVEL_NR 4 -void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr); - -struct page_table -{ - unsigned long page[512]; -}; - -#ifndef RT_USING_SMART -#define MMU_TBL_PAGE_NR_MAX 32 - -static rt_mutex_t mm_lock = RT_NULL; - -void rt_mm_lock(void) -{ - if (rt_thread_self()) - { - if (!mm_lock) - { - mm_lock = rt_mutex_create("mm_lock", RT_IPC_FLAG_FIFO); - } - if (mm_lock) - { - rt_mutex_take(mm_lock, RT_WAITING_FOREVER); - } - } -} - -void rt_mm_unlock(void) -{ - if (rt_thread_self()) - { - if (mm_lock) - { - rt_mutex_release(mm_lock); - } - } -} - -static volatile struct page_table MMUPage[MMU_TBL_PAGE_NR_MAX] __attribute__((aligned(4096))); - -#define rt_page_ref_inc(...) - -unsigned long rt_pages_alloc(rt_size_t size_bits) -{ - static unsigned long i = 0; - - if (i >= MMU_TBL_PAGE_NR_MAX) - { - return RT_NULL; - } - - ++i; - - return (unsigned long)&MMUPage[i - 1].page; -} -#endif - -static struct page_table *__init_page_array; -static unsigned long __page_off = 0UL; -unsigned long get_free_page(void) -{ - if (!__init_page_array) - { - unsigned long temp_page_start; - asm volatile("mov %0, sp":"=r"(temp_page_start)); - __init_page_array = (struct page_table *)(temp_page_start & ~(ARCH_SECTION_MASK)); - __page_off = 2; /* 0, 1 for ttbr0, ttrb1 */ - } - __page_off++; - return (unsigned long)(__init_page_array[__page_off - 1].page); -} - -void mmu_memset(char *dst, char v, size_t len) -{ - while (len--) - { - *dst++ = v; - } -} - -static int _map_single_page_2M(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long attr) -{ - int level; - unsigned long *cur_lv_tbl = lv0_tbl; - unsigned long page; - unsigned long off; - int level_shift = MMU_ADDRESS_BITS; - - if (va & ARCH_SECTION_MASK) - { - return MMU_MAP_ERROR_VANOTALIGN; - } - if (pa & ARCH_SECTION_MASK) - { - return MMU_MAP_ERROR_PANOTALIGN; - } - for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++) - { - off = (va >> level_shift); - off &= MMU_LEVEL_MASK; - if (!(cur_lv_tbl[off] & MMU_TYPE_USED)) - { - page = get_free_page(); - if (!page) - { - return MMU_MAP_ERROR_NOPAGE; - } - mmu_memset((char *)page, 0, ARCH_PAGE_SIZE); - cur_lv_tbl[off] = page | MMU_TYPE_TABLE; - } - page = cur_lv_tbl[off]; - if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK) - { - /* is block! error! */ - return MMU_MAP_ERROR_CONFLICT; - } - cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK); - level_shift -= MMU_LEVEL_SHIFT; - } - attr &= MMU_ATTRIB_MASK; - pa |= (attr | MMU_TYPE_BLOCK); /* block */ - off = (va >> ARCH_SECTION_SHIFT); - off &= MMU_LEVEL_MASK; - cur_lv_tbl[off] = pa; - return 0; -} - -int armv8_init_map_2M(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long count, unsigned long attr) -{ - unsigned long i; - int ret; - - if (va & ARCH_SECTION_MASK) - { - return -1; - } - if (pa & ARCH_SECTION_MASK) - { - return -1; - } - for (i = 0; i < count; i++) - { - ret = _map_single_page_2M(lv0_tbl, va, pa, attr); - va += ARCH_SECTION_SIZE; - pa += ARCH_SECTION_SIZE; - if (ret != 0) - { - return ret; - } - } - return 0; -} - -static int _kenrel_map_2M(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long attr) -{ - int level; - unsigned long *cur_lv_tbl = lv0_tbl; - unsigned long page; - unsigned long off; - int level_shift = MMU_ADDRESS_BITS; - - if (va & ARCH_SECTION_MASK) - { - return MMU_MAP_ERROR_VANOTALIGN; - } - if (pa & ARCH_SECTION_MASK) - { - return MMU_MAP_ERROR_PANOTALIGN; - } - for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++) - { - off = (va >> level_shift); - off &= MMU_LEVEL_MASK; - if (!(cur_lv_tbl[off] & MMU_TYPE_USED)) - { - page = (unsigned long)rt_pages_alloc(0); - if (!page) - { - return MMU_MAP_ERROR_NOPAGE; - } - rt_memset((char *)page, 0, ARCH_PAGE_SIZE); - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE); - cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE; - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *)); - } - else - { - page = cur_lv_tbl[off]; - page &= MMU_ADDRESS_MASK; - /* page to va */ - page -= PV_OFFSET; - rt_page_ref_inc((void *)page, 0); - } - page = cur_lv_tbl[off]; - if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK) - { - /* is block! error! */ - return MMU_MAP_ERROR_CONFLICT; - } - cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK); - cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET); - level_shift -= MMU_LEVEL_SHIFT; - } - attr &= MMU_ATTRIB_MASK; - pa |= (attr | MMU_TYPE_BLOCK); /* block */ - off = (va >> ARCH_SECTION_SHIFT); - off &= MMU_LEVEL_MASK; - cur_lv_tbl[off] = pa; - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *)); - - return 0; -} +volatile unsigned long MMUTable[512] __attribute__((aligned(4 * 1024))); struct mmu_level_info { @@ -254,7 +53,6 @@ struct mmu_level_info void *page; }; -#ifdef RT_USING_LWP static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr) { int level; @@ -313,17 +111,20 @@ static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr) } level--; } - asm volatile("tlbi vae1, %0\ndsb sy"::"r"(v_addr):"memory"); + return; } -static int _kenrel_map_4K(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long attr) +static int _kenrel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr, + unsigned long attr) { int ret = 0; int level; unsigned long *cur_lv_tbl = lv0_tbl; unsigned long page; unsigned long off; + intptr_t va = (intptr_t)vaddr; + intptr_t pa = (intptr_t)paddr; int level_shift = MMU_ADDRESS_BITS; if (va & ARCH_PAGE_MASK) @@ -347,9 +148,11 @@ static int _kenrel_map_4K(unsigned long *lv0_tbl, unsigned long va, unsigned lon goto err; } rt_memset((void *)page, 0, ARCH_PAGE_SIZE); - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE); + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, + ARCH_PAGE_SIZE); cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE; - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *)); + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, + sizeof(void *)); } else { @@ -382,139 +185,156 @@ err: _kenrel_unmap_4K(lv0_tbl, (void *)va); return ret; } -#endif -static int _kernel_map_fixed(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long count, unsigned long attr) +void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size, + size_t attr) { - unsigned long i; - int ret; - unsigned long _attr = MMU_MAP_CUSTOM(MMU_AP_KAUN, attr); + int ret = -1; - if (va & ARCH_SECTION_MASK) - { - return -1; - } - if (pa & ARCH_SECTION_MASK) - { - return -1; - } - for (i = 0; i < count; i++) + void *unmap_va = v_addr; + size_t npages = size >> ARCH_PAGE_SHIFT; + + // TODO trying with HUGEPAGE here + while (npages--) { - ret = _kenrel_map_2M(lv0_tbl, va, pa, _attr); - va += ARCH_SECTION_SIZE; - pa += ARCH_SECTION_SIZE; + ret = _kenrel_map_4K(aspace->page_table, v_addr, p_addr, attr); if (ret != 0) { - return ret; + /* error, undo map */ + while (unmap_va != v_addr) + { + _kenrel_unmap_4K(aspace->page_table, (void *)unmap_va); + unmap_va += ARCH_PAGE_SIZE; + } + break; } + v_addr += ARCH_PAGE_SIZE; + p_addr += ARCH_PAGE_SIZE; } - return 0; -} -/************ setting el1 mmu register************** - MAIR_EL1 - index 0 : memory outer writeback, write/read alloc - index 1 : memory nocache - index 2 : device nGnRnE - *****************************************************/ -void mmu_tcr_init(void) -{ - unsigned long val64; - - val64 = 0x00447fUL; - __asm__ volatile("msr MAIR_EL1, %0\n dsb sy\n"::"r"(val64)); - - /* TCR_EL1 */ - val64 = (16UL << 0) /* t0sz 48bit */ - | (0x0UL << 6) /* reserved */ - | (0x0UL << 7) /* epd0 */ - | (0x3UL << 8) /* t0 wb cacheable */ - | (0x3UL << 10) /* inner shareable */ - | (0x2UL << 12) /* t0 outer shareable */ - | (0x0UL << 14) /* t0 4K */ - | (16UL << 16) /* t1sz 48bit */ - | (0x0UL << 22) /* define asid use ttbr0.asid */ - | (0x0UL << 23) /* epd1 */ - | (0x3UL << 24) /* t1 inner wb cacheable */ - | (0x3UL << 26) /* t1 outer wb cacheable */ - | (0x2UL << 28) /* t1 outer shareable */ - | (0x2UL << 30) /* t1 4k */ - | (0x1UL << 32) /* 001b 64GB PA */ - | (0x0UL << 35) /* reserved */ - | (0x1UL << 36) /* as: 0:8bit 1:16bit */ - | (0x0UL << 37) /* tbi0 */ - | (0x0UL << 38); /* tbi1 */ - __asm__ volatile("msr TCR_EL1, %0\n"::"r"(val64)); -} - -/* dump 2nd level page table */ -void rt_hw_cpu_dump_page_table_2nd(rt_uint32_t *ptb) -{ -} + if (ret == 0) + { + return unmap_va; + } -void rt_hw_cpu_dump_page_table(rt_uint32_t *ptb) -{ + return NULL; } -volatile unsigned long MMUTable[512] __attribute__((aligned(4 * 1024))); -void rt_hw_mmu_setmtt(unsigned long vaddrStart, - unsigned long vaddrEnd, - unsigned long paddrStart, - unsigned long attr) +void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size) { - unsigned long count; + // caller guarantee that v_addr & size are page aligned + size_t npages = size >> ARCH_PAGE_SHIFT; - if (vaddrStart & ARCH_SECTION_MASK) + if (!aspace->page_table) { - while (1); - } - if (paddrStart & ARCH_SECTION_MASK) - { - while (1); - } - if (vaddrStart > vaddrEnd) - { - while (1); + return; } - count = vaddrEnd + 1; - if (count & ARCH_SECTION_MASK) + + while (npages--) { - while (1); + _kenrel_unmap_4K(aspace->page_table, v_addr); + v_addr += ARCH_PAGE_SIZE; } - count -= vaddrStart; - if (count == 0) +} + +void rt_hw_aspace_switch(rt_aspace_t aspace) +{ + if (aspace != &rt_kernel_space) { - while (1); + void *pgtbl = aspace->page_table; + pgtbl = _rt_kmem_v2p(pgtbl); + uintptr_t tcr; + + __asm__ volatile("msr ttbr0_el1, %0" ::"r"(pgtbl) : "memory"); + + __asm__ volatile("mrs %0, tcr_el1" : "=r"(tcr)); + tcr &= ~(1ul << 7); + __asm__ volatile("msr tcr_el1, %0\n" + "isb" ::"r"(tcr) + : "memory"); + + rt_hw_tlb_invalidate_all_local(); } - count >>= ARCH_SECTION_SHIFT; - _kernel_map_fixed((unsigned long *)MMUTable, vaddrStart, paddrStart, count, attr); } void rt_hw_mmu_ktbl_set(unsigned long tbl) { #ifdef RT_USING_SMART tbl += PV_OFFSET; - __asm__ volatile("msr TTBR1_EL1, %0\n dsb sy\nisb"::"r"(tbl):"memory"); + __asm__ volatile("msr TTBR1_EL1, %0\n dsb sy\nisb" ::"r"(tbl) : "memory"); #else - __asm__ volatile("msr TTBR0_EL1, %0\n dsb sy\nisb"::"r"(tbl):"memory"); + __asm__ volatile("msr TTBR0_EL1, %0\n dsb sy\nisb" ::"r"(tbl) : "memory"); #endif - __asm__ volatile("tlbi vmalle1\n dsb sy\nisb":::"memory"); - __asm__ volatile("ic ialluis\n dsb sy\nisb":::"memory"); + __asm__ volatile("tlbi vmalle1\n dsb sy\nisb" ::: "memory"); + __asm__ volatile("ic ialluis\n dsb sy\nisb" ::: "memory"); } -void rt_hw_mmu_setup(struct mem_desc *mdesc, int desc_nr) +/** + * @brief setup Page Table for kernel space. It's a fixed map + * and all mappings cannot be changed after initialization. + * + * Memory region in struct mem_desc must be page aligned, + * otherwise is a failure and no report will be + * returned. + * + * @param mmu_info + * @param mdesc + * @param desc_nr + */ +void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr) { - /* set page table */ - for (; desc_nr > 0; desc_nr--) + void *err; + for (size_t i = 0; i < desc_nr; i++) { - rt_hw_mmu_setmtt(mdesc->vaddr_start, mdesc->vaddr_end, - mdesc->paddr_start, mdesc->attr); + size_t attr; + switch (mdesc->attr) + { + case NORMAL_MEM: + attr = MMU_MAP_K_RWCB; + break; + case NORMAL_NOCACHE_MEM: + attr = MMU_MAP_K_RWCB; + break; + case DEVICE_MEM: + attr = MMU_MAP_K_DEVICE; + break; + default: + attr = MMU_MAP_K_DEVICE; + } + + struct rt_mm_va_hint hint = {.flags = MMF_MAP_FIXED, + .limit_start = aspace->start, + .limit_range_size = aspace->size, + .map_size = mdesc->vaddr_end - + mdesc->vaddr_start + 1, + .prefer = (void *)mdesc->vaddr_start}; + + rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr, + mdesc->paddr_start >> MM_PAGE_SHIFT, &err); mdesc++; } - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)MMUTable, sizeof MMUTable); - rt_hw_mmu_ktbl_set((unsigned long)MMUTable); + + rt_hw_mmu_ktbl_set((unsigned long)rt_kernel_space.page_table); + rt_page_cleanup(); } + +#ifdef RT_USING_SMART +static inline void _init_region(void *vaddr, size_t size) +{ + rt_ioremap_start = vaddr; + rt_ioremap_size = size; + rt_mpr_start = rt_ioremap_start - rt_mpr_size; +} +#else + +#define RTOS_VEND ((void *)0xfffffffff000UL) +static inline void _init_region(void *vaddr, size_t size) +{ + rt_mpr_start = RTOS_VEND - rt_mpr_size; +} +#endif + /** * This function will initialize rt_mmu_info structure. * @@ -526,11 +346,12 @@ void rt_hw_mmu_setup(struct mem_desc *mdesc, int desc_nr) * * @return 0 on successful and -1 for fail */ -int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void *v_address, size_t size, size_t *vtable, size_t pv_off) +int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, size_t size, + size_t *vtable, size_t pv_off) { size_t va_s, va_e; - if (!mmu_info || !vtable) + if (!aspace || !vtable) { return -1; } @@ -551,463 +372,292 @@ int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void *v_address, size_t size, size return -1; } - mmu_info->vtable = vtable; - mmu_info->vstart = va_s; - mmu_info->vend = va_e; - mmu_info->pv_off = pv_off; +#ifdef RT_USING_SMART + rt_aspace_init(aspace, (void *)KERNEL_VADDR_START, 0 - KERNEL_VADDR_START, + vtable); +#else + rt_aspace_init(aspace, (void *)0x1000, RTOS_VEND - (void *)0x1000, vtable); +#endif + + _init_region(v_address, size); return 0; } -int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void *v_address, size_t size) +/************ setting el1 mmu register************** + MAIR_EL1 + index 0 : memory outer writeback, write/read alloc + index 1 : memory nocache + index 2 : device nGnRnE + *****************************************************/ +void mmu_tcr_init(void) { - return 0; + unsigned long val64; + + val64 = 0x00447fUL; + __asm__ volatile("msr MAIR_EL1, %0\n dsb sy\n" ::"r"(val64)); + + /* TCR_EL1 */ + val64 = (16UL << 0) /* t0sz 48bit */ + | (0x0UL << 6) /* reserved */ + | (0x0UL << 7) /* epd0 */ + | (0x3UL << 8) /* t0 wb cacheable */ + | (0x3UL << 10) /* inner shareable */ + | (0x2UL << 12) /* t0 outer shareable */ + | (0x0UL << 14) /* t0 4K */ + | (16UL << 16) /* t1sz 48bit */ + | (0x0UL << 22) /* define asid use ttbr0.asid */ + | (0x0UL << 23) /* epd1 */ + | (0x3UL << 24) /* t1 inner wb cacheable */ + | (0x3UL << 26) /* t1 outer wb cacheable */ + | (0x2UL << 28) /* t1 outer shareable */ + | (0x2UL << 30) /* t1 4k */ + | (0x1UL << 32) /* 001b 64GB PA */ + | (0x0UL << 35) /* reserved */ + | (0x1UL << 36) /* as: 0:8bit 1:16bit */ + | (0x0UL << 37) /* tbi0 */ + | (0x0UL << 38); /* tbi1 */ + __asm__ volatile("msr TCR_EL1, %0\n" ::"r"(val64)); } -#ifdef RT_USING_SMART -static size_t find_vaddr(rt_mmu_info *mmu_info, int pages) +struct page_table { - size_t loop_pages; - size_t va; - size_t find_va = 0; - int n = 0; - size_t i; + unsigned long page[512]; +}; - if (!pages) +static struct page_table *__init_page_array; +static unsigned long __page_off = 0UL; +unsigned long get_free_page(void) +{ + if (!__init_page_array) { - return 0; + unsigned long temp_page_start; + asm volatile("mov %0, sp" : "=r"(temp_page_start)); + __init_page_array = + (struct page_table *)(temp_page_start & ~(ARCH_SECTION_MASK)); + __page_off = 2; /* 0, 1 for ttbr0, ttrb1 */ } + __page_off++; + return (unsigned long)(__init_page_array[__page_off - 1].page); +} - if (!mmu_info) +static int _map_single_page_2M(unsigned long *lv0_tbl, unsigned long va, + unsigned long pa, unsigned long attr) +{ + int level; + unsigned long *cur_lv_tbl = lv0_tbl; + unsigned long page; + unsigned long off; + int level_shift = MMU_ADDRESS_BITS; + + if (va & ARCH_SECTION_MASK) { - return 0; + return MMU_MAP_ERROR_VANOTALIGN; } - - loop_pages = mmu_info->vend - mmu_info->vstart + 1; - loop_pages <<= (ARCH_SECTION_SHIFT - ARCH_PAGE_SHIFT); - va = mmu_info->vstart; - va <<= ARCH_SECTION_SHIFT; - for (i = 0; i < loop_pages; i++, va += ARCH_PAGE_SIZE) + if (pa & ARCH_SECTION_MASK) { - if (_rt_hw_mmu_v2p(mmu_info, (void *)va)) - { - n = 0; - find_va = 0; - continue; - } - if (!find_va) + return MMU_MAP_ERROR_PANOTALIGN; + } + for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++) + { + off = (va >> level_shift); + off &= MMU_LEVEL_MASK; + if (!(cur_lv_tbl[off] & MMU_TYPE_USED)) { - find_va = va; + page = get_free_page(); + if (!page) + { + return MMU_MAP_ERROR_NOPAGE; + } + rt_memset((char *)page, 0, ARCH_PAGE_SIZE); + cur_lv_tbl[off] = page | MMU_TYPE_TABLE; } - n++; - if (n >= pages) + page = cur_lv_tbl[off]; + if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK) { - return find_va; + /* is block! error! */ + return MMU_MAP_ERROR_CONFLICT; } + cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK); + level_shift -= MMU_LEVEL_SHIFT; } + attr &= MMU_ATTRIB_MASK; + pa |= (attr | MMU_TYPE_BLOCK); /* block */ + off = (va >> ARCH_SECTION_SHIFT); + off &= MMU_LEVEL_MASK; + cur_lv_tbl[off] = pa; return 0; } -static int check_vaddr(rt_mmu_info *mmu_info, void *va, int pages) +static int _init_map_2M(unsigned long *lv0_tbl, unsigned long va, + unsigned long pa, unsigned long count, + unsigned long attr) { - size_t loop_va; - - if (!pages) - { - return -1; - } - - if (!mmu_info) - { - return -1; - } + unsigned long i; + int ret; - loop_va = ((size_t)va >> ARCH_SECTION_SHIFT); - if (loop_va < mmu_info->vstart || loop_va > mmu_info->vend) + if (va & ARCH_SECTION_MASK) { return -1; } - loop_va += ((pages << ARCH_PAGE_SHIFT) >> ARCH_SECTION_SHIFT); - if (loop_va < mmu_info->vstart || loop_va > mmu_info->vend + 1) + if (pa & ARCH_SECTION_MASK) { return -1; } - - loop_va = (size_t)va & ~ARCH_PAGE_MASK; - while (pages--) + for (i = 0; i < count; i++) { - if (_rt_hw_mmu_v2p(mmu_info, (void *)loop_va)) + ret = _map_single_page_2M(lv0_tbl, va, pa, attr); + va += ARCH_SECTION_SIZE; + pa += ARCH_SECTION_SIZE; + if (ret != 0) { - return -1; + return ret; } - loop_va += ARCH_PAGE_SIZE; } return 0; } -static void __rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, size_t npages) +static unsigned long *_query(rt_aspace_t aspace, void *vaddr, int *plvl_shf) { - size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK; + int level; + unsigned long va = (unsigned long)vaddr; + unsigned long *cur_lv_tbl; + unsigned long page; + unsigned long off; + int level_shift = MMU_ADDRESS_BITS; - if (!mmu_info || !mmu_info->vtable) - { - return; - } + cur_lv_tbl = aspace->page_table; + RT_ASSERT(cur_lv_tbl); - while (npages--) + for (level = 0; level < MMU_TBL_PAGE_4k_LEVEL; level++) { - _kenrel_unmap_4K(mmu_info->vtable, (void *)loop_va); - loop_va += ARCH_PAGE_SIZE; - } -} + off = (va >> level_shift); + off &= MMU_LEVEL_MASK; -static int __rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, size_t npages, size_t attr) -{ - int ret = -1; - size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK; - size_t loop_pa = (size_t)p_addr & ~ARCH_PAGE_MASK; - size_t unmap_va = loop_va; + if (!(cur_lv_tbl[off] & MMU_TYPE_USED)) + { + return (void *)0; + } - if (mmu_info) - { - while (npages--) + page = cur_lv_tbl[off]; + if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK) { - ret = _kenrel_map_4K(mmu_info->vtable, loop_va, loop_pa, attr); - if (ret != 0) - { - /* error, undo map */ - while (unmap_va != loop_va) - { - _kenrel_unmap_4K(mmu_info->vtable, (void *)unmap_va); - unmap_va += ARCH_PAGE_SIZE; - } - break; - } - loop_va += ARCH_PAGE_SIZE; - loop_pa += ARCH_PAGE_SIZE; + *plvl_shf = level_shift; + return &cur_lv_tbl[off]; } + + cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK); + cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET); + level_shift -= MMU_LEVEL_SHIFT; } - return ret; -} -#endif + /* now is level MMU_TBL_PAGE_4k_LEVEL */ + off = (va >> ARCH_PAGE_SHIFT); + off &= MMU_LEVEL_MASK; + page = cur_lv_tbl[off]; -static void rt_hw_cpu_tlb_invalidate(void) -{ - __asm__ volatile("tlbi vmalle1\n dsb sy\n isb sy\n"); + if (!(page & MMU_TYPE_USED)) + { + return (void *)0; + } + *plvl_shf = level_shift; + return &cur_lv_tbl[off]; } -#ifdef RT_USING_SMART -void *_rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, size_t size, size_t attr) +void *rt_hw_mmu_v2p(rt_aspace_t aspace, void *v_addr) { - size_t pa_s, pa_e; - size_t vaddr; - int pages; - int ret; + int level_shift; + unsigned long paddr; + unsigned long *pte = _query(aspace, v_addr, &level_shift); - if (!size) + if (pte) { - return 0; - } - pa_s = (size_t)p_addr; - pa_e = (size_t)p_addr + size - 1; - pa_s >>= ARCH_PAGE_SHIFT; - pa_e >>= ARCH_PAGE_SHIFT; - pages = pa_e - pa_s + 1; - if (v_addr) - { - vaddr = (size_t)v_addr; - pa_s = (size_t)p_addr; - if ((vaddr & ARCH_PAGE_MASK) != (pa_s & ARCH_PAGE_MASK)) - { - return 0; - } - vaddr &= ~ARCH_PAGE_MASK; - if (check_vaddr(mmu_info, (void *)vaddr, pages) != 0) - { - return 0; - } + paddr = *pte & MMU_ADDRESS_MASK; + paddr |= (uintptr_t)v_addr & ((1ul << level_shift) - 1); } else { - vaddr = find_vaddr(mmu_info, pages); + paddr = (unsigned long)ARCH_MAP_FAILED; } - if (vaddr) - { - rt_enter_critical(); - ret = __rt_hw_mmu_map(mmu_info, (void *)vaddr, p_addr, pages, attr); - if (ret == 0) - { - rt_hw_cpu_tlb_invalidate(); - rt_exit_critical(); - return (void *)(vaddr + ((size_t)p_addr & ARCH_PAGE_MASK)); - } - rt_exit_critical(); - } - return 0; + return (void *)paddr; } -#else -void *_rt_hw_mmu_map(rt_mmu_info *mmu_info, void* p_addr, size_t size, size_t attr) -{ - return p_addr; -} -#endif -#ifdef RT_USING_SMART -static int __rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t npages, size_t attr) +static int _noncache(uintptr_t *pte) { - size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK; - size_t loop_pa; - - if (!mmu_info) + int err = 0; + const uintptr_t idx_shift = 2; + const uintptr_t idx_mask = 0x7 << idx_shift; + uintptr_t entry = *pte; + if ((entry & idx_mask) == (NORMAL_MEM << idx_shift)) { - return -1; - } - - while (npages--) - { - loop_pa = (size_t)rt_pages_alloc(0); - if (!loop_pa) - { - goto err; - } - loop_pa += mmu_info->pv_off; - _kenrel_map_4K(mmu_info->vtable, loop_va, loop_pa, attr); - loop_va += ARCH_PAGE_SIZE; + *pte = (entry & ~idx_mask) | (NORMAL_NOCACHE_MEM << idx_shift); } - return 0; -err: + else { - /* error, unmap and quit */ - int i; - void *va, *pa; - - va = (void *)((size_t)v_addr & ~ARCH_PAGE_MASK); - for (i = 0; i < npages; i++) - { - pa = rt_hw_mmu_v2p(mmu_info, va); - pa = (void *)((char *)pa - mmu_info->pv_off); - rt_pages_free(pa, 0); - va = (void *)((char *)va + ARCH_PAGE_SIZE); - } - - __rt_hw_mmu_unmap(mmu_info, v_addr, npages); - return -1; + // do not support other type to be noncache + err = RT_ENOSYS; } + return err; } -void *_rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr) +static int _cache(uintptr_t *pte) { - size_t vaddr; - size_t offset; - int pages; - int ret; - - if (!size) + int err = 0; + const uintptr_t idx_shift = 2; + const uintptr_t idx_mask = 0x7 << idx_shift; + uintptr_t entry = *pte; + if ((entry & idx_mask) == (NORMAL_NOCACHE_MEM << idx_shift)) { - return 0; - } - offset = (size_t)v_addr & ARCH_PAGE_MASK; - size += (offset + ARCH_PAGE_SIZE - 1); - pages = (size >> ARCH_PAGE_SHIFT); - if (v_addr) - { - vaddr = (size_t)v_addr; - vaddr &= ~ARCH_PAGE_MASK; - if (check_vaddr(mmu_info, (void *)vaddr, pages) != 0) - { - return 0; - } + *pte = (entry & ~idx_mask) | (NORMAL_MEM << idx_shift); } else { - vaddr = find_vaddr(mmu_info, pages); + // do not support other type to be cache + err = -RT_ENOSYS; } - if (vaddr) - { - rt_enter_critical(); - ret = __rt_hw_mmu_map_auto(mmu_info, (void *)vaddr, pages, attr); - if (ret == 0) - { - rt_hw_cpu_tlb_invalidate(); - rt_exit_critical(); - return (void *)((char *)vaddr + offset); - } - rt_exit_critical(); - } - return 0; -} - -void _rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, size_t size) -{ - size_t va_s, va_e; - int pages; - - va_s = (size_t)v_addr; - va_e = (size_t)v_addr + size - 1; - va_s >>= ARCH_PAGE_SHIFT; - va_e >>= ARCH_PAGE_SHIFT; - pages = va_e - va_s + 1; - rt_enter_critical(); - __rt_hw_mmu_unmap(mmu_info, v_addr, pages); - rt_hw_cpu_tlb_invalidate(); - rt_exit_critical(); + return err; } -void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, size_t size, size_t attr) -{ - void *ret; - - rt_mm_lock(); - ret = _rt_hw_mmu_map(mmu_info, v_addr, p_addr, size, attr); - rt_mm_unlock(); - return ret; -} - -void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr) -{ - void *ret; - - rt_mm_lock(); - ret = _rt_hw_mmu_map_auto(mmu_info, v_addr, size, attr); - rt_mm_unlock(); - return ret; -} - -void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, size_t size) -{ - rt_mm_lock(); - _rt_hw_mmu_unmap(mmu_info, v_addr, size); - rt_mm_unlock(); -} -#else - -#include <cache.h> +static int (*control_handler[MMU_CNTL_DUMMY_END])(uintptr_t *pte) = { + [MMU_CNTL_CACHE] = _cache, + [MMU_CNTL_NONCACHE] = _noncache, +}; -void rt_hw_init_mmu_table(struct mem_desc *mdesc, rt_size_t desc_nr) +int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, + enum rt_mmu_cntl cmd) { - rt_memset((void *)MMUTable, 0, sizeof(MMUTable)); - rt_memset((void *)MMUPage, 0, sizeof(MMUPage)); + int level_shift; + int err = -RT_EINVAL; + void *vend = vaddr + size; - /* set page table */ - for (; desc_nr > 0; --desc_nr) + int (*handler)(uintptr_t * pte); + if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END) { - rt_hw_mmu_setmtt(mdesc->vaddr_start, mdesc->vaddr_end, mdesc->paddr_start, mdesc->attr); - ++mdesc; - } - - rt_hw_cpu_dcache_clean((void *)MMUTable, sizeof(MMUTable)); -} - + handler = control_handler[cmd]; -void rt_hw_mmu_init(void) -{ - unsigned long reg_val; - - reg_val = 0x00447fUL; - __asm__ volatile("msr mair_el1, %0"::"r"(reg_val)); - - rt_hw_isb(); - - reg_val = (16UL << 0) /* t0sz 48bit */ - | (0UL << 6) /* reserved */ - | (0UL << 7) /* epd0 */ - | (3UL << 8) /* t0 wb cacheable */ - | (3UL << 10) /* inner shareable */ - | (2UL << 12) /* t0 outer shareable */ - | (0UL << 14) /* t0 4K */ - | (16UL << 16) /* t1sz 48bit */ - | (0UL << 22) /* define asid use ttbr0.asid */ - | (0UL << 23) /* epd1 */ - | (3UL << 24) /* t1 inner wb cacheable */ - | (3UL << 26) /* t1 outer wb cacheable */ - | (2UL << 28) /* t1 outer shareable */ - | (2UL << 30) /* t1 4k */ - | (1UL << 32) /* 001b 64GB PA */ - | (0UL << 35) /* reserved */ - | (1UL << 36) /* as: 0:8bit 1:16bit */ - | (0UL << 37) /* tbi0 */ - | (0UL << 38); /* tbi1 */ - __asm__ volatile("msr tcr_el1, %0"::"r"(reg_val)); - - rt_hw_isb(); - - __asm__ volatile ("mrs %0, sctlr_el1":"=r"(reg_val)); - - reg_val |= 1 << 2; /* enable dcache */ - reg_val |= 1 << 0; /* enable mmu */ - - __asm__ volatile ( - "msr ttbr0_el1, %0\n\r" - "msr sctlr_el1, %1\n\r" - "dsb sy\n\r" - "isb sy\n\r" - ::"r"(MMUTable), "r"(reg_val) :"memory"); - - rt_hw_cpu_tlb_invalidate(); -} -#endif - -void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr) -{ - int level; - unsigned long va = (unsigned long)v_addr; - unsigned long pa; - unsigned long *cur_lv_tbl; - unsigned long page; - unsigned long off; - unsigned long off_addr; - int level_shift = MMU_ADDRESS_BITS; - - if (!mmu_info) - { - return (void *)0; - } - cur_lv_tbl = mmu_info->vtable; - for (level = 0; level < MMU_TBL_PAGE_4k_LEVEL; level++) - { - off = (va >> level_shift); - off &= MMU_LEVEL_MASK; - if (!(cur_lv_tbl[off] & MMU_TYPE_USED)) - { - return (void *)0; - } - page = cur_lv_tbl[off]; - if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK) + while (vaddr < vend) { - off_addr = va & ((1UL << level_shift) - 1); - pa = (page & MMU_ADDRESS_MASK); - pa += off_addr; - return (void *)pa; + uintptr_t *pte = _query(aspace, vaddr, &level_shift); + void *range_end = vaddr + (1ul << level_shift); + RT_ASSERT(range_end <= vend); + + if (pte) + { + err = handler(pte); + RT_ASSERT(err == RT_EOK); + } + vaddr = range_end; } - cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK); - cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET); - level_shift -= MMU_LEVEL_SHIFT; } - /* now is level MMU_TBL_PAGE_4k_LEVEL */ - off = (va >> ARCH_PAGE_SHIFT); - off &= MMU_LEVEL_MASK; - page = cur_lv_tbl[off]; - if (!(page & MMU_TYPE_USED)) + else { - return (void *)0; + err = -RT_ENOSYS; } - pa = (page & MMU_ADDRESS_MASK); - pa += (va & ARCH_PAGE_MASK); - return (void *)pa; -} -void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr) -{ - void *ret; - - rt_mm_lock(); - ret = _rt_hw_mmu_v2p(mmu_info, v_addr); - rt_mm_unlock(); - return ret; + return err; } - -void rt_hw_mmu_setup_early(unsigned long *tbl0, unsigned long *tbl1, unsigned long size, unsigned long pv_off) +void rt_hw_mmu_setup_early(unsigned long *tbl0, unsigned long *tbl1, + unsigned long size, unsigned long pv_off) { int ret; unsigned long va = KERNEL_VADDR_START; @@ -1015,15 +665,15 @@ void rt_hw_mmu_setup_early(unsigned long *tbl0, unsigned long *tbl1, unsigned lo unsigned long normal_attr = MMU_MAP_CUSTOM(MMU_AP_KAUN, NORMAL_MEM); /* clean the first two pages */ - mmu_memset((char *)tbl0, 0, ARCH_PAGE_SIZE); - mmu_memset((char *)tbl1, 0, ARCH_PAGE_SIZE); + rt_memset((char *)tbl0, 0, ARCH_PAGE_SIZE); + rt_memset((char *)tbl1, 0, ARCH_PAGE_SIZE); - ret = armv8_init_map_2M(tbl1, va, va + pv_off, count, normal_attr); + ret = _init_map_2M(tbl1, va, va + pv_off, count, normal_attr); if (ret != 0) { while (1); } - ret = armv8_init_map_2M(tbl0, va + pv_off, va + pv_off, count, normal_attr); + ret = _init_map_2M(tbl0, va + pv_off, va + pv_off, count, normal_attr); if (ret != 0) { while (1); diff --git a/libcpu/aarch64/common/mmu.h b/libcpu/aarch64/common/mmu.h index 0e61eeac70..6dc8cdca31 100644 --- a/libcpu/aarch64/common/mmu.h +++ b/libcpu/aarch64/common/mmu.h @@ -11,6 +11,7 @@ #define __MMU_H_ #include <rtthread.h> +#include <mm_aspace.h> /* normal memory wra mapping type */ #define NORMAL_MEM 0 @@ -25,6 +26,7 @@ struct mem_desc unsigned long vaddr_end; unsigned long paddr_start; unsigned long attr; + struct rt_varea varea; }; #define MMU_AF_SHIFT 10 @@ -37,60 +39,17 @@ struct mem_desc #define MMU_AP_KRUN 2UL /* kernel r, user none */ #define MMU_AP_KRUR 3UL /* kernel r, user r */ -#define MMU_MAP_K_RO (\ - (0x1UL << MMU_AF_SHIFT) |\ - (0x2UL << MMU_SHARED_SHIFT) |\ - (MMU_AP_KRUN << MMU_AP_SHIFT) |\ - (NORMAL_MEM << MMU_MA_SHIFT)\ - ) -#define MMU_MAP_K_RWCB (\ - (0x1UL << MMU_AF_SHIFT) |\ - (0x2UL << MMU_SHARED_SHIFT) |\ - (MMU_AP_KAUN << MMU_AP_SHIFT) |\ - (NORMAL_MEM << MMU_MA_SHIFT)\ - ) -#define MMU_MAP_K_RW (\ - (0x1UL << MMU_AF_SHIFT) |\ - (0x2UL << MMU_SHARED_SHIFT) |\ - (MMU_AP_KAUN << MMU_AP_SHIFT) |\ - (NORMAL_NOCACHE_MEM << MMU_MA_SHIFT)\ - ) -#define MMU_MAP_K_DEVICE (\ - (0x1UL << MMU_AF_SHIFT) |\ - (0x2UL << MMU_SHARED_SHIFT) |\ - (MMU_AP_KAUN << MMU_AP_SHIFT) |\ - (DEVICE_MEM << MMU_MA_SHIFT)\ - ) -#define MMU_MAP_U_RO (\ - (0x1UL << MMU_AF_SHIFT) |\ - (0x2UL << MMU_SHARED_SHIFT) |\ - (MMU_AP_KRUR << MMU_AP_SHIFT) |\ - (NORMAL_NOCACHE_MEM << MMU_MA_SHIFT)\ - ) -#define MMU_MAP_U_RWCB (\ - (0x1UL << MMU_AF_SHIFT) |\ - (0x2UL << MMU_SHARED_SHIFT) |\ - (MMU_AP_KAUA << MMU_AP_SHIFT) |\ - (NORMAL_MEM << MMU_MA_SHIFT)\ - ) -#define MMU_MAP_U_RW (\ - (0x1UL << MMU_AF_SHIFT) |\ - (0x2UL << MMU_SHARED_SHIFT) |\ - (MMU_AP_KAUA << MMU_AP_SHIFT) |\ - (NORMAL_NOCACHE_MEM << MMU_MA_SHIFT)\ - ) -#define MMU_MAP_U_DEVICE (\ - (0x1UL << MMU_AF_SHIFT) |\ - (0x2UL << MMU_SHARED_SHIFT) |\ - (MMU_AP_KAUA << MMU_AP_SHIFT) |\ - (DEVICE_MEM << MMU_MA_SHIFT)\ - ) -#define MMU_MAP_CUSTOM(ap, mtype) (\ - (0x1UL << MMU_AF_SHIFT) |\ - (0x2UL << MMU_SHARED_SHIFT) |\ - ((ap) << MMU_AP_SHIFT) |\ - ((mtype) << MMU_MA_SHIFT)\ - ) +#define MMU_MAP_CUSTOM(ap, mtype) \ + ((0x1UL << MMU_AF_SHIFT) | (0x2UL << MMU_SHARED_SHIFT) | \ + ((ap) << MMU_AP_SHIFT) | ((mtype) << MMU_MA_SHIFT)) +#define MMU_MAP_K_RO MMU_MAP_CUSTOM(MMU_AP_KRUN, NORMAL_MEM) +#define MMU_MAP_K_RWCB MMU_MAP_CUSTOM(MMU_AP_KAUN, NORMAL_MEM) +#define MMU_MAP_K_RW MMU_MAP_CUSTOM(MMU_AP_KAUN, NORMAL_NOCACHE_MEM) +#define MMU_MAP_K_DEVICE MMU_MAP_CUSTOM(MMU_AP_KAUN, DEVICE_MEM) +#define MMU_MAP_U_RO MMU_MAP_CUSTOM(MMU_AP_KRUR, NORMAL_NOCACHE_MEM) +#define MMU_MAP_U_RWCB MMU_MAP_CUSTOM(MMU_AP_KAUA, NORMAL_MEM) +#define MMU_MAP_U_RW MMU_MAP_CUSTOM(MMU_AP_KAUA, NORMAL_NOCACHE_MEM) +#define MMU_MAP_U_DEVICE MMU_MAP_CUSTOM(MMU_AP_KAUA, DEVICE_MEM) #define ARCH_SECTION_SHIFT 21 #define ARCH_SECTION_SIZE (1 << ARCH_SECTION_SHIFT) @@ -102,6 +61,7 @@ struct mem_desc #define ARCH_PAGE_TBL_SIZE (1 << ARCH_PAGE_TBL_SHIFT) #define ARCH_PAGE_TBL_MASK (ARCH_PAGE_TBL_SIZE - 1) +#define ARCH_VADDR_WIDTH 48 #define ARCH_ADDRESS_WIDTH_BITS 64 #define MMU_MAP_ERROR_VANOTALIGN -1 @@ -109,37 +69,48 @@ struct mem_desc #define MMU_MAP_ERROR_NOPAGE -3 #define MMU_MAP_ERROR_CONFLICT -4 -typedef struct -{ - size_t *vtable; - size_t vstart; - size_t vend; - size_t pv_off; -} rt_mmu_info; +#define ARCH_MAP_FAILED ((void *)0x1ffffffffffff) -void rt_hw_mmu_setup_early(unsigned long *tbl0, unsigned long *tbl1, unsigned long size, unsigned long pv_off); -void rt_hw_mmu_setup(struct mem_desc *mdesc, int desc_nr); +struct rt_aspace; -int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void* v_address, size_t size, size_t *vtable, size_t pv_off); -int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void* v_address, size_t size); +void rt_hw_mmu_ktbl_set(unsigned long tbl); +void rt_hw_mmu_setup_early(unsigned long *tbl0, unsigned long *tbl1, + unsigned long size, unsigned long pv_off); +void rt_hw_mmu_setup(struct rt_aspace *aspace, struct mem_desc *mdesc, + int desc_nr); -#ifdef RT_USING_SMART -void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void* p_addr, size_t size, size_t attr); -void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr); -#else -void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void* p_addr, size_t size, size_t attr); -#endif +int rt_hw_mmu_map_init(struct rt_aspace *aspace, void *v_address, + rt_size_t size, rt_size_t *vtable, rt_size_t pv_off); +void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr, + size_t size, size_t attr); +void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size); +void rt_hw_aspace_switch(struct rt_aspace *aspace); +void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr); +void rt_hw_mmu_kernel_map_init(struct rt_aspace *aspace, rt_size_t vaddr_start, + rt_size_t size); +void rt_hw_mmu_ktbl_set(unsigned long tbl); -void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void* v_addr, size_t size); -void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr); +static inline void *rt_hw_mmu_tbl_get() +{ + uintptr_t tbl; + __asm__ volatile("MRS %0, TTBR0_EL1" : "=r"(tbl)); + return (void *)(tbl & ((1ul << 48) - 2)); +} -void rt_hw_mmu_ktbl_set(unsigned long tbl); -void *rt_hw_mmu_tbl_get(); -void rt_hw_mmu_switch(void *mmu_table); +static inline void *_rt_kmem_v2p(void *vaddr) +{ + return rt_hw_mmu_v2p(&rt_kernel_space, vaddr); +} -void rt_hw_init_mmu_table(struct mem_desc *mdesc, rt_size_t desc_nr); -void rt_hw_mmu_init(void); +static inline void *rt_kmem_v2p(void *vaddr) +{ + MM_PGTBL_LOCK(&rt_kernel_space); + void *paddr = _rt_kmem_v2p(vaddr); + MM_PGTBL_UNLOCK(&rt_kernel_space); + return paddr; +} -extern rt_mmu_info mmu_info; +int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, + enum rt_mmu_cntl cmd); #endif diff --git a/libcpu/aarch64/common/tlb.h b/libcpu/aarch64/common/tlb.h new file mode 100644 index 0000000000..088af488fc --- /dev/null +++ b/libcpu/aarch64/common/tlb.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2006-2022, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-11-28 WangXiaoyao the first version + */ +#ifndef __TLB_H__ +#define __TLB_H__ + +#include <rtthread.h> +#include <stddef.h> +#include <stdint.h> +#include "mm_aspace.h" +#include "mmu.h" + +#define TLBI_ARG(addr, asid) \ + ({ \ + uintptr_t arg = (uintptr_t)(addr) >> 12; \ + arg &= (1ull << 44) - 1; \ + arg |= (uintptr_t)(asid) << 48; \ + (void *)arg; \ + }) + +static inline void rt_hw_tlb_invalidate_all(void) +{ + __asm__ volatile( + // ensure updates to pte completed + "dsb ishst\n" + "tlbi vmalle1is\n" + "dsb ish\n" + // after tlb in new context, refresh inst + "isb\n" :: + : "memory"); +} + +static inline void rt_hw_tlb_invalidate_all_local(void) +{ + __asm__ volatile( + // ensure updates to pte completed + "dsb nshst\n" + "tlbi vmalle1is\n" + "dsb nsh\n" + // after tlb in new context, refresh inst + "isb\n" :: + : "memory"); +} + +static inline void rt_hw_tlb_invalidate_aspace(rt_aspace_t aspace) +{ + rt_hw_tlb_invalidate_all_local(); +} + +static inline void rt_hw_tlb_invalidate_page(rt_aspace_t aspace, void *start) +{ + start = TLBI_ARG(start, 0); + __asm__ volatile( + "dsb ishst\n" + "tlbi vaae1is, %0\n" + "dsb ish\n" + "isb\n" ::"r"(start) + : "memory"); +} + +static inline void rt_hw_tlb_invalidate_range(rt_aspace_t aspace, void *start, + size_t size, size_t stride) +{ + if (size < ARCH_PAGE_SIZE) + { + rt_hw_tlb_invalidate_page(aspace, start); + } + else + { + rt_hw_tlb_invalidate_aspace(aspace); + } +} + +#endif /* __TLB_H__ */ diff --git a/libcpu/aarch64/common/trap.c b/libcpu/aarch64/common/trap.c index 0a449efeb8..0a2dd2e9ed 100644 --- a/libcpu/aarch64/common/trap.c +++ b/libcpu/aarch64/common/trap.c @@ -14,6 +14,7 @@ #include <armv8.h> #include "interrupt.h" +#include "mm_fault.h" #include <backtrace.h> @@ -48,6 +49,29 @@ void check_user_fault(struct rt_hw_exp_stack *regs, uint32_t pc_adj, char *info) } } +int _get_type(unsigned long esr) +{ + int ret; + int fsc = esr & 0x3f; + switch (fsc) + { + case 0x4: + case 0x5: + case 0x6: + case 0x7: + ret = MM_FAULT_TYPE_PAGE_FAULT; + break; + case 0x9: + case 0xa: + case 0xb: + ret = MM_FAULT_TYPE_ACCESS_FAULT; + break; + default: + ret = MM_FAULT_TYPE_GENERIC; + } + return ret; +} + int check_user_stack(unsigned long esr, struct rt_hw_exp_stack *regs) { unsigned char ec; @@ -55,19 +79,36 @@ int check_user_stack(unsigned long esr, struct rt_hw_exp_stack *regs) int ret = 0; ec = (unsigned char)((esr >> 26) & 0x3fU); + enum rt_mm_fault_op fault_op; + enum rt_mm_fault_type fault_type; switch (ec) { case 0x20: + fault_op = MM_FAULT_OP_EXECUTE; + fault_type = _get_type(esr); + break; case 0x21: case 0x24: + fault_op = MM_FAULT_OP_WRITE; + fault_type = _get_type(esr); + break; + default: + fault_op = 0; + break; + } + + if (fault_op) + { asm volatile("mrs %0, far_el1":"=r"(dfar)); - if (arch_expand_user_stack(dfar)) + struct rt_mm_fault_msg msg = { + .fault_op = fault_op, + .fault_type = fault_type, + .vaddr = dfar, + }; + if (rt_mm_fault_try_fix(&msg)) { ret = 1; } - break; - default: - break; } return ret; } diff --git a/libcpu/aarch64/cortex-a/entry_point.S b/libcpu/aarch64/cortex-a/entry_point.S index ae3fa02ccd..89c383a1e5 100644 --- a/libcpu/aarch64/cortex-a/entry_point.S +++ b/libcpu/aarch64/cortex-a/entry_point.S @@ -91,10 +91,11 @@ __start: msr cpacr_el1, x1 /* clear bss */ - ldr x1, =__bss_start /* get bss start address */ - ldr x2, =__bss_end + adrp x1, __bss_start /* get bss start address */ + add x1, x1, #:lo12:__bss_start + adrp x2, __bss_end + add x1, x1, #:lo12:__bss_end sub x2, x2, x1 /* get bss size */ - add x1, x1, x9 and x3, x2, #7 /* x3 is < 7 */ ldr x4, =~0x7 @@ -140,14 +141,14 @@ __start: orr x1, x1, #(1 << 0) /* M */ msr sctlr_el1, x1 /* enable MMU */ - dsb sy - isb sy + dsb ish + isb ic ialluis /* Invalidate all instruction caches in Inner Shareable domain to Point of Unification */ - dsb sy - isb sy + dsb ish + isb tlbi vmalle1 /* Invalidate all stage 1 translations used at EL1 with the current VMID */ - dsb sy - isb sy + dsb ish + isb ret after_mmu_enable: diff --git a/libcpu/arm/cortex-a/context_gcc.S b/libcpu/arm/cortex-a/context_gcc.S index c4531dede5..98080de5c1 100644 --- a/libcpu/arm/cortex-a/context_gcc.S +++ b/libcpu/arm/cortex-a/context_gcc.S @@ -53,7 +53,7 @@ rt_hw_context_switch_to: #ifdef RT_USING_SMART bl rt_thread_self mov r4, r0 - bl lwp_mmu_switch + bl lwp_aspace_switch mov r0, r4 bl lwp_user_setting_restore #endif @@ -116,7 +116,7 @@ rt_hw_context_switch: #ifdef RT_USING_SMART bl rt_thread_self mov r4, r0 - bl lwp_mmu_switch + bl lwp_aspace_switch mov r0, r4 bl lwp_user_setting_restore #endif diff --git a/libcpu/arm/cortex-a/cpuport.h b/libcpu/arm/cortex-a/cpuport.h index ab68d1e311..e881da9147 100644 --- a/libcpu/arm/cortex-a/cpuport.h +++ b/libcpu/arm/cortex-a/cpuport.h @@ -10,6 +10,8 @@ #ifndef CPUPORT_H__ #define CPUPORT_H__ +#include <rtthread.h> + /* the exception stack without VFP registers */ struct rt_hw_exp_stack { diff --git a/libcpu/arm/cortex-a/interrupt.c b/libcpu/arm/cortex-a/interrupt.c index fb257cc2de..d0cc89d6e2 100644 --- a/libcpu/arm/cortex-a/interrupt.c +++ b/libcpu/arm/cortex-a/interrupt.c @@ -70,8 +70,8 @@ void rt_hw_interrupt_init(void) /* initialize ARM GIC */ #ifdef RT_USING_SMART - gic_dist_base = (uint32_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_dist_base(), 0x2000, MMU_MAP_K_RW); - gic_cpu_base = (uint32_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_cpu_base(), 0x1000, MMU_MAP_K_RW); + gic_dist_base = (uint32_t)rt_ioremap((void*)platform_get_gic_dist_base(), 0x2000); + gic_cpu_base = (uint32_t)rt_ioremap((void*)platform_get_gic_cpu_base(), 0x1000); #else gic_dist_base = platform_get_gic_dist_base(); gic_cpu_base = platform_get_gic_cpu_base(); @@ -100,7 +100,8 @@ void rt_hw_interrupt_init(void) /* initialize ARM GIC */ #ifdef RT_USING_SMART - gic_dist_base = (uint32_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_dist_base(), 0x2000, MMU_MAP_K_RW); + gic_dist_base = (uint32_t)rt_ioremap((void*)platform_get_gic_dist_base(), 0x2000); + gic_cpu_base = (uint32_t)rt_ioremap((void*)platform_get_gic_cpu_base(), 0x1000); #else gic_dist_base = platform_get_gic_dist_base(); #endif diff --git a/libcpu/arm/cortex-a/mmu.c b/libcpu/arm/cortex-a/mmu.c index 19fd0f10d2..150dd98b27 100644 --- a/libcpu/arm/cortex-a/mmu.c +++ b/libcpu/arm/cortex-a/mmu.c @@ -12,52 +12,23 @@ #include <rtthread.h> #include <board.h> - #include "cp15.h" +#include "mm_page.h" #include "mmu.h" +#include <mm_aspace.h> +#include <tlb.h> #ifdef RT_USING_SMART #include <lwp_mm.h> -#include "page.h" +#include <lwp_arch.h> +#include "ioremap.h" +#else +#define KERNEL_VADDR_START 0 +#define PV_OFFSET 0 #endif /* level1 page table, each entry for 1MB memory. */ -volatile unsigned long MMUTable[4*1024] __attribute__((aligned(16*1024))); - -#ifndef RT_USING_SMART -static rt_mutex_t mm_lock = RT_NULL; - -void rt_mm_lock(void) -{ - if (rt_thread_self()) - { - if (!mm_lock) - { - mm_lock = rt_mutex_create("mm_lock", RT_IPC_FLAG_FIFO); - } - if (mm_lock) - { - rt_mutex_take(mm_lock, RT_WAITING_FOREVER); - } - } -} - -void rt_mm_unlock(void) -{ - if (rt_thread_self()) - { - if (mm_lock) - { - rt_mutex_release(mm_lock); - } - } -} -#endif - -void rt_hw_cpu_tlb_invalidate(void) -{ - asm volatile ("mcr p15, 0, r0, c8, c7, 0\ndsb\nisb" ::: "memory"); -} +volatile unsigned long MMUTable[4 * 1024] __attribute__((aligned(16 * 1024))); unsigned long rt_hw_set_domain_register(unsigned long domain_val) { @@ -69,117 +40,8 @@ unsigned long rt_hw_set_domain_register(unsigned long domain_val) return old_domain; } -/* dump 2nd level page table */ -void rt_hw_cpu_dump_page_table_2nd(rt_uint32_t *ptb) -{ - int i; - int fcnt = 0; - - for (i = 0; i < 256; i++) - { - rt_uint32_t pte2 = ptb[i]; - if ((pte2 & 0x3) == 0) - { - if (fcnt == 0) - rt_kprintf(" "); - rt_kprintf("%04x: ", i); - fcnt++; - if (fcnt == 16) - { - rt_kprintf("fault\n"); - fcnt = 0; - } - continue; - } - if (fcnt != 0) - { - rt_kprintf("fault\n"); - fcnt = 0; - } - - rt_kprintf(" %04x: %x: ", i, pte2); - if ((pte2 & 0x3) == 0x1) - { - rt_kprintf("L,ap:%x,xn:%d,texcb:%02x\n", - ((pte2 >> 7) | (pte2 >> 4))& 0xf, - (pte2 >> 15) & 0x1, - ((pte2 >> 10) | (pte2 >> 2)) & 0x1f); - } - else - { - rt_kprintf("S,ap:%x,xn:%d,texcb:%02x\n", - ((pte2 >> 7) | (pte2 >> 4))& 0xf, pte2 & 0x1, - ((pte2 >> 4) | (pte2 >> 2)) & 0x1f); - } - } -} - -void rt_hw_cpu_dump_page_table(rt_uint32_t *ptb) -{ - int i; - int fcnt = 0; - - rt_kprintf("page table@%p\n", ptb); - for (i = 0; i < 1024*4; i++) - { - rt_uint32_t pte1 = ptb[i]; - if ((pte1 & 0x3) == 0) - { - rt_kprintf("%03x: ", i); - fcnt++; - if (fcnt == 16) - { - rt_kprintf("fault\n"); - fcnt = 0; - } - continue; - } - if (fcnt != 0) - { - rt_kprintf("fault\n"); - fcnt = 0; - } - - rt_kprintf("%03x: %08x: ", i, pte1); - if ((pte1 & 0x3) == 0x3) - { - rt_kprintf("LPAE\n"); - } - else if ((pte1 & 0x3) == 0x1) - { - rt_kprintf("pte,ns:%d,domain:%d\n", - (pte1 >> 3) & 0x1, (pte1 >> 5) & 0xf); - /* - *rt_hw_cpu_dump_page_table_2nd((void*)((pte1 & 0xfffffc000) - * - 0x80000000 + 0xC0000000)); - */ - } - else if (pte1 & (1 << 18)) - { - rt_kprintf("super section,ns:%d,ap:%x,xn:%d,texcb:%02x\n", - (pte1 >> 19) & 0x1, - ((pte1 >> 13) | (pte1 >> 10))& 0xf, - (pte1 >> 4) & 0x1, - ((pte1 >> 10) | (pte1 >> 2)) & 0x1f); - } - else - { - rt_kprintf("section,ns:%d,ap:%x," - "xn:%d,texcb:%02x,domain:%d\n", - (pte1 >> 19) & 0x1, - ((pte1 >> 13) | (pte1 >> 10))& 0xf, - (pte1 >> 4) & 0x1, - (((pte1 & (0x7 << 12)) >> 10) | - ((pte1 & 0x0c) >> 2)) & 0x1f, - (pte1 >> 5) & 0xf); - } - } -} - -void rt_hw_mmu_setmtt(rt_uint32_t vaddrStart, - rt_uint32_t vaddrEnd, - rt_uint32_t paddrStart, - rt_uint32_t attr) +void rt_hw_mmu_setmtt(rt_uint32_t vaddrStart, rt_uint32_t vaddrEnd, + rt_uint32_t paddrStart, rt_uint32_t attr) { volatile rt_uint32_t *pTT; volatile int i, nSec; @@ -224,11 +86,11 @@ void rt_hw_mmu_init(void) rt_hw_cpu_dcache_enable(); } -int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void* v_address, size_t size, size_t *vtable, size_t pv_off) +int rt_hw_mmu_map_init(struct rt_aspace *aspace, void* v_address, size_t size, size_t *vtable, size_t pv_off) { size_t l1_off, va_s, va_e; - if (!mmu_info || !vtable) + if (!aspace || !vtable) { return -1; } @@ -259,15 +121,20 @@ int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void* v_address, size_t size, size } } - mmu_info->vtable = vtable; - mmu_info->vstart = va_s; - mmu_info->vend = va_e; - mmu_info->pv_off = pv_off; +#ifdef RT_USING_SMART + rt_aspace_init(&rt_kernel_space, (void *)USER_VADDR_TOP, 0 - USER_VADDR_TOP, vtable); + rt_ioremap_start = v_address; + rt_ioremap_size = size; + rt_mpr_start = rt_ioremap_start - rt_mpr_size; +#else + rt_aspace_init(&rt_kernel_space, (void *)0x1000, 0 - 0x1000, vtable); + rt_mpr_start = (void *)0 - rt_mpr_size; +#endif return 0; } -int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void* v_address, size_t size) +int rt_hw_mmu_ioremap_init(rt_aspace_t aspace, void* v_address, size_t size) { #ifdef RT_IOREMAP_LATE size_t loop_va; @@ -296,7 +163,7 @@ int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void* v_address, size_t size) while (sections--) { l1_off = (loop_va >> ARCH_SECTION_SHIFT); - mmu_l1 = (size_t*)mmu_info->vtable + l1_off; + mmu_l1 = (size_t*)aspace->page_table + l1_off; RT_ASSERT((*mmu_l1 & ARCH_MMU_USED_MASK) == 0); mmu_l2 = (size_t*)rt_pages_alloc(0); @@ -306,7 +173,7 @@ int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void* v_address, size_t size) /* cache maintain */ rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2, ARCH_PAGE_TBL_SIZE); - *mmu_l1 = (((size_t)mmu_l2 + mmu_info->pv_off) | 0x1); + *mmu_l1 = (((size_t)mmu_l2 + PV_OFFSET) | 0x1); /* cache maintain */ rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l1, 4); } @@ -323,427 +190,184 @@ int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void* v_address, size_t size) return 0; } -#ifdef RT_USING_SMART -static size_t find_vaddr(rt_mmu_info *mmu_info, int pages) -{ - size_t l1_off, l2_off; - size_t *mmu_l1, *mmu_l2; - size_t find_off = 0; - size_t find_va = 0; - int n = 0; - - if (!pages) - { - return 0; - } - if (!mmu_info) - { - return 0; - } - for (l1_off = mmu_info->vstart; l1_off <= mmu_info->vend; l1_off++) - { - mmu_l1 = (size_t*)mmu_info->vtable + l1_off; - if (*mmu_l1 & ARCH_MMU_USED_MASK) - { - mmu_l2 = (size_t *)((*mmu_l1 & ~ARCH_PAGE_TBL_MASK) - mmu_info->pv_off); - for (l2_off = 0; l2_off < (ARCH_SECTION_SIZE/ARCH_PAGE_SIZE); l2_off++) - { - if (*(mmu_l2 + l2_off) & ARCH_MMU_USED_MASK) - { - /* in use */ - n = 0; - } - else - { - if (!n) - { - find_va = l1_off; - find_off = l2_off; - } - n++; - if (n >= pages) - { - return (find_va << ARCH_SECTION_SHIFT) + (find_off << ARCH_PAGE_SHIFT); - } - } - } - } - else - { - if (!n) - { - find_va = l1_off; - find_off = 0; - } - n += (ARCH_SECTION_SIZE/ARCH_PAGE_SIZE); - if (n >= pages) - { - return (find_va << ARCH_SECTION_SHIFT) + (find_off << ARCH_PAGE_SHIFT); - } - } - } - return 0; -} -static int check_vaddr(rt_mmu_info *mmu_info, void *va, int pages) +static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr) { - size_t loop_va = (size_t)va & ~ARCH_PAGE_MASK; + size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK; size_t l1_off, l2_off; size_t *mmu_l1, *mmu_l2; - if (!pages) - { - return -1; - } + l1_off = (loop_va >> ARCH_SECTION_SHIFT); - if (!mmu_info) - { - return -1; - } + l2_off = ((loop_va & ARCH_SECTION_MASK) >> ARCH_PAGE_SHIFT); + mmu_l1 = (size_t *)lv0_tbl + l1_off; - l1_off = ((size_t)va >> ARCH_SECTION_SHIFT); - if (l1_off < mmu_info->vstart || l1_off > mmu_info->vend) + if (*mmu_l1 & ARCH_MMU_USED_MASK) { - return -1; + mmu_l2 = (size_t *)((*mmu_l1 & ~ARCH_PAGE_TBL_MASK) - PV_OFFSET); } - l1_off += ((pages << ARCH_PAGE_SHIFT) >> ARCH_SECTION_SHIFT); - if (l1_off < mmu_info->vstart || l1_off > mmu_info->vend + 1) + else { - return -1; + return; } - while (pages--) + if (*(mmu_l2 + l2_off) & ARCH_MMU_USED_MASK) { - l1_off = (loop_va >> ARCH_SECTION_SHIFT); - l2_off = ((loop_va & ARCH_SECTION_MASK) >> ARCH_PAGE_SHIFT); - mmu_l1 = (size_t*)mmu_info->vtable + l1_off; + *(mmu_l2 + l2_off) = 0; + /* cache maintain */ + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2 + l2_off, 4); - if (*mmu_l1 & ARCH_MMU_USED_MASK) + if (rt_pages_free(mmu_l2, 0)) { - mmu_l2 = (size_t *)((*mmu_l1 & ~ARCH_PAGE_TBL_MASK) - mmu_info->pv_off); - if (*(mmu_l2 + l2_off) & ARCH_MMU_USED_MASK) - { - return -1; - } + *mmu_l1 = 0; + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l1, 4); } - loop_va += ARCH_PAGE_SIZE; } - return 0; + loop_va += ARCH_PAGE_SIZE; } -static void __rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void* v_addr, size_t npages) +static int _kenrel_map_4K(unsigned long *lv0_tbl, void *v_addr, void *p_addr, + size_t attr) { size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK; + size_t loop_pa = (size_t)p_addr & ~ARCH_PAGE_MASK; size_t l1_off, l2_off; size_t *mmu_l1, *mmu_l2; - if (!mmu_info) + l1_off = (loop_va >> ARCH_SECTION_SHIFT); + l2_off = ((loop_va & ARCH_SECTION_MASK) >> ARCH_PAGE_SHIFT); + mmu_l1 = (size_t *)lv0_tbl + l1_off; + + if (*mmu_l1 & ARCH_MMU_USED_MASK) { - return; + mmu_l2 = (size_t *)((*mmu_l1 & ~ARCH_PAGE_TBL_MASK) - PV_OFFSET); + rt_page_ref_inc(mmu_l2, 0); } - - while (npages--) + else { - l1_off = (loop_va >> ARCH_SECTION_SHIFT); - if (l1_off < mmu_info->vstart || l1_off > mmu_info->vend) + mmu_l2 = (size_t *)rt_pages_alloc(0); + if (mmu_l2) { - return; - } - - l2_off = ((loop_va & ARCH_SECTION_MASK) >> ARCH_PAGE_SHIFT); - mmu_l1 = (size_t*)mmu_info->vtable + l1_off; + rt_memset(mmu_l2, 0, ARCH_PAGE_TBL_SIZE * 2); + /* cache maintain */ + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2, ARCH_PAGE_TBL_SIZE); - if (*mmu_l1 & ARCH_MMU_USED_MASK) - { - mmu_l2 = (size_t *)((*mmu_l1 & ~ARCH_PAGE_TBL_MASK) - mmu_info->pv_off); + *mmu_l1 = (((size_t)mmu_l2 + PV_OFFSET) | 0x1); + /* cache maintain */ + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l1, 4); } else { - return; + /* error, quit */ + return -1; } + } - if (*(mmu_l2 + l2_off) & ARCH_MMU_USED_MASK) - { - *(mmu_l2 + l2_off) = 0; - /* cache maintain */ - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2 + l2_off, 4); + *(mmu_l2 + l2_off) = (loop_pa | attr); + /* cache maintain */ + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2 + l2_off, 4); - if (rt_pages_free(mmu_l2, 0)) - { - *mmu_l1 = 0; - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l1, 4); - } - } - loop_va += ARCH_PAGE_SIZE; - } + loop_va += ARCH_PAGE_SIZE; + loop_pa += ARCH_PAGE_SIZE; + + return 0; } -static int __rt_hw_mmu_map(rt_mmu_info *mmu_info, void* v_addr, void* p_addr, size_t npages, size_t attr) +void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size, + size_t attr) { - size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK; - size_t loop_pa = (size_t)p_addr & ~ARCH_PAGE_MASK; - size_t l1_off, l2_off; - size_t *mmu_l1, *mmu_l2; - - if (!mmu_info) - { - return -1; - } + int ret = -1; + void *unmap_va = v_addr; + size_t npages = size >> ARCH_PAGE_SHIFT; + // TODO trying with HUGEPAGE here while (npages--) { - l1_off = (loop_va >> ARCH_SECTION_SHIFT); - l2_off = ((loop_va & ARCH_SECTION_MASK) >> ARCH_PAGE_SHIFT); - mmu_l1 = (size_t*)mmu_info->vtable + l1_off; - - if (*mmu_l1 & ARCH_MMU_USED_MASK) - { - mmu_l2 = (size_t *)((*mmu_l1 & ~ARCH_PAGE_TBL_MASK) - mmu_info->pv_off); - rt_page_ref_inc(mmu_l2, 0); - } - else + ret = _kenrel_map_4K(aspace->page_table, v_addr, p_addr, attr); + if (ret != 0) { - mmu_l2 = (size_t*)rt_pages_alloc(0); - if (mmu_l2) + /* error, undo map */ + while (unmap_va != v_addr) { - rt_memset(mmu_l2, 0, ARCH_PAGE_TBL_SIZE * 2); - /* cache maintain */ - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2, ARCH_PAGE_TBL_SIZE); + rt_enter_critical(); + _kenrel_unmap_4K(aspace->page_table, (void *)unmap_va); + rt_exit_critical(); - *mmu_l1 = (((size_t)mmu_l2 + mmu_info->pv_off) | 0x1); - /* cache maintain */ - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l1, 4); - } - else - { - /* error, unmap and quit */ - __rt_hw_mmu_unmap(mmu_info, v_addr, npages); - return -1; + unmap_va += ARCH_PAGE_SIZE; } + break; } + v_addr += ARCH_PAGE_SIZE; + p_addr += ARCH_PAGE_SIZE; + } - *(mmu_l2 + l2_off) = (loop_pa | attr); - /* cache maintain */ - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2 + l2_off, 4); - - loop_va += ARCH_PAGE_SIZE; - loop_pa += ARCH_PAGE_SIZE; + if (ret == 0) + { + return v_addr; } - return 0; + return NULL; } -void *_rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void* p_addr, size_t size, size_t attr) +void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size) { - size_t pa_s, pa_e; - size_t vaddr; - int pages; - int ret; + // caller guarantee that v_addr & size are page aligned + size_t npages = size >> ARCH_PAGE_SHIFT; - if (!size) - { - return 0; - } - pa_s = (size_t)p_addr; - pa_e = (size_t)p_addr + size - 1; - pa_s >>= ARCH_PAGE_SHIFT; - pa_e >>= ARCH_PAGE_SHIFT; - pages = pa_e - pa_s + 1; - if (v_addr) + if (!aspace->page_table) { - vaddr = (size_t)v_addr; - pa_s = (size_t)p_addr; - if ((vaddr & ARCH_PAGE_MASK) != (pa_s & ARCH_PAGE_MASK)) - { - return 0; - } - vaddr &= ~ARCH_PAGE_MASK; - if (check_vaddr(mmu_info, (void*)vaddr, pages) != 0) - { - return 0; - } + return; } - else + + while (npages--) { - vaddr = find_vaddr(mmu_info, pages); - } - if (vaddr) { rt_enter_critical(); - ret = __rt_hw_mmu_map(mmu_info, (void*)vaddr, p_addr, pages, attr); - if (ret == 0) - { - rt_hw_cpu_tlb_invalidate(); - rt_exit_critical(); - return (void*)(vaddr + ((size_t)p_addr & ARCH_PAGE_MASK)); - } + _kenrel_unmap_4K(aspace->page_table, v_addr); rt_exit_critical(); + + v_addr += ARCH_PAGE_SIZE; } - return 0; } -static int __rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void* v_addr, size_t npages, size_t attr) +void rt_hw_aspace_switch(rt_aspace_t aspace) { - size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK; - size_t loop_pa; - size_t l1_off, l2_off; - size_t *mmu_l1, *mmu_l2; - - if (!mmu_info) + if (aspace != &rt_kernel_space) { - return -1; - } - - while (npages--) - { - loop_pa = (size_t)rt_pages_alloc(0); - if (!loop_pa) - goto err; - - l1_off = (loop_va >> ARCH_SECTION_SHIFT); - l2_off = ((loop_va & ARCH_SECTION_MASK) >> ARCH_PAGE_SHIFT); - mmu_l1 = (size_t*)mmu_info->vtable + l1_off; + void *pgtbl = aspace->page_table; + pgtbl = _rt_kmem_v2p(pgtbl); - if (*mmu_l1 & ARCH_MMU_USED_MASK) - { - mmu_l2 = (size_t *)((*mmu_l1 & ~ARCH_PAGE_TBL_MASK) - mmu_info->pv_off); - rt_page_ref_inc(mmu_l2, 0); - } - else - { - //mmu_l2 = (size_t*)rt_malloc_align(ARCH_PAGE_TBL_SIZE * 2, ARCH_PAGE_TBL_SIZE); - mmu_l2 = (size_t*)rt_pages_alloc(0); - if (mmu_l2) - { - rt_memset(mmu_l2, 0, ARCH_PAGE_TBL_SIZE * 2); - /* cache maintain */ - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2, ARCH_PAGE_TBL_SIZE); - - *mmu_l1 = (((size_t)mmu_l2 + mmu_info->pv_off) | 0x1); - /* cache maintain */ - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l1, 4); - } - else - goto err; - } - - loop_pa += mmu_info->pv_off; - *(mmu_l2 + l2_off) = (loop_pa | attr); - /* cache maintain */ - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2 + l2_off, 4); - - loop_va += ARCH_PAGE_SIZE; - } - return 0; -err: - { - /* error, unmap and quit */ - int i; - void *va, *pa; - - va = (void*)((size_t)v_addr & ~ARCH_PAGE_MASK); - for (i = 0; i < npages; i++) - { - pa = rt_hw_mmu_v2p(mmu_info, va); - pa = (void*)((char*)pa - mmu_info->pv_off); - rt_pages_free(pa, 0); - va = (void*)((char*)va + ARCH_PAGE_SIZE); - } + rt_hw_mmu_switch(pgtbl); - __rt_hw_mmu_unmap(mmu_info, v_addr, npages); - return -1; + rt_hw_tlb_invalidate_all_local(); } } -void *_rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr) +void init_mm_setup(unsigned int *mtbl, unsigned int size, unsigned int pv_off) { - size_t vaddr; - size_t offset; - int pages; - int ret; + unsigned int va; - if (!size) + for (va = 0; va < 0x1000; va++) { - return 0; - } + unsigned int vaddr = (va << 20); - offset = (size_t)v_addr & ARCH_PAGE_MASK; - size += (offset + ARCH_PAGE_SIZE - 1); - pages = (size >> ARCH_PAGE_SHIFT); - if (v_addr) - { - vaddr = (size_t)v_addr; - vaddr &= ~ARCH_PAGE_MASK; - if (check_vaddr(mmu_info, (void*)vaddr, pages) != 0) + if (vaddr >= KERNEL_VADDR_START && vaddr - KERNEL_VADDR_START < size) { - return 0; + mtbl[va] = ((va << 20) + pv_off) | NORMAL_MEM; } - } - else - { - vaddr = find_vaddr(mmu_info, pages); - } - - if (vaddr) - { - rt_enter_critical(); - ret = __rt_hw_mmu_map_auto(mmu_info, (void*)vaddr, pages, attr); - if (ret == 0) + else if (vaddr >= (KERNEL_VADDR_START + pv_off) && vaddr - (KERNEL_VADDR_START + pv_off) < size) { - rt_hw_cpu_tlb_invalidate(); - rt_exit_critical(); - return (void*)((char*)vaddr + offset); + mtbl[va] = (va << 20) | NORMAL_MEM; + } + else + { + mtbl[va] = 0; } - rt_exit_critical(); } - return 0; } -void _rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void* v_addr, size_t size) -{ - size_t va_s, va_e; - int pages; - - va_s = (size_t)v_addr; - va_e = (size_t)v_addr + size - 1; - va_s >>= ARCH_PAGE_SHIFT; - va_e >>= ARCH_PAGE_SHIFT; - pages = va_e - va_s + 1; - rt_enter_critical(); - __rt_hw_mmu_unmap(mmu_info, v_addr, pages); - rt_hw_cpu_tlb_invalidate(); - rt_exit_critical(); -} - -void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void* p_addr, size_t size, size_t attr) -{ - void *ret; - - rt_mm_lock(); - ret = _rt_hw_mmu_map(mmu_info, v_addr, p_addr, size, attr); - rt_mm_unlock(); - return ret; -} - -void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr) -{ - void *ret; - - rt_mm_lock(); - ret = _rt_hw_mmu_map_auto(mmu_info, v_addr, size, attr); - rt_mm_unlock(); - return ret; -} - -void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void* v_addr, size_t size) -{ - rt_mm_lock(); - _rt_hw_mmu_unmap(mmu_info, v_addr, size); - rt_mm_unlock(); -} - -void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr) +void *rt_hw_mmu_v2p(rt_aspace_t aspace, void* v_addr) { size_t l1_off, l2_off; size_t *mmu_l1, *mmu_l2; @@ -752,12 +376,9 @@ void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr) l1_off = (size_t)v_addr >> ARCH_SECTION_SHIFT; - if (!mmu_info) - { - return (void*)0; - } + RT_ASSERT(aspace); - mmu_l1 = (size_t*)mmu_info->vtable + l1_off; + mmu_l1 = (size_t*)aspace->page_table + l1_off; tmp = *mmu_l1; @@ -766,7 +387,7 @@ void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr) case 0: /* not used */ break; case 1: /* page table */ - mmu_l2 = (size_t *)((tmp & ~ARCH_PAGE_TBL_MASK) - mmu_info->pv_off); + mmu_l2 = (size_t *)((tmp & ~ARCH_PAGE_TBL_MASK) - PV_OFFSET); l2_off = (((size_t)v_addr & ARCH_SECTION_MASK) >> ARCH_PAGE_SHIFT); pa = *(mmu_l2 + l2_off); if (pa & ARCH_MMU_USED_MASK) @@ -793,39 +414,11 @@ void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr) pa += ((size_t)v_addr & ARCH_SECTION_MASK); return (void*)pa; } - return (void*)0; + return ARCH_MAP_FAILED; } -void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr) +int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, + enum rt_mmu_cntl cmd) { - void *ret; - - rt_mm_lock(); - ret = _rt_hw_mmu_v2p(mmu_info, v_addr); - rt_mm_unlock(); - return ret; + return -RT_ENOSYS; } - -void init_mm_setup(unsigned int *mtbl, unsigned int size, unsigned int pv_off) -{ - unsigned int va; - - for (va = 0; va < 0x1000; va++) - { - unsigned int vaddr = (va << 20); - - if (vaddr >= KERNEL_VADDR_START && vaddr - KERNEL_VADDR_START < size) - { - mtbl[va] = ((va << 20) + pv_off) | NORMAL_MEM; - } - else if (vaddr >= (KERNEL_VADDR_START + pv_off) && vaddr - (KERNEL_VADDR_START + pv_off) < size) - { - mtbl[va] = (va << 20) | NORMAL_MEM; - } - else - { - mtbl[va] = 0; - } - } -} -#endif diff --git a/libcpu/arm/cortex-a/mmu.h b/libcpu/arm/cortex-a/mmu.h index 923d10df4b..04b4f485b0 100644 --- a/libcpu/arm/cortex-a/mmu.h +++ b/libcpu/arm/cortex-a/mmu.h @@ -11,6 +11,7 @@ #define __MMU_H_ #include <rtthread.h> +#include <mm_aspace.h> #define DESC_SEC (0x2) #define MEMWBWA ((1<<12)|(3<<2)) /* write back, write allocate */ @@ -87,27 +88,43 @@ struct mem_desc #define ARCH_TYPE_SUPERSECTION (1 << 18) #define ARCH_ADDRESS_WIDTH_BITS 32 +#define ARCH_VADDR_WIDTH 32 -typedef struct -{ - size_t *vtable; - size_t vstart; - size_t vend; - size_t pv_off; -} rt_mmu_info; +/** + * *info it's possible to map (-1ul & ~ARCH_PAGE_MASK) but a not aligned -1 is + * never returned on a successful mapping + */ +#define ARCH_MAP_FAILED ((void *)-1) -int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void* v_address, size_t size, size_t *vtable, size_t pv_off); -int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void* v_address, size_t size); +int rt_hw_mmu_ioremap_init(struct rt_aspace *aspace, void *v_address, size_t size); void rt_hw_init_mmu_table(struct mem_desc *mdesc, rt_uint32_t size); -#ifdef RT_USING_SMART -void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void* p_addr, size_t size, size_t attr); -void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr); -#else -void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void* p_addr, size_t size, size_t attr); -#endif +void rt_hw_mmu_setup(struct rt_aspace *aspace, struct mem_desc *mdesc, int desc_nr); + +int rt_hw_mmu_map_init(struct rt_aspace *aspace, void *v_address, size_t size, size_t *vtable, size_t pv_off); +void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr, size_t size, size_t attr); +void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size); + +void rt_hw_aspace_switch(struct rt_aspace *aspace); +void rt_hw_mmu_switch(void *tbl); + +void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr); +void rt_hw_mmu_kernel_map_init(struct rt_aspace *aspace, size_t vaddr_start, size_t size); +void *rt_hw_mmu_tbl_get(); + +static inline void *_rt_kmem_v2p(void *vaddr) +{ + return rt_hw_mmu_v2p(&rt_kernel_space, vaddr); +} + +static inline void *rt_kmem_v2p(void *vaddr) +{ + MM_PGTBL_LOCK(&rt_kernel_space); + void *paddr = _rt_kmem_v2p(vaddr); + MM_PGTBL_UNLOCK(&rt_kernel_space); + return paddr; +} -void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void* v_addr, size_t size); -void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr); +int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, enum rt_mmu_cntl cmd); #endif diff --git a/libcpu/arm/cortex-a/start_gcc.S b/libcpu/arm/cortex-a/start_gcc.S index 865f005dc8..94f96468fa 100644 --- a/libcpu/arm/cortex-a/start_gcc.S +++ b/libcpu/arm/cortex-a/start_gcc.S @@ -258,34 +258,27 @@ rt_hw_set_process_id: MCR p15, 0, r0, c13, c0, 1 mov pc, lr +#endif .global rt_hw_mmu_switch rt_hw_mmu_switch: - mov r3, #0 - mcr p15, 0, r3, c13, c0, 1 /* set contextid = 0, for synchronization*/ - isb - orr r0, #0x18 - mcr p15, 0, r0, c2, c0, 0 /* ttbr0 */ - - isb - mov r1, r1, LSL #0x8 - and r2, r2, #0xff - orr r1, r1, r2 /* contextid.PROCID = pid, contextid.ASID = asid*/ - mcr p15, 0, r1, c13, c0, 1 /* set contextid = r1*/ - isb + mcr p15, 0, r0, c2, c0, 0 // ttbr0 - mcr p15, 0, r0, c7, c5, 0 /* iciallu */ - mcr p15, 0, r0, c7, c5, 6 /* bpiall */ + //invalid tlb + mov r0, #0 + mcr p15, 0, r0, c8, c7, 0 + mcr p15, 0, r0, c7, c5, 0 //iciallu + mcr p15, 0, r0, c7, c5, 6 //bpiall dsb isb mov pc, lr + .global rt_hw_mmu_tbl_get rt_hw_mmu_tbl_get: mrc p15, 0, r0, c2, c0, 0 /* ttbr0 */ bic r0, #0x18 mov pc, lr -#endif _halt: wfe @@ -459,7 +452,7 @@ rt_hw_context_switch_interrupt_do: bl rt_thread_self #ifdef RT_USING_SMART mov r4, r0 - bl lwp_mmu_switch + bl lwp_aspace_switch mov r0, r4 bl lwp_user_setting_restore #endif diff --git a/libcpu/arm/cortex-a/tlb.h b/libcpu/arm/cortex-a/tlb.h new file mode 100644 index 0000000000..1dfb741163 --- /dev/null +++ b/libcpu/arm/cortex-a/tlb.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2006-2022, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-12-12 WangXiaoyao the first version + */ +#ifndef __TLB_H__ +#define __TLB_H__ + +#include "mm_aspace.h" +#include <rtthread.h> +#include <stddef.h> +#include <stdint.h> + +#define dsb(scope) __asm__ volatile("dsb " #scope : : : "memory") +#define isb() __asm__ volatile("isb" : : : "memory") +#define STORE_CP32(r, name...) "mcr " RT_STRINGIFY(CP32(%r, name)) ";" + +#define WRITE_CP32(v, name...) do { \ + register uint32_t _r = (v); \ + asm volatile(STORE_CP32(0, name) : : "r" (_r)); \ +} while (0) + +static inline void rt_hw_tlb_invalidate_all(void) +{ + asm volatile ("mcr p15, 0, r0, c8, c7, 0\ndsb\nisb" ::: "memory"); +} + +static inline void rt_hw_tlb_invalidate_all_local(void) +{ + rt_hw_tlb_invalidate_all(); +} + +static inline void rt_hw_tlb_invalidate_aspace(rt_aspace_t aspace) +{ + rt_hw_tlb_invalidate_all(); +} + +static inline void rt_hw_tlb_invalidate_range(rt_aspace_t aspace, void *start, + size_t size, size_t stride) +{ + rt_hw_tlb_invalidate_all(); +} + +#endif /* __TLB_H__ */ diff --git a/libcpu/arm/cortex-a/trap.c b/libcpu/arm/cortex-a/trap.c index 79897cd39c..425e51b9d0 100644 --- a/libcpu/arm/cortex-a/trap.c +++ b/libcpu/arm/cortex-a/trap.c @@ -8,12 +8,13 @@ * 2013-07-20 Bernard first version */ -#include <rtthread.h> -#include <rthw.h> -#include <board.h> #include <backtrace.h> +#include <board.h> +#include <rthw.h> +#include <rtthread.h> #include "interrupt.h" +#include "mm_fault.h" #ifdef RT_USING_FINSH extern long list_thread(void); @@ -44,14 +45,23 @@ void check_user_fault(struct rt_hw_exp_stack *regs, uint32_t pc_adj, char *info) int check_user_stack(struct rt_hw_exp_stack *regs) { - void* dfar = RT_NULL; + void *dfar = RT_NULL; + asm volatile("MRC p15, 0, %0, c6, c0, 0" : "=r"(dfar)); - asm volatile ("MRC p15, 0, %0, c6, c0, 0":"=r"(dfar)); - if (arch_expand_user_stack(dfar)) + if ((dfar >= (void *)USER_STACK_VSTART) && (dfar < (void *)USER_STACK_VEND)) { - regs->pc -= 8; - return 1; + struct rt_mm_fault_msg msg = { + .fault_op = MM_FAULT_OP_WRITE, + .fault_type = MM_FAULT_TYPE_PAGE_FAULT, + .vaddr = dfar, + }; + if (rt_mm_fault_try_fix(&msg)) + { + regs->pc -= 8; + return 1; + } } + return 0; } #endif @@ -79,7 +89,7 @@ void rt_hw_show_register(struct rt_hw_exp_stack *regs) rt_kprintf("ttbr0:0x%08x\n", v); asm volatile ("MRC p15, 0, %0, c6, c0, 0":"=r"(v)); rt_kprintf("dfar:0x%08x\n", v); - rt_kprintf("0x%08x -> 0x%08x\n", v, rt_hw_mmu_v2p(&mmu_info, (void *)v)); + rt_kprintf("0x%08x -> 0x%08x\n", v, rt_kmem_v2p((void *)v)); } #endif } diff --git a/libcpu/risc-v/t-head/c906/backtrace.c b/libcpu/risc-v/t-head/c906/backtrace.c new file mode 100644 index 0000000000..ad5dd7d2d3 --- /dev/null +++ b/libcpu/risc-v/t-head/c906/backtrace.c @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2006-2018, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + */ + +#include <rtthread.h> +#ifdef RT_USING_SMART +#include <lwp_arch.h> + +#define TRANCE_LEVEL 20 + +extern rt_ubase_t __text_start[]; +extern rt_ubase_t __text_end[]; + +static char *_get_elf_name(size_t sepc); + +void rt_hw_backtrace(rt_uint32_t *ffp, rt_ubase_t sepc) +{ + rt_ubase_t *ra; + rt_ubase_t *fp; + rt_ubase_t vas, vae; + int i, j; + + rt_kprintf("riscv64-unknown-linux-musl-addr2line -e %s -a -f", _get_elf_name(sepc)); + + fp = (rt_ubase_t *)ffp; + + if (!fp) + { + asm volatile("mv %0, s0" + : "=r"(fp)); + } + + if (sepc) + { + rt_kprintf(" %p", sepc - 0x4); + } + + if (fp > (rt_ubase_t *)USER_VADDR_START && fp < (rt_ubase_t *)USER_VADDR_TOP) + { + vas = USER_VADDR_START; + vae = USER_VADDR_TOP; + } + else + { + vas = (rt_ubase_t)&__text_start; + vae = (rt_ubase_t)&__text_end; + } + + for (i = j = 0; i < TRANCE_LEVEL; i++) + { + if (RT_ALIGN((rt_ubase_t)fp, sizeof(void *)) != (rt_ubase_t)fp) + { + break; + } + + ra = fp - 1; + if (!_rt_kmem_v2p(ra) || *ra < vas || *ra > vae) + break; + + rt_kprintf(" %p", *ra - 0x04); + + fp = fp - 2; + if (!_rt_kmem_v2p(fp)) + break; + fp = (rt_ubase_t *)(*fp); + if (!fp) + break; + } + + rt_kputs("\r\n"); +} + +static void _assert_backtrace_cb(const char *ex, const char *func, rt_size_t line) +{ + rt_hw_interrupt_disable(); + rt_kprintf("(%s) assertion failed at function:%s, line number:%d \n", ex, func, line); + + rt_hw_backtrace(0, 0); + rt_hw_cpu_shutdown(); +} + +static int rt_hw_backtrace_init(void) +{ + rt_assert_set_hook(_assert_backtrace_cb); + + return 0; +} +INIT_BOARD_EXPORT(rt_hw_backtrace_init); + +static void backtrace_test(int args, char *argv[]) +{ + int *p = (void *)-1; + init_fn_t ft = 0; + + if (args < 2) + { + rt_kprintf("backtrace_test usage:backtrace_test a(assert)/m(invalid memory)/i(illegal instruction)\r\n"); + return; + } + + if (!rt_strcmp(argv[1], "a")) + { + rt_kprintf("Assert test:\r\n", argv[1]); + RT_ASSERT(0); + } + else if (!rt_strcmp(argv[1], "m")) + { + rt_kprintf("Access invalid memory:\r\n", argv[1]); + *p = 0; + } + else if (!rt_strcmp(argv[1], "i")) + { + rt_kprintf("Illegal instruction:\r\n", argv[1]); + ft(); + } + else + { + rt_kprintf("Unknown cmd :%s.\r\n", argv[1]); + } +} +MSH_CMD_EXPORT(backtrace_test, backtrace test case); + +extern struct rt_thread *rt_current_thread; + +#define IN_USERSPACE (sepc > USER_VADDR_START && sepc < USER_VADDR_TOP) + +static char *_get_elf_name(size_t sepc) +{ + return IN_USERSPACE ? rt_current_thread->name : "rtthread.elf"; +} + +#endif /* RT_USING_SMART */ diff --git a/libcpu/risc-v/t-head/c906/context_gcc.S b/libcpu/risc-v/t-head/c906/context_gcc.S index 1442e2bf0a..595cd8f3d7 100644 --- a/libcpu/risc-v/t-head/c906/context_gcc.S +++ b/libcpu/risc-v/t-head/c906/context_gcc.S @@ -23,7 +23,7 @@ rt_hw_context_switch_to: #ifdef RT_USING_SMART mv a0, s1 - jal lwp_mmu_switch + jal lwp_aspace_switch #endif RESTORE_ALL @@ -56,7 +56,7 @@ rt_hw_context_switch: #ifdef RT_USING_SMART mv a0, s1 - jal lwp_mmu_switch + jal lwp_aspace_switch #endif RESTORE_ALL diff --git a/libcpu/risc-v/t-head/c906/cpuport.c b/libcpu/risc-v/t-head/c906/cpuport.c index 592f66bfb2..847d0b7671 100644 --- a/libcpu/risc-v/t-head/c906/cpuport.c +++ b/libcpu/risc-v/t-head/c906/cpuport.c @@ -14,6 +14,7 @@ #include <rtthread.h> #include "cpuport.h" +#include "sbi.h" #include "stack.h" #include <lwp_arch.h> @@ -110,10 +111,8 @@ void rt_hw_cpu_shutdown() rt_kprintf("shutdown...\n"); level = rt_hw_interrupt_disable(); - while (level) - { - RT_ASSERT(0); - } + sbi_shutdown(); + while (1); } int rt_hw_cpu_id(void) diff --git a/libcpu/risc-v/t-head/c906/cpuport.h b/libcpu/risc-v/t-head/c906/cpuport.h index ceea4ef5af..640c5281cf 100644 --- a/libcpu/risc-v/t-head/c906/cpuport.h +++ b/libcpu/risc-v/t-head/c906/cpuport.h @@ -37,19 +37,21 @@ #define CTX_REG_NR (CTX_GENERAL_REG_NR + CTX_FPU_REG_NR) #ifndef __ASSEMBLY__ +#include <rtthread.h> + rt_inline void rt_hw_dsb() { - asm volatile("fence":::"memory"); + __asm__ volatile("fence":::"memory"); } rt_inline void rt_hw_dmb() { - asm volatile("fence":::"memory"); + __asm__ volatile("fence":::"memory"); } rt_inline void rt_hw_isb() { - asm volatile(OPC_FENCE_I:::"memory"); + __asm__ volatile(OPC_FENCE_I:::"memory"); } int rt_hw_cpu_id(void); diff --git a/libcpu/risc-v/t-head/c906/encoding.h b/libcpu/risc-v/t-head/c906/encoding.h index 7197e6b84b..25f1b771db 100644 --- a/libcpu/risc-v/t-head/c906/encoding.h +++ b/libcpu/risc-v/t-head/c906/encoding.h @@ -33,7 +33,7 @@ #define SSTATUS_FS_CLEAN 0x00004000 #define SSTATUS_FS_DIRTY 0x00006000 #define SSTATUS_XS 0x00018000 -#define SSTATUS_PUM 0x00040000 +#define SSTATUS_SUM 0x00040000 #define SSTATUS32_SD 0x80000000 #define SSTATUS64_SD 0x8000000000000000 diff --git a/libcpu/risc-v/t-head/c906/interrupt_gcc.S b/libcpu/risc-v/t-head/c906/interrupt_gcc.S index 084dfa25f4..c9659eb512 100644 --- a/libcpu/risc-v/t-head/c906/interrupt_gcc.S +++ b/libcpu/risc-v/t-head/c906/interrupt_gcc.S @@ -96,7 +96,7 @@ copy_context_loop_interrupt: #ifdef RT_USING_SMART mv a0, s1 jal rt_thread_sp_to_thread - jal lwp_mmu_switch + jal lwp_aspace_switch #endif spurious_interrupt: diff --git a/libcpu/risc-v/t-head/c906/mmu.c b/libcpu/risc-v/t-head/c906/mmu.c index 0d461bc861..0093f5e836 100644 --- a/libcpu/risc-v/t-head/c906/mmu.c +++ b/libcpu/risc-v/t-head/c906/mmu.c @@ -8,628 +8,498 @@ * 2021-01-30 lizhirui first version */ +#include "rtconfig.h" #include <rtthread.h> -#include <rthw.h> -#include <board.h> -#include "page.h" -#include <stdlib.h> -#include <string.h> +#include <stddef.h> +#include <stdint.h> + #include <cache.h> +#include <mm_aspace.h> +#include <mm_page.h> +#include <mmu.h> +#include <riscv_mmu.h> +#include <tlb.h> + +#ifdef RT_USING_SMART +#include <ioremap.h> +#include <lwp_user_mm.h> +#include <tlb.h> +#endif -#include "riscv.h" -#include "riscv_mmu.h" -#include "mmu.h" +#define DBG_TAG "MMU" +#define DBG_LVL DBG_LOG +#include <rtdbg.h> -void *current_mmu_table = RT_NULL; +#ifndef RT_USING_SMART +#define PV_OFFSET 0 +#define USER_VADDR_START 0 +#endif -static void rt_hw_cpu_tlb_invalidate() +static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr, size_t size); + +void rt_hw_aspace_switch(rt_aspace_t aspace) { - rt_size_t satpv = read_csr(satp); - write_csr(satp,satpv); - mmu_flush_tlb(); + uintptr_t page_table = (uintptr_t)_rt_kmem_v2p(aspace->page_table); + + write_csr(satp, (((size_t)SATP_MODE) << SATP_MODE_OFFSET) | + ((rt_ubase_t)page_table >> PAGE_OFFSET_BIT)); + rt_hw_tlb_invalidate_all_local(); } +static void *current_mmu_table = RT_NULL; + +volatile __attribute__((aligned(4 * 1024))) +rt_ubase_t MMUTable[__SIZE(VPN2_BIT)]; + void *rt_hw_mmu_tbl_get() { return current_mmu_table; } -void rt_hw_mmu_switch(void *mmu_table) +static int _map_one_page(struct rt_aspace *aspace, void *va, void *pa, + size_t attr) { - current_mmu_table = mmu_table; - RT_ASSERT(__CHECKALIGN(mmu_table,PAGE_OFFSET_BIT)); - mmu_set_pagetable((rt_ubase_t)mmu_table); - rt_hw_cpu_dcache_clean_all(); - rt_hw_cpu_icache_invalidate_all(); -} + rt_size_t l1_off, l2_off, l3_off; + rt_size_t *mmu_l1, *mmu_l2, *mmu_l3; -int rt_hw_mmu_map_init(rt_mmu_info *mmu_info,void *v_address,rt_size_t size,rt_size_t *vtable,rt_size_t pv_off) -{ - size_t l1_off,va_s,va_e; - rt_base_t level; + l1_off = GET_L1((size_t)va); + l2_off = GET_L2((size_t)va); + l3_off = GET_L3((size_t)va); - if((!mmu_info) || (!vtable)) - { - return -1; - } + mmu_l1 = ((rt_size_t *)aspace->page_table) + l1_off; - va_s = (rt_size_t)v_address; - va_e = ((rt_size_t)v_address) + size - 1; - - if(va_e < va_s) + if (PTE_USED(*mmu_l1)) { - return -1; + mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET); } - - //convert address to level 1 page frame id - va_s = GET_L1(va_s); - va_e = GET_L1(va_e); - - if(va_s == 0) - { - return -1; - } - - level = rt_hw_interrupt_disable(); - - //vtable initialization check - for(l1_off = va_s;l1_off <= va_e;l1_off++) + else { - size_t v = vtable[l1_off]; + mmu_l2 = (rt_size_t *)rt_pages_alloc(0); - if(v) + if (mmu_l2) { - rt_hw_interrupt_enable(level); - return 0; + rt_memset(mmu_l2, 0, PAGE_SIZE); + rt_hw_cpu_dcache_clean(mmu_l2, PAGE_SIZE); + *mmu_l1 = COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l2, PV_OFFSET), + PAGE_DEFAULT_ATTR_NEXT); + rt_hw_cpu_dcache_clean(mmu_l1, sizeof(*mmu_l1)); + } + else + { + return -1; } } - mmu_info -> vtable = vtable; - mmu_info -> vstart = va_s; - mmu_info -> vend = va_e; - mmu_info -> pv_off = pv_off; - - rt_hw_interrupt_enable(level); - return 0; -} - -void rt_hw_mmu_kernel_map_init(rt_mmu_info *mmu_info,rt_size_t vaddr_start,rt_size_t size) -{ - rt_size_t paddr_start = __UMASKVALUE(VPN_TO_PPN(vaddr_start,mmu_info -> pv_off),PAGE_OFFSET_MASK); - rt_size_t va_s = GET_L1(vaddr_start); - rt_size_t va_e = GET_L1(vaddr_start + size - 1); - rt_size_t i; - - for(i = va_s;i <= va_e;i++) - { - mmu_info -> vtable[i] = COMBINEPTE(paddr_start,PAGE_ATTR_RWX | PTE_G | PTE_V | PTE_CACHE | PTE_SHARE | PTE_BUF | PTE_A | PTE_D); - paddr_start += L1_PAGE_SIZE; - } - - rt_hw_cpu_tlb_invalidate(); -} - -//find a range of free virtual address specified by pages -static rt_size_t find_vaddr(rt_mmu_info *mmu_info,rt_size_t pages) -{ - rt_size_t l1_off,l2_off,l3_off; - rt_size_t *mmu_l1,*mmu_l2,*mmu_l3; - rt_size_t find_l1 = 0,find_l2 = 0,find_l3 = 0; - rt_size_t n = 0; - - if(!pages) - { - return 0; - } - - if(!mmu_info) + if (PTE_USED(*(mmu_l2 + l2_off))) { - return 0; + RT_ASSERT(!PAGE_IS_LEAF(*(mmu_l2 + l2_off))); + mmu_l3 = + (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), PV_OFFSET); } - - for(l1_off = mmu_info -> vstart;l1_off <= mmu_info -> vend;l1_off++) + else { - mmu_l1 = ((rt_size_t *)mmu_info -> vtable) + l1_off; + mmu_l3 = (rt_size_t *)rt_pages_alloc(0); - if(PTE_USED(*mmu_l1)) + if (mmu_l3) { - RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1)); - mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1),mmu_info -> pv_off); - - for(l2_off = 0;l2_off < __SIZE(VPN1_BIT);l2_off++) - { - if(PTE_USED(*(mmu_l2 + l2_off))) - { - RT_ASSERT(!PAGE_IS_LEAF(*(mmu_l2 + l2_off))); - mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)),mmu_info -> pv_off); - - for(l3_off = 0;l3_off < __SIZE(VPN0_BIT);l3_off++) - { - if(PTE_USED(*(mmu_l3 + l3_off))) - { - RT_ASSERT(PAGE_IS_LEAF(*(mmu_l3 + l3_off))); - n = 0;//in use - } - else - { - if(!n) - { - find_l1 = l1_off; - find_l2 = l2_off; - find_l3 = l3_off; - } - - n++; - - if(n >= pages) - { - return COMBINEVADDR(find_l1,find_l2,find_l3); - } - } - } - } - else - { - if(!n) - { - find_l1 = l1_off; - find_l2 = l2_off; - find_l3 = 0; - } - - n += __SIZE(VPN0_BIT); - - if(n >= pages) - { - return COMBINEVADDR(find_l1,find_l2,find_l3); - } - } - } + rt_memset(mmu_l3, 0, PAGE_SIZE); + rt_hw_cpu_dcache_clean(mmu_l3, PAGE_SIZE); + *(mmu_l2 + l2_off) = + COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l3, PV_OFFSET), + PAGE_DEFAULT_ATTR_NEXT); + rt_hw_cpu_dcache_clean(mmu_l2, sizeof(*mmu_l2)); + // declares a reference to parent page table + rt_page_ref_inc((void *)mmu_l2, 0); } else { - if(!n) - { - find_l1 = l1_off; - find_l2 = 0; - find_l3 = 0; - } - - n += __SIZE(VPN1_BIT); - - if(n >= pages) - { - return COMBINEVADDR(find_l1,find_l2,find_l3); - } + return -1; } } + RT_ASSERT(!PTE_USED(*(mmu_l3 + l3_off))); + // declares a reference to parent page table + rt_page_ref_inc((void *)mmu_l3, 0); + *(mmu_l3 + l3_off) = COMBINEPTE((rt_size_t)pa, attr); + rt_hw_cpu_dcache_clean(mmu_l3 + l3_off, sizeof(*(mmu_l3 + l3_off))); return 0; } -//check whether the range of virtual address are free -static int check_vaddr(rt_mmu_info *mmu_info,void *va,rt_size_t pages) +/** rt_hw_mmu_map will never override existed page table entry */ +void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr, + size_t size, size_t attr) { - rt_size_t loop_va = __UMASKVALUE((rt_size_t)va,PAGE_OFFSET_MASK); - rt_size_t l1_off,l2_off,l3_off; - rt_size_t *mmu_l1,*mmu_l2,*mmu_l3; + int ret = -1; + void *unmap_va = v_addr; + size_t npages = size >> ARCH_PAGE_SHIFT; - if(!pages) + // TODO trying with HUGEPAGE here + while (npages--) { - return -1; - } - - if(!mmu_info) - { - return -1; - } - - while(pages--) - { - l1_off = GET_L1(loop_va); - l2_off = GET_L2(loop_va); - l3_off = GET_L3(loop_va); - mmu_l1 = ((rt_size_t *)mmu_info -> vtable) + l1_off; - - if(PTE_USED(*mmu_l1)) + ret = _map_one_page(aspace, v_addr, p_addr, attr); + if (ret != 0) { - RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1)); - mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1),mmu_info -> pv_off) + l2_off; - - if(PTE_USED(*mmu_l2)) + /* error, undo map */ + while (unmap_va != v_addr) { - RT_ASSERT(!PAGE_IS_LEAF(*mmu_l2)); - mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l2),mmu_info -> pv_off) + l3_off; - - if(PTE_USED(*mmu_l3)) - { - RT_ASSERT(PAGE_IS_LEAF(*mmu_l3)); - return -1; - } + MM_PGTBL_LOCK(aspace); + _unmap_area(aspace, unmap_va, ARCH_PAGE_SIZE); + MM_PGTBL_UNLOCK(aspace); + unmap_va += ARCH_PAGE_SIZE; } + break; } + v_addr += ARCH_PAGE_SIZE; + p_addr += ARCH_PAGE_SIZE; + } - loop_va += PAGE_SIZE; + if (ret == 0) + { + return unmap_va; } - return 0; + return NULL; } -static void __rt_hw_mmu_unmap(rt_mmu_info *mmu_info,void *v_addr,rt_size_t npages) +static void _unmap_pte(rt_size_t *pentry, rt_size_t *lvl_entry[], int level) { - rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr,PAGE_OFFSET_MASK); - rt_size_t l1_off,l2_off,l3_off; - rt_size_t *mmu_l1,*mmu_l2,*mmu_l3; - rt_size_t *ref_cnt; - - RT_ASSERT(mmu_info); - - while(npages--) + int loop_flag = 1; + while (loop_flag) { - l1_off = (rt_size_t)GET_L1(loop_va); - RT_ASSERT((l1_off >= mmu_info -> vstart) && (l1_off <= mmu_info -> vend)); - l2_off = (rt_size_t)GET_L2(loop_va); - l3_off = (rt_size_t)GET_L3(loop_va); - mmu_l1 = ((rt_size_t *)mmu_info -> vtable) + l1_off; - RT_ASSERT(PTE_USED(*mmu_l1)) - RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1)); - mmu_l2 = ((rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1),mmu_info -> pv_off)) + l2_off; - RT_ASSERT(PTE_USED(*mmu_l2)); - RT_ASSERT(!PAGE_IS_LEAF(*mmu_l2)); - mmu_l3 = ((rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l2),mmu_info -> pv_off)) + l3_off; - RT_ASSERT(PTE_USED(*mmu_l3)); - RT_ASSERT(PAGE_IS_LEAF(*(mmu_l3))); - *mmu_l3 = 0; - rt_hw_cpu_dcache_clean(mmu_l3,sizeof(*mmu_l3)); - mmu_l3 -= l3_off; - ref_cnt = mmu_l3 + __SIZE(VPN0_BIT); - (*ref_cnt)--; + loop_flag = 0; + *pentry = 0; + rt_hw_cpu_dcache_clean(pentry, sizeof(*pentry)); - if(!*ref_cnt) + // we don't handle level 0, which is maintained by caller + if (level > 0) { - //release level 3 page - rt_pages_free(mmu_l3,1);//entry page and ref_cnt page - *mmu_l2 = 0; - rt_hw_cpu_dcache_clean(mmu_l2,sizeof(*mmu_l2)); - mmu_l2 -= l2_off; + void *page = (void *)((rt_ubase_t)pentry & ~ARCH_PAGE_MASK); - ref_cnt = mmu_l2 + __SIZE(VPN1_BIT); - (*ref_cnt)--; + // decrease reference from child page to parent + rt_pages_free(page, 0); - if(!*ref_cnt) + int free = rt_page_ref_get(page, 0); + if (free == 1) { - //release level 2 page - rt_pages_free(mmu_l2,1);//entry page and ref_cnt page - *mmu_l1 = 0; - rt_hw_cpu_dcache_clean(mmu_l1,sizeof(*mmu_l1)); + rt_pages_free(page, 0); + pentry = lvl_entry[--level]; + loop_flag = 1; } } - - loop_va += PAGE_SIZE; } } -static int __rt_hw_mmu_map(rt_mmu_info *mmu_info,void *v_addr,void *p_addr,rt_size_t npages,rt_size_t attr) +static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr, size_t size) { - rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr,PAGE_OFFSET_MASK); - rt_size_t loop_pa = __UMASKVALUE((rt_size_t)p_addr,PAGE_OFFSET_MASK); - rt_size_t l1_off,l2_off,l3_off; - rt_size_t *mmu_l1,*mmu_l2,*mmu_l3; - rt_size_t *ref_cnt; - //rt_kprintf("v_addr = 0x%p,p_addr = 0x%p,npages = %lu\n",v_addr,p_addr,npages); + rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr, PAGE_OFFSET_MASK); + size_t unmapped = 0; + + int i = 0; + rt_size_t lvl_off[3]; + rt_size_t *lvl_entry[3]; + lvl_off[0] = (rt_size_t)GET_L1(loop_va); + lvl_off[1] = (rt_size_t)GET_L2(loop_va); + lvl_off[2] = (rt_size_t)GET_L3(loop_va); + unmapped = 1 << (ARCH_PAGE_SHIFT + ARCH_INDEX_WIDTH * 2ul); + + rt_size_t *pentry; + lvl_entry[i] = ((rt_size_t *)aspace->page_table + lvl_off[i]); + pentry = lvl_entry[i]; - if(!mmu_info) + // find leaf page table entry + while (PTE_USED(*pentry) && !PAGE_IS_LEAF(*pentry)) { - return -1; + i += 1; + lvl_entry[i] = ((rt_size_t *)PPN_TO_VPN(GET_PADDR(*pentry), PV_OFFSET) + + lvl_off[i]); + pentry = lvl_entry[i]; + unmapped >>= ARCH_INDEX_WIDTH; } - while(npages--) + // clear PTE & setup its + if (PTE_USED(*pentry)) { - l1_off = GET_L1(loop_va); - l2_off = GET_L2(loop_va); - l3_off = GET_L3(loop_va); - mmu_l1 = ((rt_size_t *)mmu_info -> vtable) + l1_off; - - if(PTE_USED(*mmu_l1)) - { - RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1)); - mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1),mmu_info -> pv_off); - } - else - { - mmu_l2 = (rt_size_t *)rt_pages_alloc(1); + _unmap_pte(pentry, lvl_entry, i); + } - if(mmu_l2) - { - rt_memset(mmu_l2,0,PAGE_SIZE * 2); - rt_hw_cpu_dcache_clean(mmu_l2,PAGE_SIZE * 2); - *mmu_l1 = COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l2,mmu_info -> pv_off),PAGE_DEFAULT_ATTR_NEXT); - rt_hw_cpu_dcache_clean(mmu_l1,sizeof(*mmu_l1)); - } - else - { - __rt_hw_mmu_unmap(mmu_info,v_addr,npages); - return -1; - } - } + return unmapped; +} - if(PTE_USED(*(mmu_l2 + l2_off))) - { - RT_ASSERT(!PAGE_IS_LEAF(*(mmu_l2 + l2_off))); - mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)),mmu_info -> pv_off); - } - else - { - mmu_l3 = (rt_size_t *)rt_pages_alloc(1); +/** unmap is different from map that it can handle multiple pages */ +void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size) +{ + // caller guarantee that v_addr & size are page aligned + if (!aspace->page_table) + { + return; + } + size_t unmapped = 0; - if(mmu_l3) - { - rt_memset(mmu_l3,0,PAGE_SIZE * 2); - rt_hw_cpu_dcache_clean(mmu_l3,PAGE_SIZE * 2); - *(mmu_l2 + l2_off) = COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l3,mmu_info -> pv_off),PAGE_DEFAULT_ATTR_NEXT); - rt_hw_cpu_dcache_clean(mmu_l2,sizeof(*mmu_l2)); - ref_cnt = mmu_l2 + __SIZE(VPN1_BIT); - (*ref_cnt)++; - } - else - { - __rt_hw_mmu_unmap(mmu_info,v_addr,npages); - return -1; - } - } + while (size > 0) + { + MM_PGTBL_LOCK(aspace); + unmapped = _unmap_area(aspace, v_addr, size); + MM_PGTBL_UNLOCK(aspace); - RT_ASSERT(!PTE_USED(*(mmu_l3 + l3_off))); - ref_cnt = mmu_l3 + __SIZE(VPN0_BIT); - (*ref_cnt)++; - *(mmu_l3 + l3_off) = COMBINEPTE((rt_size_t)loop_pa,PAGE_DEFAULT_ATTR_LEAF); - rt_hw_cpu_dcache_clean(mmu_l3 + l3_off,sizeof(*(mmu_l3 + l3_off))); + // when unmapped == 0, region not exist in pgtbl + if (!unmapped || unmapped > size) + break; - loop_va += PAGE_SIZE; - loop_pa += PAGE_SIZE; + size -= unmapped; + v_addr += unmapped; } +} - return 0; +#ifdef RT_USING_SMART +static inline void _init_region(void *vaddr, size_t size) +{ + rt_ioremap_start = vaddr; + rt_ioremap_size = size; + rt_mpr_start = rt_ioremap_start - rt_mpr_size; + rt_kprintf("rt_ioremap_start: %p, rt_mpr_start: %p\n", rt_ioremap_start, rt_mpr_start); +} +#else +static inline void _init_region(void *vaddr, size_t size) +{ + rt_mpr_start = vaddr - rt_mpr_size; } +#endif -void *_rt_hw_mmu_map(rt_mmu_info *mmu_info,void *v_addr,void *p_addr,rt_size_t size,rt_size_t attr) +int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_size_t size, + rt_size_t *vtable, rt_size_t pv_off) { - rt_size_t pa_s,pa_e; - rt_size_t vaddr; - rt_size_t pages; - int ret; + size_t l1_off, va_s, va_e; + rt_base_t level; - if(!size) + if ((!aspace) || (!vtable)) { - return 0; + return -1; } - pa_s = (rt_size_t)p_addr; - pa_e = ((rt_size_t)p_addr) + size - 1; - pa_s = GET_PF_ID(pa_s); - pa_e = GET_PF_ID(pa_e); - pages = pa_e - pa_s + 1; + va_s = (rt_size_t)v_address; + va_e = ((rt_size_t)v_address) + size - 1; - if(v_addr) + if (va_e < va_s) { - vaddr = (rt_size_t)v_addr; - pa_s = (rt_size_t)p_addr; - - if(GET_PF_OFFSET(vaddr) != GET_PF_OFFSET(pa_s)) - { - return 0; - } + return -1; + } - vaddr = __UMASKVALUE(vaddr,PAGE_OFFSET_MASK); + // convert address to PPN2 index + va_s = GET_L1(va_s); + va_e = GET_L1(va_e); - if(check_vaddr(mmu_info,(void *)vaddr,pages) != 0) - { - return 0; - } - } - else + if (va_s == 0) { - vaddr = find_vaddr(mmu_info,pages); + return -1; } - if(vaddr) + // vtable initialization check + for (l1_off = va_s; l1_off <= va_e; l1_off++) { - ret = __rt_hw_mmu_map(mmu_info,(void *)vaddr,p_addr,pages,attr); + size_t v = vtable[l1_off]; - if(ret == 0) + if (v) { - rt_hw_cpu_tlb_invalidate(); - return (void *)(vaddr | GET_PF_OFFSET((rt_size_t)p_addr)); + return -1; } } + rt_aspace_init(&rt_kernel_space, (void *)0x1000, USER_VADDR_START - 0x1000, + vtable); + + _init_region(v_address, size); return 0; } -static int __rt_hw_mmu_map_auto(rt_mmu_info *mmu_info,void *v_addr,rt_size_t npages,rt_size_t attr) +const static int max_level = + (ARCH_VADDR_WIDTH - ARCH_PAGE_SHIFT) / ARCH_INDEX_WIDTH; + +static inline uintptr_t _get_level_size(int level) +{ + return 1ul << (ARCH_PAGE_SHIFT + (max_level - level) * ARCH_INDEX_WIDTH); +} + +static rt_size_t *_query(struct rt_aspace *aspace, void *vaddr, int *level) { - rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr,PAGE_OFFSET_MASK); - rt_size_t loop_pa; - rt_size_t l1_off,l2_off,l3_off; - rt_size_t *mmu_l1,*mmu_l2,*mmu_l3; - rt_size_t *ref_cnt; - rt_size_t i; - void *va,*pa; + rt_size_t l1_off, l2_off, l3_off; + rt_size_t *mmu_l1, *mmu_l2, *mmu_l3; + rt_size_t pa; - if(!mmu_info) - { - return -1; - } + l1_off = GET_L1((rt_size_t)vaddr); + l2_off = GET_L2((rt_size_t)vaddr); + l3_off = GET_L3((rt_size_t)vaddr); - while(npages--) + if (!aspace) { - loop_pa = (rt_size_t)rt_pages_alloc(0); + LOG_W("%s: no aspace", __func__); + return RT_NULL; + } - if(!loop_pa) - { - goto err; - } + mmu_l1 = ((rt_size_t *)aspace->page_table) + l1_off; - if(__rt_hw_mmu_map(mmu_info,(void *)loop_va,(void *)loop_pa,1,attr) < 0) + if (PTE_USED(*mmu_l1)) + { + if (*mmu_l1 & PTE_XWR_MASK) { - goto err; + *level = 1; + return mmu_l1; } - loop_va += PAGE_SIZE; - } + mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET); - return 0; - - err: - va = (void *)__UMASKVALUE((rt_size_t)v_addr,PAGE_OFFSET_MASK); - - for(i = 0;i < npages;i++) + if (PTE_USED(*(mmu_l2 + l2_off))) { - pa = rt_hw_mmu_v2p(mmu_info,va); - - if(pa) + if (*(mmu_l2 + l2_off) & PTE_XWR_MASK) { - rt_pages_free((void *)PPN_TO_VPN(pa,mmu_info -> pv_off),0); + *level = 2; + return mmu_l2 + l2_off; } - va = (void *)((rt_uint8_t *)va + PAGE_SIZE); + mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), + PV_OFFSET); + + if (PTE_USED(*(mmu_l3 + l3_off))) + { + *level = 3; + return mmu_l3 + l3_off; + } } + } - __rt_hw_mmu_unmap(mmu_info,v_addr,npages); - return -1; + return RT_NULL; } -void *_rt_hw_mmu_map_auto(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size,rt_size_t attr) +void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr) { - rt_size_t vaddr; - rt_size_t offset; - rt_size_t pages; - int ret; - - if(!size) - { - return 0; - } + int level; + uintptr_t *pte = _query(aspace, vaddr, &level); + uintptr_t paddr; - offset = GET_PF_OFFSET((rt_size_t)v_addr); - size += (offset + PAGE_SIZE - 1); - pages = size >> PAGE_OFFSET_BIT; - - if(v_addr) + if (pte) { - vaddr = __UMASKVALUE((rt_size_t)v_addr,PAGE_OFFSET_MASK); - - if(check_vaddr(mmu_info,(void *)vaddr,pages) != 0) - { - return 0; - } + paddr = GET_PADDR(*pte); + paddr |= ((intptr_t)vaddr & (_get_level_size(level) - 1)); } else { - vaddr = find_vaddr(mmu_info,pages); - } - - if(vaddr) - { - ret = __rt_hw_mmu_map_auto(mmu_info,(void *)vaddr,pages,attr); - - if(ret == 0) - { - rt_hw_cpu_tlb_invalidate(); - return (void *)(vaddr | offset); - } + paddr = 0; } - - return 0; + return (void *)paddr; } -void _rt_hw_mmu_unmap(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size) +static int _noncache(uintptr_t *pte) { - rt_size_t va_s,va_e; - rt_size_t pages; - - va_s = ((rt_size_t)v_addr) >> PAGE_OFFSET_BIT; - va_e = (((rt_size_t)v_addr) + size - 1) >> PAGE_OFFSET_BIT; - pages = va_e - va_s + 1; - __rt_hw_mmu_unmap(mmu_info,v_addr,pages); - rt_hw_cpu_tlb_invalidate(); + return 0; } -void *rt_hw_mmu_map(rt_mmu_info *mmu_info,void *v_addr,void *p_addr,rt_size_t size,rt_size_t attr) +static int _cache(uintptr_t *pte) { - void *ret; - rt_base_t level; - - level = rt_hw_interrupt_disable(); - ret = _rt_hw_mmu_map(mmu_info,v_addr,p_addr,size,attr); - rt_hw_interrupt_enable(level); - return ret; + return 0; } -void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size,rt_size_t attr) -{ - void *ret; - rt_base_t level; - - level = rt_hw_interrupt_disable(); - ret = _rt_hw_mmu_map_auto(mmu_info,v_addr,size,attr); - rt_hw_interrupt_enable(level); - return ret; -} +static int (*control_handler[MMU_CNTL_DUMMY_END])(uintptr_t *pte) = { + [MMU_CNTL_CACHE] = _cache, + [MMU_CNTL_NONCACHE] = _noncache, +}; -void rt_hw_mmu_unmap(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size) +int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, + enum rt_mmu_cntl cmd) { - rt_base_t level; - - level = rt_hw_interrupt_disable(); - _rt_hw_mmu_unmap(mmu_info,v_addr,size); - rt_hw_interrupt_enable(level); -} + int level; + int err = -RT_EINVAL; + void *vend = vaddr + size; -void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info,void *v_addr) -{ - rt_size_t l1_off,l2_off,l3_off; - rt_size_t *mmu_l1,*mmu_l2,*mmu_l3; - rt_size_t pa; + int (*handler)(uintptr_t * pte); + if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END) + { + handler = control_handler[cmd]; - l1_off = GET_L1((rt_size_t)v_addr); - l2_off = GET_L2((rt_size_t)v_addr); - l3_off = GET_L3((rt_size_t)v_addr); + while (vaddr < vend) + { + uintptr_t *pte = _query(aspace, vaddr, &level); + void *range_end = vaddr + _get_level_size(level); + RT_ASSERT(range_end < vend); - if(!mmu_info) + if (pte) + { + err = handler(pte); + RT_ASSERT(err == RT_EOK); + } + vaddr = range_end; + } + } + else { - return RT_NULL; + err = -RT_ENOSYS; } - mmu_l1 = ((rt_size_t *)mmu_info -> vtable) + l1_off; + return err; +} - if(PTE_USED(*mmu_l1)) +/** + * @brief setup Page Table for kernel space. It's a fixed map + * and all mappings cannot be changed after initialization. + * + * Memory region in struct mem_desc must be page aligned, + * otherwise is a failure and no report will be + * returned. + * + * @param mmu_info + * @param mdesc + * @param desc_nr + */ +void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr) +{ + void *err; + for (size_t i = 0; i < desc_nr; i++) { - RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1)); - mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1),mmu_info -> pv_off); - - if(PTE_USED(*(mmu_l2 + l2_off))) + size_t attr; + switch (mdesc->attr) { - RT_ASSERT(!PAGE_IS_LEAF(*(mmu_l2 + l2_off))); - mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)),mmu_info -> pv_off); - - if(PTE_USED(*(mmu_l3 + l3_off))) - { - RT_ASSERT(PAGE_IS_LEAF(*(mmu_l3 + l3_off))); - return (void *)(GET_PADDR(*(mmu_l3 + l3_off)) | GET_PF_OFFSET((rt_size_t)v_addr)); - } + case NORMAL_MEM: + attr = MMU_MAP_K_RWCB; + break; + case NORMAL_NOCACHE_MEM: + attr = MMU_MAP_K_RWCB; + break; + case DEVICE_MEM: + attr = MMU_MAP_K_DEVICE; + break; + default: + attr = MMU_MAP_K_DEVICE; } + + struct rt_mm_va_hint hint = {.flags = MMF_MAP_FIXED, + .limit_start = aspace->start, + .limit_range_size = aspace->size, + .map_size = mdesc->vaddr_end - + mdesc->vaddr_start + 1, + .prefer = (void *)mdesc->vaddr_start}; + + rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr, + mdesc->paddr_start >> MM_PAGE_SHIFT, &err); + mdesc++; } - return RT_NULL; + rt_hw_aspace_switch(&rt_kernel_space); + rt_page_cleanup(); } -void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info,void *v_addr) +void rt_hw_mmu_kernel_map_init(rt_aspace_t aspace, rt_size_t vaddr_start, rt_size_t size) { - void *ret; - rt_base_t level; + rt_size_t paddr_start = + __UMASKVALUE(VPN_TO_PPN(vaddr_start, PV_OFFSET), PAGE_OFFSET_MASK); + rt_size_t va_s = GET_L1(vaddr_start); + rt_size_t va_e = GET_L1(vaddr_start + size - 1); + rt_size_t i; + + for (i = va_s; i <= va_e; i++) + { + MMUTable[i] = + COMBINEPTE(paddr_start, PAGE_ATTR_RWX | PTE_G | PTE_V | PTE_CACHE | + PTE_SHARE | PTE_BUF | PTE_A | PTE_D); + paddr_start += L1_PAGE_SIZE; + } - level = rt_hw_interrupt_disable(); - ret = _rt_hw_mmu_v2p(mmu_info,v_addr); - rt_hw_interrupt_enable(level); - return ret; + rt_hw_tlb_invalidate_all_local(); } diff --git a/libcpu/risc-v/t-head/c906/mmu.h b/libcpu/risc-v/t-head/c906/mmu.h index 09fdc494e7..bb3eb69cd0 100644 --- a/libcpu/risc-v/t-head/c906/mmu.h +++ b/libcpu/risc-v/t-head/c906/mmu.h @@ -13,45 +13,83 @@ #include "riscv.h" #include "riscv_mmu.h" +#include <mm_aspace.h> +#include <stddef.h> + +/* RAM, Flash, or ROM */ +#define NORMAL_MEM 0 +/* normal nocache memory mapping type */ +#define NORMAL_NOCACHE_MEM 1 +/* MMIO region */ +#define DEVICE_MEM 2 + +typedef size_t rt_pte_t; + struct mem_desc { rt_size_t vaddr_start; rt_size_t vaddr_end; rt_size_t paddr_start; rt_size_t attr; + struct rt_varea varea; }; -#define GET_PF_ID(addr) ((addr) >> PAGE_OFFSET_BIT) -#define GET_PF_OFFSET(addr) __MASKVALUE(addr,PAGE_OFFSET_MASK) -#define GET_L1(addr) __PARTBIT(addr,VPN2_SHIFT,VPN2_BIT) -#define GET_L2(addr) __PARTBIT(addr,VPN1_SHIFT,VPN1_BIT) -#define GET_L3(addr) __PARTBIT(addr,VPN0_SHIFT,VPN0_BIT) -#define GET_PPN(pte) (__PARTBIT(pte,PTE_PPN_SHIFT,PHYSICAL_ADDRESS_WIDTH_BITS - PTE_PPN_SHIFT)) -#define GET_PADDR(pte) (GET_PPN(pte) << PAGE_OFFSET_BIT) -#define VPN_TO_PPN(vaddr,pv_off) (((rt_size_t)(vaddr)) + (pv_off)) -#define PPN_TO_VPN(paddr,pv_off) (((rt_size_t)(paddr)) - (pv_off)) -#define COMBINEVADDR(l1_off,l2_off,l3_off) (((l1_off) << VPN2_SHIFT) | ((l2_off) << VPN1_SHIFT) | ((l3_off) << VPN0_SHIFT)) -#define COMBINEPTE(paddr,attr) ((((paddr) >> PAGE_OFFSET_BIT) << PTE_PPN_SHIFT) | (attr)) +#define GET_PF_ID(addr) ((addr) >> PAGE_OFFSET_BIT) +#define GET_PF_OFFSET(addr) __MASKVALUE(addr, PAGE_OFFSET_MASK) +#define GET_L1(addr) __PARTBIT(addr, VPN2_SHIFT, VPN2_BIT) +#define GET_L2(addr) __PARTBIT(addr, VPN1_SHIFT, VPN1_BIT) +#define GET_L3(addr) __PARTBIT(addr, VPN0_SHIFT, VPN0_BIT) +#define GET_PPN(pte) \ + (__PARTBIT(pte, PTE_PPN_SHIFT, PHYSICAL_ADDRESS_WIDTH_BITS - PTE_PPN_SHIFT)) +#define GET_PADDR(pte) (GET_PPN(pte) << PAGE_OFFSET_BIT) +#define VPN_TO_PPN(vaddr, pv_off) (((rt_size_t)(vaddr)) + (pv_off)) +#define PPN_TO_VPN(paddr, pv_off) (((rt_size_t)(paddr)) - (pv_off)) +#define COMBINEVADDR(l1_off, l2_off, l3_off) \ + (((l1_off) << VPN2_SHIFT) | ((l2_off) << VPN1_SHIFT) | \ + ((l3_off) << VPN0_SHIFT)) +#define COMBINEPTE(paddr, attr) \ + ((((paddr) >> PAGE_OFFSET_BIT) << PTE_PPN_SHIFT) | (attr)) -typedef struct -{ - size_t *vtable; - size_t vstart; - size_t vend; - size_t pv_off; -}rt_mmu_info; +#define ARCH_ADDRESS_WIDTH_BITS 64 + +#define MMU_MAP_ERROR_VANOTALIGN -1 +#define MMU_MAP_ERROR_PANOTALIGN -2 +#define MMU_MAP_ERROR_NOPAGE -3 +#define MMU_MAP_ERROR_CONFLICT -4 void *rt_hw_mmu_tbl_get(); -void rt_hw_mmu_switch(void *mmu_table); -int rt_hw_mmu_map_init(rt_mmu_info *mmu_info,void *v_address,rt_size_t size,rt_size_t *vtable,rt_size_t pv_off); -void rt_hw_mmu_kernel_map_init(rt_mmu_info *mmu_info,rt_size_t vaddr_start,rt_size_t size); -void *_rt_hw_mmu_map(rt_mmu_info *mmu_info,void *v_addr,void *p_addr,rt_size_t size,rt_size_t attr); -void *_rt_hw_mmu_map_auto(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size,rt_size_t attr); -void _rt_hw_mmu_unmap(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size); -void *rt_hw_mmu_map(rt_mmu_info *mmu_info,void *v_addr,void *p_addr,rt_size_t size,rt_size_t attr); -void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size,rt_size_t attr); -void rt_hw_mmu_unmap(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size); -void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info,void *v_addr); -void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info,void *v_addr); +int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_size_t size, + rt_size_t *vtable, rt_size_t pv_off); +void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr); +void rt_hw_mmu_kernel_map_init(rt_aspace_t aspace, rt_size_t vaddr_start, + rt_size_t size); +void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size, + size_t attr); +void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size); +void rt_hw_aspace_switch(rt_aspace_t aspace); +void *rt_hw_mmu_v2p(rt_aspace_t aspace, void *vaddr); + +static inline void *_rt_kmem_v2p(void *vaddr) +{ + return rt_hw_mmu_v2p(&rt_kernel_space, vaddr); +} + +static inline void *rt_kmem_v2p(void *vaddr) +{ + MM_PGTBL_LOCK(&rt_kernel_space); + void *paddr = _rt_kmem_v2p(vaddr); + MM_PGTBL_UNLOCK(&rt_kernel_space); + return paddr; +} + +enum rt_mmu_cntl +{ + MMU_CNTL_NONCACHE, + MMU_CNTL_CACHE, + MMU_CNTL_DUMMY_END, +}; + +int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, + enum rt_mmu_cntl cmd); #endif diff --git a/libcpu/risc-v/t-head/c906/riscv_mmu.c b/libcpu/risc-v/t-head/c906/riscv_mmu.c index 9ba48e34c4..e648c52269 100644 --- a/libcpu/risc-v/t-head/c906/riscv_mmu.c +++ b/libcpu/risc-v/t-head/c906/riscv_mmu.c @@ -18,20 +18,12 @@ #include "riscv_mmu.h" -void mmu_set_pagetable(rt_ubase_t addr) -{ - RT_ASSERT(__CHECKALIGN(addr,PAGE_OFFSET_BIT)); - RT_ASSERT(__CHECKUPBOUND(addr,PHYSICAL_ADDRESS_WIDTH_BITS)); - write_csr(satp,(((size_t)8) << 60) | (addr >> PAGE_OFFSET_BIT)); - mmu_flush_tlb(); -} - void mmu_enable_user_page_access() { - set_csr(sstatus,SSTATUS_PUM); + set_csr(sstatus,SSTATUS_SUM); } void mmu_disable_user_page_access() { - clear_csr(sstatus,SSTATUS_PUM); + clear_csr(sstatus,SSTATUS_SUM); } diff --git a/libcpu/risc-v/t-head/c906/riscv_mmu.h b/libcpu/risc-v/t-head/c906/riscv_mmu.h index 138bd596e9..386bda9884 100644 --- a/libcpu/risc-v/t-head/c906/riscv_mmu.h +++ b/libcpu/risc-v/t-head/c906/riscv_mmu.h @@ -12,8 +12,8 @@ #ifndef __RISCV_MMU_H__ #define __RISCV_MMU_H__ -#include <rthw.h> #include <rtthread.h> +#include <rthw.h> #include "riscv.h" #undef PAGE_SIZE @@ -59,21 +59,42 @@ #define PAGE_ATTR_USER (PTE_U) #define PAGE_ATTR_SYSTEM (0) -#define PAGE_DEFAULT_ATTR_LEAF (PAGE_ATTR_RWX | PAGE_ATTR_USER | PTE_V | PTE_G | PTE_SHARE | PTE_BUF | PTE_CACHE | PTE_A | PTE_D) -#define PAGE_DEFAULT_ATTR_NEXT (PAGE_ATTR_NEXT_LEVEL | PTE_V | PTE_G | PTE_SHARE | PTE_BUF | PTE_CACHE | PTE_A | PTE_D) +#define PAGE_DEFAULT_ATTR_LEAF (PAGE_ATTR_RWX | PAGE_ATTR_USER | PTE_V | PTE_G) +#define PAGE_DEFAULT_ATTR_NEXT (PAGE_ATTR_NEXT_LEVEL | PTE_V | PTE_G) + +#define PAGE_IS_LEAF(pte) __MASKVALUE(pte, PAGE_ATTR_RWX) + +#define PTE_USED(pte) __MASKVALUE(pte, PTE_V) + +/** + * encoding of SATP (Supervisor Address Translation and Protection register) + */ +#define SATP_MODE_OFFSET 60 +#define SATP_MODE_BARE 0 +#define SATP_MODE_SV39 8 +#define SATP_MODE_SV48 9 +#define SATP_MODE_SV57 10 +#define SATP_MODE_SV64 11 + +#define ARCH_VADDR_WIDTH 39 +#define SATP_MODE SATP_MODE_SV39 -#define PAGE_IS_LEAF(pte) __MASKVALUE(pte,PAGE_ATTR_RWX) +#define MMU_MAP_K_DEVICE (PTE_G | PTE_W | PTE_R | PTE_V) +#define MMU_MAP_K_RWCB (PTE_G | PTE_X | PTE_W | PTE_R | PTE_V) +#define MMU_MAP_U_RWCB (PTE_U | PTE_X | PTE_W | PTE_R | PTE_V) +#define MMU_MAP_U_RWCB_XN (PTE_U | PTE_W | PTE_R | PTE_V) +#define MMU_MAP_U_RW (PTE_U | PTE_X | PTE_W | PTE_R | PTE_V) -#define PTE_USED(pte) __MASKVALUE(pte,PTE_V) +#define PTE_XWR_MASK 0xe -#define mmu_flush_tlb() do{asm volatile("sfence.vma x0,x0");}while(0) +#define ARCH_PAGE_SIZE PAGE_SIZE +#define ARCH_PAGE_MASK (ARCH_PAGE_SIZE - 1) +#define ARCH_PAGE_SHIFT PAGE_OFFSET_BIT +#define ARCH_INDEX_WIDTH 9 +#define ARCH_INDEX_SIZE (1ul << ARCH_INDEX_WIDTH) +#define ARCH_INDEX_MASK (ARCH_INDEX_SIZE - 1) -//compatible to rt-smart new version -#define MMU_MAP_K_DEVICE (PAGE_ATTR_RWX | PTE_V | PTE_G | PTE_SO | PTE_BUF | PTE_A | PTE_D) -#define MMU_MAP_K_RWCB (PAGE_ATTR_RWX | PTE_V | PTE_G | PTE_SHARE | PTE_BUF | PTE_CACHE | PTE_A | PTE_D) -#define ARCH_PAGE_SIZE PAGE_SIZE -#define ARCH_PAGE_MASK (ARCH_PAGE_SIZE - 1) -#define ARCH_PAGE_SHIFT PAGE_OFFSET_BIT +#define ARCH_MAP_FAILED ((void *)0x8000000000000000) void mmu_set_pagetable(rt_ubase_t addr); void mmu_enable_user_page_access(); diff --git a/libcpu/risc-v/t-head/c906/sbi.c b/libcpu/risc-v/t-head/c906/sbi.c index f848ec7810..e4577e8f05 100644 --- a/libcpu/risc-v/t-head/c906/sbi.c +++ b/libcpu/risc-v/t-head/c906/sbi.c @@ -36,12 +36,12 @@ */ #include "sbi.h" -#include <stdbool.h> #include <rtthread.h> +#include <stdbool.h> /* SBI Implementation-Specific Definitions */ -#define OPENSBI_VERSION_MAJOR_OFFSET 16 -#define OPENSBI_VERSION_MINOR_MASK 0xFFFF +#define OPENSBI_VERSION_MAJOR_OFFSET 16 +#define OPENSBI_VERSION_MINOR_MASK 0xFFFF unsigned long sbi_spec_version; unsigned long sbi_impl_id; @@ -51,26 +51,22 @@ static bool has_time_extension = false; static bool has_ipi_extension = false; static bool has_rfnc_extension = false; -static struct sbi_ret -sbi_get_spec_version(void) +static struct sbi_ret sbi_get_spec_version(void) { return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_SPEC_VERSION)); } -static struct sbi_ret -sbi_get_impl_id(void) +static struct sbi_ret sbi_get_impl_id(void) { return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_IMPL_ID)); } -static struct sbi_ret -sbi_get_impl_version(void) +static struct sbi_ret sbi_get_impl_version(void) { return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_IMPL_VERSION)); } -void -sbi_print_version(void) +void sbi_print_version(void) { u_int major; u_int minor; @@ -89,7 +85,8 @@ sbi_print_version(void) rt_kprintf("SBI: Berkely Boot Loader %lu\n", sbi_impl_version); break; case (SBI_IMPL_ID_XVISOR): - rt_kprintf("SBI: eXtensible Versatile hypervISOR %lu\n", sbi_impl_version); + rt_kprintf("SBI: eXtensible Versatile hypervISOR %lu\n", + sbi_impl_version); break; case (SBI_IMPL_ID_KVM): rt_kprintf("SBI: Kernel-based Virtual Machine %lu\n", sbi_impl_version); @@ -116,8 +113,7 @@ sbi_print_version(void) rt_kprintf("SBI Specification Version: %u.%u\n", major, minor); } -void -sbi_set_timer(uint64_t val) +void sbi_set_timer(uint64_t val) { struct sbi_ret ret; @@ -133,16 +129,14 @@ sbi_set_timer(uint64_t val) } } -void -sbi_send_ipi(const unsigned long *hart_mask) +void sbi_send_ipi(const unsigned long *hart_mask) { struct sbi_ret ret; /* Use the IPI legacy replacement extension, if available. */ if (has_ipi_extension) { - ret = SBI_CALL2(SBI_EXT_ID_IPI, SBI_IPI_SEND_IPI, - *hart_mask, 0); + ret = SBI_CALL2(SBI_EXT_ID_IPI, SBI_IPI_SEND_IPI, *hart_mask, 0); RT_ASSERT(ret.error == SBI_SUCCESS); } else @@ -151,16 +145,15 @@ sbi_send_ipi(const unsigned long *hart_mask) } } -void -sbi_remote_fence_i(const unsigned long *hart_mask) +void sbi_remote_fence_i(const unsigned long *hart_mask) { struct sbi_ret ret; /* Use the RFENCE legacy replacement extension, if available. */ if (has_rfnc_extension) { - ret = SBI_CALL2(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_FENCE_I, - *hart_mask, 0); + ret = + SBI_CALL2(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_FENCE_I, *hart_mask, 0); RT_ASSERT(ret.error == SBI_SUCCESS); } else @@ -169,28 +162,29 @@ sbi_remote_fence_i(const unsigned long *hart_mask) } } -void -sbi_remote_sfence_vma(const unsigned long *hart_mask, unsigned long start, unsigned long size) +int sbi_remote_sfence_vma(const unsigned long *hart_mask, + const unsigned long hart_mask_base, + unsigned long start, unsigned long size) { - struct sbi_ret ret; + struct sbi_ret ret = {.error = SBI_SUCCESS}; /* Use the RFENCE legacy replacement extension, if available. */ if (has_rfnc_extension) { - ret = SBI_CALL4(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_SFENCE_VMA, - *hart_mask, 0, start, size); - RT_ASSERT(ret.error == SBI_SUCCESS); + ret = SBI_CALL4(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_SFENCE_VMA, *hart_mask, + hart_mask_base, start, size); } else { - (void)SBI_CALL3(SBI_REMOTE_SFENCE_VMA, 0, (uint64_t)hart_mask, - start, size); + (void)SBI_CALL3(SBI_REMOTE_SFENCE_VMA, 0, (uint64_t)hart_mask, start, + size); } + return ret.error; } -void -sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start, unsigned long size, - unsigned long asid) +void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, + unsigned long start, unsigned long size, + unsigned long asid) { struct sbi_ret ret; @@ -203,13 +197,13 @@ sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start, } else { - (void)SBI_CALL4(SBI_REMOTE_SFENCE_VMA_ASID, 0, - (uint64_t)hart_mask, start, size, asid); + (void)SBI_CALL4(SBI_REMOTE_SFENCE_VMA_ASID, 0, (uint64_t)hart_mask, + start, size, asid); } } -int -sbi_hsm_hart_start(unsigned long hart, unsigned long start_addr, unsigned long priv) +int sbi_hsm_hart_start(unsigned long hart, unsigned long start_addr, + unsigned long priv) { struct sbi_ret ret; @@ -217,14 +211,12 @@ sbi_hsm_hart_start(unsigned long hart, unsigned long start_addr, unsigned long p return (ret.error != 0 ? (int)ret.error : 0); } -void -sbi_hsm_hart_stop(void) +void sbi_hsm_hart_stop(void) { (void)SBI_CALL0(SBI_EXT_ID_HSM, SBI_HSM_HART_STOP); } -int -sbi_hsm_hart_status(unsigned long hart) +int sbi_hsm_hart_status(unsigned long hart) { struct sbi_ret ret; @@ -233,8 +225,7 @@ sbi_hsm_hart_status(unsigned long hart) return (ret.error != 0 ? (int)ret.error : (int)ret.value); } -void -sbi_init(void) +void sbi_init(void) { struct sbi_ret sret; @@ -263,3 +254,11 @@ sbi_init(void) if (sbi_probe_extension(SBI_EXT_ID_RFNC) != 0) has_rfnc_extension = true; } + +void rt_hw_console_output(const char *str) +{ + while (*str) + { + sbi_console_putchar(*str++); + } +} diff --git a/libcpu/risc-v/t-head/c906/sbi.h b/libcpu/risc-v/t-head/c906/sbi.h index 0e95ede334..ed29ecfd20 100644 --- a/libcpu/risc-v/t-head/c906/sbi.h +++ b/libcpu/risc-v/t-head/c906/sbi.h @@ -140,7 +140,7 @@ struct sbi_ret long value; }; -static __inline struct sbi_ret +static inline struct sbi_ret sbi_call(uint64_t arg7, uint64_t arg6, uint64_t arg0, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4) { @@ -184,7 +184,9 @@ void sbi_send_ipi(const unsigned long *hart_mask); /* RFENCE extension functions. */ void sbi_remote_fence_i(const unsigned long *hart_mask); -void sbi_remote_sfence_vma(const unsigned long *hart_mask, unsigned long start, unsigned long size); +int sbi_remote_sfence_vma(const unsigned long *hart_mask, + const unsigned long hart_mask_base, + unsigned long start, unsigned long size); void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start, unsigned long size, unsigned long asid); diff --git a/libcpu/risc-v/t-head/c906/tlb.h b/libcpu/risc-v/t-head/c906/tlb.h new file mode 100644 index 0000000000..5bdf18c189 --- /dev/null +++ b/libcpu/risc-v/t-head/c906/tlb.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2006-2022, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-11-28 WangXiaoyao the first version + */ +#ifndef __TLB_H__ +#define __TLB_H__ + +#include "mm_aspace.h" +#include "riscv_mmu.h" +#include "rtdbg.h" +#include "rtthread.h" +#include <sbi.h> +#include <stddef.h> +#include <stdint.h> + +#define HANDLE_FAULT(ret) \ + if (__builtin_expect((ret) != SBI_SUCCESS, 0)) \ + LOG_W("%s failed", __FUNCTION__); + +static inline void rt_hw_tlb_invalidate_all(void) +{ + uintptr_t mask = -1ul; + HANDLE_FAULT(sbi_remote_sfence_vma(&mask, -1ul, 0, mask)); +} + +static inline void rt_hw_tlb_invalidate_all_local(void) +{ + __asm__ volatile("sfence.vma" ::: "memory"); +} + +static inline void rt_hw_tlb_invalidate_aspace(rt_aspace_t aspace) +{ + // TODO ASID + rt_hw_tlb_invalidate_all_local(); +} + +static inline void rt_hw_tlb_invalidate_page(rt_aspace_t aspace, void *start) +{ + __asm__ volatile("sfence.vma %0, zero" ::"r"(start) : "memory"); +} + +static inline void rt_hw_tlb_invalidate_range(rt_aspace_t aspace, void *start, + size_t size, size_t stride) +{ + // huge page is taking as normal page + if (size <= ARCH_PAGE_SIZE) + { + rt_hw_tlb_invalidate_page(aspace, start); + } + else + { + rt_hw_tlb_invalidate_aspace(aspace); + } +} + +#endif /* __TLB_H__ */ diff --git a/libcpu/risc-v/t-head/c906/trap.c b/libcpu/risc-v/t-head/c906/trap.c index beca180831..541d24377b 100644 --- a/libcpu/risc-v/t-head/c906/trap.c +++ b/libcpu/risc-v/t-head/c906/trap.c @@ -107,7 +107,7 @@ void dump_regs(struct rt_hw_stack_frame *regs) rt_kprintf("\t%s\n",(regs->sstatus & SSTATUS_SIE) ? "Supervisor Interrupt Enabled" : "Supervisor Interrupt Disabled"); rt_kprintf("\t%s\n",(regs->sstatus & SSTATUS_SPIE) ? "Last Time Supervisor Interrupt Enabled" : "Last Time Supervisor Interrupt Disabled"); rt_kprintf("\t%s\n",(regs->sstatus & SSTATUS_SPP) ? "Last Privilege is Supervisor Mode" : "Last Privilege is User Mode"); - rt_kprintf("\t%s\n",(regs->sstatus & SSTATUS_PUM) ? "Permit to Access User Page" : "Not Permit to Access User Page"); + rt_kprintf("\t%s\n",(regs->sstatus & SSTATUS_SUM) ? "Permit to Access User Page" : "Not Permit to Access User Page"); rt_kprintf("\t%s\n",(regs->sstatus & (1 << 19)) ? "Permit to Read Executable-only Page" : "Not Permit to Read Executable-only Page"); rt_size_t satp_v = read_csr(satp); rt_kprintf("satp = 0x%p\n",satp_v); @@ -202,6 +202,45 @@ void generic_handle_irq(int irq) plic_complete(irq); } +static const char *get_exception_msg(int id) +{ + const char *msg; + if (id < sizeof(Exception_Name) / sizeof(const char *)) + { + msg = Exception_Name[id]; + } + else + { + msg = "Unknown Exception"; + } + return msg; +} + +void handle_user(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw_stack_frame *sp) +{ + rt_size_t id = __MASKVALUE(scause, __MASK(63UL)); + +#ifdef RT_USING_USERSPACE + /* user page fault */ + if (id == EP_LOAD_PAGE_FAULT || + id == EP_STORE_PAGE_FAULT) + { + if (arch_expand_user_stack((void *)stval)) + { + return; + } + } +#endif + LOG_E("[FATAL ERROR] Exception %ld:%s", id, get_exception_msg(id)); + LOG_E("scause:0x%p,stval:0x%p,sepc:0x%p", scause, stval, sepc); + dump_regs(sp); + + rt_hw_backtrace((uint32_t *)sp->s0_fp, sepc); + + LOG_E("User Fault, killing thread: %s", rt_thread_self()->name); + sys_exit(-1); +} + /* Trap entry */ void handle_trap(rt_size_t scause,rt_size_t stval,rt_size_t sepc,struct rt_hw_stack_frame *sp) { @@ -238,28 +277,28 @@ void handle_trap(rt_size_t scause,rt_size_t stval,rt_size_t sepc,struct rt_hw_st } else { -#ifdef RT_USING_SMART - /* page fault */ - if (id == EP_LOAD_PAGE_FAULT || - id == EP_STORE_PAGE_FAULT) - { - arch_expand_user_stack((void *)stval); - return; - } -#endif - if(id < sizeof(Exception_Name) / sizeof(const char *)) - { - msg = Exception_Name[id]; - } - else + if (!(sp->sstatus & 0x100)) { - msg = "Unknown Exception"; + handle_user(scause, stval, sepc, sp); + // after handle_user(), return to user space. + // otherwise it never returns + return ; } - rt_kprintf("Unhandled Exception %ld:%s\n",id,msg); + // handle kernel exception: + rt_kprintf("Unhandled Exception %ld:%s\n", id, get_exception_msg(id)); } - rt_kprintf("scause:0x%p,stval:0x%p,sepc:0x%p\n",scause,stval,sepc); + rt_kprintf("scause:0x%p,stval:0x%p,sepc:0x%p\n", scause, stval, sepc); dump_regs(sp); - while(1); + rt_kprintf("--------------Thread list--------------\n"); + rt_kprintf("current thread: %s\n", rt_thread_self()->name); + list_process(); + + extern struct rt_thread *rt_current_thread; + rt_kprintf("--------------Backtrace--------------\n"); + rt_hw_backtrace((uint32_t *)sp->s0_fp, sepc); + + while (1) + ; } diff --git a/libcpu/risc-v/virt64/backtrace.c b/libcpu/risc-v/virt64/backtrace.c index 11651daa16..ad5dd7d2d3 100644 --- a/libcpu/risc-v/virt64/backtrace.c +++ b/libcpu/risc-v/virt64/backtrace.c @@ -16,11 +16,10 @@ extern rt_ubase_t __text_start[]; extern rt_ubase_t __text_end[]; -static char *_get_elf_name(); +static char *_get_elf_name(size_t sepc); void rt_hw_backtrace(rt_uint32_t *ffp, rt_ubase_t sepc) { - extern rt_mmu_info mmu_info; rt_ubase_t *ra; rt_ubase_t *fp; rt_ubase_t vas, vae; @@ -60,13 +59,13 @@ void rt_hw_backtrace(rt_uint32_t *ffp, rt_ubase_t sepc) } ra = fp - 1; - if (!_rt_hw_mmu_v2p(&mmu_info, ra) || *ra < vas || *ra > vae) + if (!_rt_kmem_v2p(ra) || *ra < vas || *ra > vae) break; rt_kprintf(" %p", *ra - 0x04); fp = fp - 2; - if (!_rt_hw_mmu_v2p(&mmu_info, fp)) + if (!_rt_kmem_v2p(fp)) break; fp = (rt_ubase_t *)(*fp); if (!fp) diff --git a/libcpu/risc-v/virt64/context_gcc.S b/libcpu/risc-v/virt64/context_gcc.S index c1fe828822..0f3f2e08a5 100644 --- a/libcpu/risc-v/virt64/context_gcc.S +++ b/libcpu/risc-v/virt64/context_gcc.S @@ -80,7 +80,7 @@ rt_hw_context_switch_to: #ifdef RT_USING_SMART mv a0, s1 - jal lwp_mmu_switch + jal lwp_aspace_switch #endif RESTORE_CONTEXT @@ -108,7 +108,7 @@ rt_hw_context_switch: #ifdef RT_USING_SMART mv a0, s1 - jal lwp_mmu_switch + jal lwp_aspace_switch #endif RESTORE_CONTEXT diff --git a/libcpu/risc-v/virt64/mmu.c b/libcpu/risc-v/virt64/mmu.c index 3a9daee3dc..17760c8779 100644 --- a/libcpu/risc-v/virt64/mmu.c +++ b/libcpu/risc-v/virt64/mmu.c @@ -6,282 +6,72 @@ * Change Logs: * Date Author Notes * 2021-01-30 lizhirui first version + * 2022-12-13 WangXiaoyao Port to new mm */ #include <rtthread.h> -#include <rthw.h> +#include <stddef.h> +#include <stdint.h> -#ifdef RT_USING_SMART - -#include <board.h> -#include <page.h> -#include <stdlib.h> -#include <string.h> -#include <lwp_mm.h> #include <cache.h> +#include <mm_aspace.h> +#include <mm_page.h> +#include <mmu.h> +#include <riscv_mmu.h> +#include <tlb.h> -#define DBG_TAG "mmu" -#define DBG_LVL DBG_INFO -#include <rtdbg.h> - -#include "riscv.h" -#include "riscv_mmu.h" -#include "mmu.h" - -void *current_mmu_table = RT_NULL; - -volatile rt_ubase_t MMUTable[__SIZE(VPN2_BIT)] __attribute__((aligned(4 * 1024))); - -static void rt_hw_cpu_tlb_invalidate() -{ - rt_size_t satpv = read_csr(satp); - write_csr(satp, satpv); - mmu_flush_tlb(); -} - -void *rt_hw_mmu_tbl_get() -{ - return current_mmu_table; -} - -void rt_hw_mmu_switch(void *mmu_table) -{ - current_mmu_table = mmu_table; - RT_ASSERT(__CHECKALIGN(mmu_table, PAGE_OFFSET_BIT)); - mmu_set_pagetable((rt_ubase_t)mmu_table); - rt_hw_cpu_dcache_clean_all(); - rt_hw_cpu_icache_invalidate_all(); -} - -int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void *v_address, rt_size_t size, rt_size_t *vtable, rt_size_t pv_off) -{ - size_t l1_off, va_s, va_e; - rt_base_t level; - - if ((!mmu_info) || (!vtable)) - { - return -1; - } - - va_s = (rt_size_t)v_address; - va_e = ((rt_size_t)v_address) + size - 1; - - if (va_e < va_s) - { - return -1; - } - - // convert address to PPN2 index - va_s = GET_L1(va_s); - va_e = GET_L1(va_e); - - if (va_s == 0) - { - return -1; - } - - rt_mm_lock(); - - // vtable initialization check - for (l1_off = va_s; l1_off <= va_e; l1_off++) - { - size_t v = vtable[l1_off]; - - if (v) - { - rt_mm_unlock(); - return -1; - } - } - - rt_mm_unlock(); - mmu_info->vtable = vtable; - mmu_info->vstart = va_s; - mmu_info->vend = va_e; - mmu_info->pv_off = pv_off; - - return 0; -} - -void rt_hw_mmu_kernel_map_init(rt_mmu_info *mmu_info, rt_size_t vaddr_start, rt_size_t size) -{ - rt_size_t paddr_start = __UMASKVALUE(VPN_TO_PPN(vaddr_start, mmu_info->pv_off), PAGE_OFFSET_MASK); - rt_size_t va_s = GET_L1(vaddr_start); - rt_size_t va_e = GET_L1(vaddr_start + size - 1); - rt_size_t i; - - for (i = va_s; i <= va_e; i++) - { - mmu_info->vtable[i] = COMBINEPTE(paddr_start, PAGE_ATTR_RWX | PTE_G | PTE_V); - paddr_start += L1_PAGE_SIZE; - } +#ifdef RT_USING_SMART +#include <ioremap.h> +#include <lwp_user_mm.h> +#include <tlb.h> +#endif - rt_hw_cpu_tlb_invalidate(); -} +#define DBG_TAG "hw.mmu" +#define DBG_LVL DBG_LOG +#include <rtdbg.h> -// find a range of free virtual address specified by pages -static size_t find_vaddr(rt_mmu_info *mmu_info, int pages) -{ - size_t loop_pages; - size_t va; - size_t find_va = 0; - int n = 0; - size_t i; +#ifndef RT_USING_SMART +#define PV_OFFSET 0 +#define USER_VADDR_START 0 +#endif - if (!pages || !mmu_info) - { - return 0; - } +static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr, size_t size); - loop_pages = (mmu_info->vend - mmu_info->vstart) ? (mmu_info->vend - mmu_info->vstart) : 1; - loop_pages <<= (ARCH_INDEX_WIDTH * 2); - va = mmu_info->vstart; - va <<= (ARCH_PAGE_SHIFT + ARCH_INDEX_WIDTH * 2); +static void *current_mmu_table = RT_NULL; - for (i = 0; i < loop_pages; i++, va += ARCH_PAGE_SIZE) - { - if (_rt_hw_mmu_v2p(mmu_info, (void *)va)) - { - n = 0; - find_va = 0; - continue; - } - if (!find_va) - { - find_va = va; - } - n++; - if (n >= pages) - { - return find_va; - } - } - return 0; -} +volatile __attribute__((aligned(4 * 1024))) +rt_ubase_t MMUTable[__SIZE(VPN2_BIT)]; -// check whether the range of virtual address are free -static int check_vaddr(rt_mmu_info *mmu_info, void *va, rt_size_t pages) +void rt_hw_aspace_switch(rt_aspace_t aspace) { - rt_size_t loop_va = __UMASKVALUE((rt_size_t)va, PAGE_OFFSET_MASK); - rt_size_t l1_off, l2_off, l3_off; - rt_size_t *mmu_l1, *mmu_l2, *mmu_l3; - - if (!pages) - { - return -1; - } - - if (!mmu_info) - { - return -1; - } - - while (pages--) - { - l1_off = GET_L1(loop_va); - l2_off = GET_L2(loop_va); - l3_off = GET_L3(loop_va); - mmu_l1 = ((rt_size_t *)mmu_info->vtable) + l1_off; - - if (PTE_USED(*mmu_l1)) - { - RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1)); - mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), mmu_info->pv_off) + l2_off; - - if (PTE_USED(*mmu_l2)) - { - RT_ASSERT(!PAGE_IS_LEAF(*mmu_l2)); - mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l2), mmu_info->pv_off) + l3_off; - - if (PTE_USED(*mmu_l3)) - { - RT_ASSERT(PAGE_IS_LEAF(*mmu_l3)); - return -1; - } - } - } + uintptr_t page_table = (uintptr_t)_rt_kmem_v2p(aspace->page_table); + current_mmu_table = aspace->page_table; - loop_va += PAGE_SIZE; - } - - return 0; + write_csr(satp, (((size_t)SATP_MODE) << SATP_MODE_OFFSET) | + ((rt_ubase_t)page_table >> PAGE_OFFSET_BIT)); + rt_hw_tlb_invalidate_all_local(); } -static void __rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, rt_size_t npages) +void *rt_hw_mmu_tbl_get() { - rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr, PAGE_OFFSET_MASK); - rt_size_t l1_off, l2_off, l3_off; - rt_size_t *mmu_l1, *mmu_l2, *mmu_l3; - - RT_ASSERT(mmu_info); - - while (npages--) - { - l1_off = (rt_size_t)GET_L1(loop_va); - RT_ASSERT((l1_off >= mmu_info->vstart) && (l1_off <= mmu_info->vend)); - l2_off = (rt_size_t)GET_L2(loop_va); - l3_off = (rt_size_t)GET_L3(loop_va); - - mmu_l1 = ((rt_size_t *)mmu_info->vtable) + l1_off; - RT_ASSERT(PTE_USED(*mmu_l1)) - RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1)); - mmu_l2 = ((rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), mmu_info->pv_off)) + l2_off; - RT_ASSERT(PTE_USED(*mmu_l2)); - RT_ASSERT(!PAGE_IS_LEAF(*mmu_l2)); - mmu_l3 = ((rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l2), mmu_info->pv_off)) + l3_off; - RT_ASSERT(PTE_USED(*mmu_l3)); - RT_ASSERT(PAGE_IS_LEAF(*(mmu_l3))); - - *mmu_l3 = 0; - rt_hw_cpu_dcache_clean(mmu_l3, sizeof(*mmu_l3)); - - // decrease reference from leaf page to l3 page - mmu_l3 -= l3_off; - rt_pages_free(mmu_l3, 0); - int free = rt_page_ref_get(mmu_l3, 0); - - if (free == 1) - { - // free l3 page - rt_pages_free(mmu_l3, 0); - - *mmu_l2 = 0; - rt_hw_cpu_dcache_clean(mmu_l2, sizeof(*mmu_l2)); - - // decrease reference from l3 page to l2 page - mmu_l2 -= l2_off; - rt_pages_free(mmu_l2, 0); - - free = rt_page_ref_get(mmu_l2, 0); - if (free == 1) - { - // free l3 page - rt_pages_free(mmu_l2, 0); - // reset PTE in l1 - *mmu_l1 = 0; - rt_hw_cpu_dcache_clean(mmu_l1, sizeof(*mmu_l1)); - } - } - - loop_va += PAGE_SIZE; - } + return current_mmu_table; } -static int _mmu_map_one_page(rt_mmu_info *mmu_info, size_t va, size_t pa, size_t attr) +static int _map_one_page(struct rt_aspace *aspace, void *va, void *pa, + size_t attr) { rt_size_t l1_off, l2_off, l3_off; rt_size_t *mmu_l1, *mmu_l2, *mmu_l3; - l1_off = GET_L1(va); - l2_off = GET_L2(va); - l3_off = GET_L3(va); + l1_off = GET_L1((size_t)va); + l2_off = GET_L2((size_t)va); + l3_off = GET_L3((size_t)va); - mmu_l1 = ((rt_size_t *)mmu_info->vtable) + l1_off; + mmu_l1 = ((rt_size_t *)aspace->page_table) + l1_off; if (PTE_USED(*mmu_l1)) { - RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1)); - mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), mmu_info->pv_off); + mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET); } else { @@ -291,7 +81,8 @@ static int _mmu_map_one_page(rt_mmu_info *mmu_info, size_t va, size_t pa, size_t { rt_memset(mmu_l2, 0, PAGE_SIZE); rt_hw_cpu_dcache_clean(mmu_l2, PAGE_SIZE); - *mmu_l1 = COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l2, mmu_info->pv_off), PAGE_DEFAULT_ATTR_NEXT); + *mmu_l1 = COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l2, PV_OFFSET), + PAGE_DEFAULT_ATTR_NEXT); rt_hw_cpu_dcache_clean(mmu_l1, sizeof(*mmu_l1)); } else @@ -303,7 +94,8 @@ static int _mmu_map_one_page(rt_mmu_info *mmu_info, size_t va, size_t pa, size_t if (PTE_USED(*(mmu_l2 + l2_off))) { RT_ASSERT(!PAGE_IS_LEAF(*(mmu_l2 + l2_off))); - mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), mmu_info->pv_off); + mmu_l3 = + (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), PV_OFFSET); } else { @@ -313,7 +105,9 @@ static int _mmu_map_one_page(rt_mmu_info *mmu_info, size_t va, size_t pa, size_t { rt_memset(mmu_l3, 0, PAGE_SIZE); rt_hw_cpu_dcache_clean(mmu_l3, PAGE_SIZE); - *(mmu_l2 + l2_off) = COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l3, mmu_info->pv_off), PAGE_DEFAULT_ATTR_NEXT); + *(mmu_l2 + l2_off) = + COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l3, PV_OFFSET), + PAGE_DEFAULT_ATTR_NEXT); rt_hw_cpu_dcache_clean(mmu_l2, sizeof(*mmu_l2)); // declares a reference to parent page table rt_page_ref_inc((void *)mmu_l2, 0); @@ -332,259 +126,242 @@ static int _mmu_map_one_page(rt_mmu_info *mmu_info, size_t va, size_t pa, size_t return 0; } -static int __rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, rt_size_t npages, rt_size_t attr) +/** rt_hw_mmu_map will never override existed page table entry */ +void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr, + size_t size, size_t attr) { - rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr, PAGE_OFFSET_MASK); - rt_size_t loop_pa = __UMASKVALUE((rt_size_t)p_addr, PAGE_OFFSET_MASK); - - if (!mmu_info) - { - return -1; - } + int ret = -1; + void *unmap_va = v_addr; + size_t npages = size >> ARCH_PAGE_SHIFT; + // TODO trying with HUGEPAGE here while (npages--) { - if (_mmu_map_one_page(mmu_info, loop_va, loop_pa, attr) != 0) + ret = _map_one_page(aspace, v_addr, p_addr, attr); + if (ret != 0) { - __rt_hw_mmu_unmap(mmu_info, v_addr, npages); - return -1; + /* error, undo map */ + while (unmap_va != v_addr) + { + MM_PGTBL_LOCK(aspace); + _unmap_area(aspace, unmap_va, ARCH_PAGE_SIZE); + MM_PGTBL_UNLOCK(aspace); + unmap_va += ARCH_PAGE_SIZE; + } + break; } - - loop_va += PAGE_SIZE; - loop_pa += PAGE_SIZE; + v_addr += ARCH_PAGE_SIZE; + p_addr += ARCH_PAGE_SIZE; } - return 0; -} - -void *_rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, rt_size_t size, rt_size_t attr) -{ - rt_size_t pa_s, pa_e; - rt_size_t vaddr; - rt_size_t pages; - int ret; - - if (!size) + if (ret == 0) { - return 0; + return unmap_va; } - pa_s = (rt_size_t)p_addr; - pa_e = ((rt_size_t)p_addr) + size - 1; - pa_s = GET_PF_ID(pa_s); - pa_e = GET_PF_ID(pa_e); - pages = pa_e - pa_s + 1; + return NULL; +} - if (v_addr) +static void _unmap_pte(rt_size_t *pentry, rt_size_t *lvl_entry[], int level) +{ + int loop_flag = 1; + while (loop_flag) { - vaddr = (rt_size_t)v_addr; - pa_s = (rt_size_t)p_addr; + loop_flag = 0; + *pentry = 0; + rt_hw_cpu_dcache_clean(pentry, sizeof(*pentry)); - if (GET_PF_OFFSET(vaddr) != GET_PF_OFFSET(pa_s)) + // we don't handle level 0, which is maintained by caller + if (level > 0) { - return 0; - } + void *page = (void *)((rt_ubase_t)pentry & ~ARCH_PAGE_MASK); - vaddr = __UMASKVALUE(vaddr, PAGE_OFFSET_MASK); + // decrease reference from child page to parent + rt_pages_free(page, 0); - if (check_vaddr(mmu_info, (void *)vaddr, pages) != 0) - { - return 0; - } - } - else - { - vaddr = find_vaddr(mmu_info, pages); - } - - if (vaddr) - { - ret = __rt_hw_mmu_map(mmu_info, (void *)vaddr, p_addr, pages, attr); - - if (ret == 0) - { - rt_hw_cpu_tlb_invalidate(); - return (void *)(vaddr | GET_PF_OFFSET((rt_size_t)p_addr)); + int free = rt_page_ref_get(page, 0); + if (free == 1) + { + rt_pages_free(page, 0); + pentry = lvl_entry[--level]; + loop_flag = 1; + } } } - - return 0; } -static int __rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, rt_size_t npages, rt_size_t attr) +static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr, size_t size) { rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr, PAGE_OFFSET_MASK); - rt_size_t loop_pa; - rt_size_t l1_off, l2_off, l3_off; - rt_size_t *mmu_l1, *mmu_l2, *mmu_l3; - rt_size_t *ref_cnt; - rt_size_t i; - void *va, *pa; + size_t unmapped = 0; - if (!mmu_info) + int i = 0; + rt_size_t lvl_off[3]; + rt_size_t *lvl_entry[3]; + lvl_off[0] = (rt_size_t)GET_L1(loop_va); + lvl_off[1] = (rt_size_t)GET_L2(loop_va); + lvl_off[2] = (rt_size_t)GET_L3(loop_va); + unmapped = 1 << (ARCH_PAGE_SHIFT + ARCH_INDEX_WIDTH * 2ul); + + rt_size_t *pentry; + lvl_entry[i] = ((rt_size_t *)aspace->page_table + lvl_off[i]); + pentry = lvl_entry[i]; + + // find leaf page table entry + while (PTE_USED(*pentry) && !PAGE_IS_LEAF(*pentry)) { - return -1; + i += 1; + lvl_entry[i] = ((rt_size_t *)PPN_TO_VPN(GET_PADDR(*pentry), PV_OFFSET) + + lvl_off[i]); + pentry = lvl_entry[i]; + unmapped >>= ARCH_INDEX_WIDTH; } - while (npages--) + // clear PTE & setup its + if (PTE_USED(*pentry)) { - loop_pa = (rt_size_t)rt_pages_alloc(0); - - if (!loop_pa) - { - goto err; - } - - if (__rt_hw_mmu_map(mmu_info, (void *)loop_va, (void *)loop_pa, 1, attr) < 0) - { - goto err; - } - - loop_va += PAGE_SIZE; + _unmap_pte(pentry, lvl_entry, i); } - return 0; + return unmapped; +} -err: - va = (void *)__UMASKVALUE((rt_size_t)v_addr, PAGE_OFFSET_MASK); +/** unmap is different from map that it can handle multiple pages */ +void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size) +{ + // caller guarantee that v_addr & size are page aligned + if (!aspace->page_table) + { + return; + } + size_t unmapped = 0; - for (i = 0; i < npages; i++) + while (size > 0) { - pa = rt_hw_mmu_v2p(mmu_info, va); + MM_PGTBL_LOCK(aspace); + unmapped = _unmap_area(aspace, v_addr, size); + MM_PGTBL_UNLOCK(aspace); - if (pa) - { - rt_pages_free((void *)PPN_TO_VPN(pa, mmu_info->pv_off), 0); - } + // when unmapped == 0, region not exist in pgtbl + if (!unmapped || unmapped > size) + break; - va = (void *)((rt_uint8_t *)va + PAGE_SIZE); + size -= unmapped; + v_addr += unmapped; } +} - __rt_hw_mmu_unmap(mmu_info, v_addr, npages); - return -1; +#ifdef RT_USING_SMART +static inline void _init_region(void *vaddr, size_t size) +{ + rt_ioremap_start = vaddr; + rt_ioremap_size = size; + rt_mpr_start = rt_ioremap_start - rt_mpr_size; +} +#else +static inline void _init_region(void *vaddr, size_t size) +{ + rt_mpr_start = vaddr - rt_mpr_size; } +#endif -void *_rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size, rt_size_t attr) +int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_size_t size, + rt_size_t *vtable, rt_size_t pv_off) { - rt_size_t vaddr; - rt_size_t offset; - rt_size_t pages; - int ret; + size_t l1_off, va_s, va_e; + rt_base_t level; - if (!size) + if ((!aspace) || (!vtable)) { - return 0; + return -1; } - offset = GET_PF_OFFSET((rt_size_t)v_addr); - size += (offset + PAGE_SIZE - 1); - pages = size >> PAGE_OFFSET_BIT; + va_s = (rt_size_t)v_address; + va_e = ((rt_size_t)v_address) + size - 1; - if (v_addr) + if (va_e < va_s) { - vaddr = __UMASKVALUE((rt_size_t)v_addr, PAGE_OFFSET_MASK); - - if (check_vaddr(mmu_info, (void *)vaddr, pages) != 0) - { - return 0; - } + return -1; } - else + + // convert address to PPN2 index + va_s = GET_L1(va_s); + va_e = GET_L1(va_e); + + if (va_s == 0) { - vaddr = find_vaddr(mmu_info, pages); + return -1; } - if (vaddr) + // vtable initialization check + for (l1_off = va_s; l1_off <= va_e; l1_off++) { - ret = __rt_hw_mmu_map_auto(mmu_info, (void *)vaddr, pages, attr); + size_t v = vtable[l1_off]; - if (ret == 0) + if (v) { - rt_hw_cpu_tlb_invalidate(); - return (void *)(vaddr | offset); + return -1; } } - return 0; -} - -void _rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size) -{ - rt_size_t va_s, va_e; - rt_size_t pages; - - va_s = ((rt_size_t)v_addr) >> PAGE_OFFSET_BIT; - va_e = (((rt_size_t)v_addr) + size - 1) >> PAGE_OFFSET_BIT; - pages = va_e - va_s + 1; - __rt_hw_mmu_unmap(mmu_info, v_addr, pages); - rt_hw_cpu_tlb_invalidate(); -} - -void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, rt_size_t size, rt_size_t attr) -{ - void *ret; - rt_base_t level; + rt_aspace_init(&rt_kernel_space, (void *)0x1000, USER_VADDR_START - 0x1000, + vtable); - rt_mm_lock(); - ret = _rt_hw_mmu_map(mmu_info, v_addr, p_addr, size, attr); - rt_mm_unlock(); - return ret; + _init_region(v_address, size); + return 0; } -void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size, rt_size_t attr) -{ - void *ret; - rt_base_t level; - - rt_mm_lock(); - ret = _rt_hw_mmu_map_auto(mmu_info, v_addr, size, attr); - rt_mm_unlock(); - return ret; -} +const static int max_level = + (ARCH_VADDR_WIDTH - ARCH_PAGE_SHIFT) / ARCH_INDEX_WIDTH; -void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size) +static inline uintptr_t _get_level_size(int level) { - rt_base_t level; - - rt_mm_lock(); - _rt_hw_mmu_unmap(mmu_info, v_addr, size); - rt_mm_unlock(); + return 1ul << (ARCH_PAGE_SHIFT + (max_level - level) * ARCH_INDEX_WIDTH); } -void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr) +static rt_size_t *_query(struct rt_aspace *aspace, void *vaddr, int *level) { rt_size_t l1_off, l2_off, l3_off; rt_size_t *mmu_l1, *mmu_l2, *mmu_l3; rt_size_t pa; - l1_off = GET_L1((rt_size_t)v_addr); - l2_off = GET_L2((rt_size_t)v_addr); - l3_off = GET_L3((rt_size_t)v_addr); + l1_off = GET_L1((rt_size_t)vaddr); + l2_off = GET_L2((rt_size_t)vaddr); + l3_off = GET_L3((rt_size_t)vaddr); - if (!mmu_info) + if (!aspace) { + LOG_W("%s: no aspace", __func__); return RT_NULL; } - mmu_l1 = ((rt_size_t *)mmu_info->vtable) + l1_off; + mmu_l1 = ((rt_size_t *)aspace->page_table) + l1_off; if (PTE_USED(*mmu_l1)) { if (*mmu_l1 & PTE_XWR_MASK) - return (void *)(GET_PADDR(*mmu_l1) | ((rt_size_t)v_addr & ((1 << 30) - 1))); + { + *level = 1; + return mmu_l1; + } - mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), mmu_info->pv_off); + mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET); if (PTE_USED(*(mmu_l2 + l2_off))) { if (*(mmu_l2 + l2_off) & PTE_XWR_MASK) - return (void *)(GET_PADDR(*(mmu_l2 + l2_off)) | ((rt_size_t)v_addr & ((1 << 21) - 1))); + { + *level = 2; + return mmu_l2 + l2_off; + } - mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), mmu_info->pv_off); + mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), + PV_OFFSET); if (PTE_USED(*(mmu_l3 + l3_off))) { - return (void *)(GET_PADDR(*(mmu_l3 + l3_off)) | GET_PF_OFFSET((rt_size_t)v_addr)); + *level = 3; + return mmu_l3 + l3_off; } } } @@ -592,15 +369,71 @@ void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr) return RT_NULL; } -void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr) +void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr) { - void *ret; - rt_base_t level; + int level; + uintptr_t *pte = _query(aspace, vaddr, &level); + uintptr_t paddr; + + if (pte) + { + paddr = GET_PADDR(*pte); + paddr |= ((intptr_t)vaddr & (_get_level_size(level) - 1)); + } + else + { + paddr = (uintptr_t)ARCH_MAP_FAILED; + } + return (void *)paddr; +} + +static int _noncache(uintptr_t *pte) +{ + return 0; +} + +static int _cache(uintptr_t *pte) +{ + return 0; +} + +static int (*control_handler[MMU_CNTL_DUMMY_END])(uintptr_t *pte) = { + [MMU_CNTL_CACHE] = _cache, + [MMU_CNTL_NONCACHE] = _noncache, +}; + +int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, + enum rt_mmu_cntl cmd) +{ + int level; + int err = -RT_EINVAL; + void *vend = vaddr + size; + + int (*handler)(uintptr_t * pte); + if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END) + { + handler = control_handler[cmd]; + + while (vaddr < vend) + { + uintptr_t *pte = _query(aspace, vaddr, &level); + void *range_end = vaddr + _get_level_size(level); + RT_ASSERT(range_end <= vend); + + if (pte) + { + err = handler(pte); + RT_ASSERT(err == RT_EOK); + } + vaddr = range_end; + } + } + else + { + err = -RT_ENOSYS; + } - rt_mm_lock(); - ret = _rt_hw_mmu_v2p(mmu_info, v_addr); - rt_mm_unlock(); - return ret; + return err; } /** @@ -615,7 +448,7 @@ void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr) * @param mdesc * @param desc_nr */ -void rt_hw_mmu_setup(rt_mmu_info *mmu_info, struct mem_desc *mdesc, int desc_nr) +void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr) { void *err; for (size_t i = 0; i < desc_nr; i++) @@ -623,24 +456,31 @@ void rt_hw_mmu_setup(rt_mmu_info *mmu_info, struct mem_desc *mdesc, int desc_nr) size_t attr; switch (mdesc->attr) { - case NORMAL_MEM: - attr = MMU_MAP_K_RWCB; - break; - case NORMAL_NOCACHE_MEM: - attr = MMU_MAP_K_RWCB; - break; - case DEVICE_MEM: - attr = MMU_MAP_K_DEVICE; - break; - default: - attr = MMU_MAP_K_DEVICE; + case NORMAL_MEM: + attr = MMU_MAP_K_RWCB; + break; + case NORMAL_NOCACHE_MEM: + attr = MMU_MAP_K_RWCB; + break; + case DEVICE_MEM: + attr = MMU_MAP_K_DEVICE; + break; + default: + attr = MMU_MAP_K_DEVICE; } - err = _rt_hw_mmu_map(mmu_info, (void *)mdesc->vaddr_start, (void *)mdesc->paddr_start, - mdesc->vaddr_end - mdesc->vaddr_start + 1, attr); + + struct rt_mm_va_hint hint = {.flags = MMF_MAP_FIXED, + .limit_start = aspace->start, + .limit_range_size = aspace->size, + .map_size = mdesc->vaddr_end - + mdesc->vaddr_start + 1, + .prefer = (void *)mdesc->vaddr_start}; + + rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr, + mdesc->paddr_start >> MM_PAGE_SHIFT, &err); mdesc++; } - rt_hw_mmu_switch((void *)MMUTable); + rt_hw_aspace_switch(&rt_kernel_space); + rt_page_cleanup(); } - -#endif /* RT_USING_SMART */ diff --git a/libcpu/risc-v/virt64/mmu.h b/libcpu/risc-v/virt64/mmu.h index 086ad1edb9..ece5c1ccac 100644 --- a/libcpu/risc-v/virt64/mmu.h +++ b/libcpu/risc-v/virt64/mmu.h @@ -13,13 +13,17 @@ #include "riscv.h" #include "riscv_mmu.h" +#include <mm_aspace.h> +#include <stddef.h> /* RAM, Flash, or ROM */ -#define NORMAL_MEM 0 +#define NORMAL_MEM 0 /* normal nocache memory mapping type */ -#define NORMAL_NOCACHE_MEM 1 +#define NORMAL_NOCACHE_MEM 1 /* MMIO region */ -#define DEVICE_MEM 2 +#define DEVICE_MEM 2 + +typedef size_t rt_pte_t; struct mem_desc { @@ -27,48 +31,58 @@ struct mem_desc rt_size_t vaddr_end; rt_size_t paddr_start; rt_size_t attr; + struct rt_varea varea; }; -#define GET_PF_ID(addr) ((addr) >> PAGE_OFFSET_BIT) +#define GET_PF_ID(addr) ((addr) >> PAGE_OFFSET_BIT) #define GET_PF_OFFSET(addr) __MASKVALUE(addr, PAGE_OFFSET_MASK) -#define GET_L1(addr) __PARTBIT(addr, VPN2_SHIFT, VPN2_BIT) -#define GET_L2(addr) __PARTBIT(addr, VPN1_SHIFT, VPN1_BIT) -#define GET_L3(addr) __PARTBIT(addr, VPN0_SHIFT, VPN0_BIT) -#define GET_PPN(pte) (__PARTBIT(pte, PTE_PPN_SHIFT, PHYSICAL_ADDRESS_WIDTH_BITS - PTE_PPN_SHIFT)) -#define GET_PADDR(pte) (GET_PPN(pte) << PAGE_OFFSET_BIT) +#define GET_L1(addr) __PARTBIT(addr, VPN2_SHIFT, VPN2_BIT) +#define GET_L2(addr) __PARTBIT(addr, VPN1_SHIFT, VPN1_BIT) +#define GET_L3(addr) __PARTBIT(addr, VPN0_SHIFT, VPN0_BIT) +#define GET_PPN(pte) \ + (__PARTBIT(pte, PTE_PPN_SHIFT, PHYSICAL_ADDRESS_WIDTH_BITS - PTE_PPN_SHIFT)) +#define GET_PADDR(pte) (GET_PPN(pte) << PAGE_OFFSET_BIT) #define VPN_TO_PPN(vaddr, pv_off) (((rt_size_t)(vaddr)) + (pv_off)) #define PPN_TO_VPN(paddr, pv_off) (((rt_size_t)(paddr)) - (pv_off)) -#define COMBINEVADDR(l1_off, l2_off, l3_off) (((l1_off) << VPN2_SHIFT) | ((l2_off) << VPN1_SHIFT) | ((l3_off) << VPN0_SHIFT)) -#define COMBINEPTE(paddr, attr) ((((paddr) >> PAGE_OFFSET_BIT) << PTE_PPN_SHIFT) | (attr)) +#define COMBINEVADDR(l1_off, l2_off, l3_off) \ + (((l1_off) << VPN2_SHIFT) | ((l2_off) << VPN1_SHIFT) | \ + ((l3_off) << VPN0_SHIFT)) +#define COMBINEPTE(paddr, attr) \ + ((((paddr) >> PAGE_OFFSET_BIT) << PTE_PPN_SHIFT) | (attr)) -typedef struct -{ - size_t *vtable; - size_t vstart; - size_t vend; - size_t pv_off; -} rt_mmu_info; +#define ARCH_ADDRESS_WIDTH_BITS 64 + +#define MMU_MAP_ERROR_VANOTALIGN -1 +#define MMU_MAP_ERROR_PANOTALIGN -2 +#define MMU_MAP_ERROR_NOPAGE -3 +#define MMU_MAP_ERROR_CONFLICT -4 void *rt_hw_mmu_tbl_get(); -void rt_hw_mmu_switch(void *mmu_table); -int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void *v_address, rt_size_t size, rt_size_t *vtable, rt_size_t pv_off); -void rt_hw_mmu_kernel_map_init(rt_mmu_info *mmu_info, rt_size_t vaddr_start, rt_size_t size); -void *_rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, rt_size_t size, rt_size_t attr); -void *_rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size, rt_size_t attr); -void _rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size); -void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, rt_size_t size, rt_size_t attr); -void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size, rt_size_t attr); -void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size); -void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr); -void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr); +int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_size_t size, + rt_size_t *vtable, rt_size_t pv_off); +void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr); +void rt_hw_mmu_kernel_map_init(rt_aspace_t aspace, rt_size_t vaddr_start, + rt_size_t size); +void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size, + size_t attr); +void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size); +void rt_hw_aspace_switch(rt_aspace_t aspace); +void *rt_hw_mmu_v2p(rt_aspace_t aspace, void *vaddr); -void rt_hw_mmu_setup(rt_mmu_info *mmu_info, struct mem_desc *mdesc, int desc_nr); +static inline void *_rt_kmem_v2p(void *vaddr) +{ + return rt_hw_mmu_v2p(&rt_kernel_space, vaddr); +} -#define ARCH_ADDRESS_WIDTH_BITS 64 +static inline void *rt_kmem_v2p(void *vaddr) +{ + MM_PGTBL_LOCK(&rt_kernel_space); + void *paddr = _rt_kmem_v2p(vaddr); + MM_PGTBL_UNLOCK(&rt_kernel_space); + return paddr; +} -#define MMU_MAP_ERROR_VANOTALIGN -1 -#define MMU_MAP_ERROR_PANOTALIGN -2 -#define MMU_MAP_ERROR_NOPAGE -3 -#define MMU_MAP_ERROR_CONFLICT -4 +int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, + enum rt_mmu_cntl cmd); #endif diff --git a/libcpu/risc-v/virt64/riscv_mmu.c b/libcpu/risc-v/virt64/riscv_mmu.c index e79ca8b37b..7c4f886345 100644 --- a/libcpu/risc-v/virt64/riscv_mmu.c +++ b/libcpu/risc-v/virt64/riscv_mmu.c @@ -11,23 +11,13 @@ #include <rthw.h> #include <rtthread.h> -#include <stdint.h> #include <riscv.h> -#include <string.h> +#include <stdint.h> #include <stdlib.h> +#include <string.h> #include "riscv_mmu.h" -void mmu_set_pagetable(rt_ubase_t addr) -{ - RT_ASSERT(__CHECKALIGN(addr, PAGE_OFFSET_BIT)); - RT_ASSERT(__CHECKUPBOUND(addr, PHYSICAL_ADDRESS_WIDTH_BITS)); - - mmu_flush_tlb(); - write_csr(satp, (((size_t)SATP_MODE) << SATP_MODE_OFFSET) | (addr >> PAGE_OFFSET_BIT)); - mmu_flush_tlb(); -} - void mmu_enable_user_page_access() { set_csr(sstatus, SSTATUS_SUM); diff --git a/libcpu/risc-v/virt64/riscv_mmu.h b/libcpu/risc-v/virt64/riscv_mmu.h index 04c4201aab..fc5a143b65 100644 --- a/libcpu/risc-v/virt64/riscv_mmu.h +++ b/libcpu/risc-v/virt64/riscv_mmu.h @@ -58,7 +58,7 @@ #define PTE_USED(pte) __MASKVALUE(pte, PTE_V) -/** +/** * encoding of SATP (Supervisor Address Translation and Protection register) */ #define SATP_MODE_OFFSET 60 @@ -68,16 +68,9 @@ #define SATP_MODE_SV57 10 #define SATP_MODE_SV64 11 -#define ARCH_VA_WIDTH 39 +#define ARCH_VADDR_WIDTH 39 #define SATP_MODE SATP_MODE_SV39 -#define mmu_flush_tlb() \ - do \ - { \ - asm volatile("sfence.vma x0,x0"); \ - } while (0) - - #define MMU_MAP_K_DEVICE (PTE_G | PTE_W | PTE_R | PTE_V) #define MMU_MAP_K_RWCB (PTE_G | PTE_X | PTE_W | PTE_R | PTE_V) #define MMU_MAP_U_RWCB (PTE_U | PTE_X | PTE_W | PTE_R | PTE_V) @@ -93,6 +86,8 @@ #define ARCH_INDEX_SIZE (1ul << ARCH_INDEX_WIDTH) #define ARCH_INDEX_MASK (ARCH_INDEX_SIZE - 1) +#define ARCH_MAP_FAILED ((void *)0x8000000000000000) + void mmu_set_pagetable(rt_ubase_t addr); void mmu_enable_user_page_access(); void mmu_disable_user_page_access(); diff --git a/libcpu/risc-v/virt64/sbi.c b/libcpu/risc-v/virt64/sbi.c index 93084bd441..b2b1e8e788 100644 --- a/libcpu/risc-v/virt64/sbi.c +++ b/libcpu/risc-v/virt64/sbi.c @@ -36,8 +36,8 @@ */ #include "sbi.h" -#include <stdbool.h> #include <rtthread.h> +#include <stdbool.h> /* SBI Implementation-Specific Definitions */ #define OPENSBI_VERSION_MAJOR_OFFSET 16 @@ -51,20 +51,17 @@ static bool has_time_extension = false; static bool has_ipi_extension = false; static bool has_rfnc_extension = false; -static struct sbi_ret -sbi_get_spec_version(void) +static struct sbi_ret sbi_get_spec_version(void) { return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_SPEC_VERSION)); } -static struct sbi_ret -sbi_get_impl_id(void) +static struct sbi_ret sbi_get_impl_id(void) { return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_IMPL_ID)); } -static struct sbi_ret -sbi_get_impl_version(void) +static struct sbi_ret sbi_get_impl_version(void) { return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_IMPL_VERSION)); } @@ -88,7 +85,8 @@ void sbi_print_version(void) rt_kprintf("SBI: Berkely Boot Loader %lu\n", sbi_impl_version); break; case (SBI_IMPL_ID_XVISOR): - rt_kprintf("SBI: eXtensible Versatile hypervISOR %lu\n", sbi_impl_version); + rt_kprintf("SBI: eXtensible Versatile hypervISOR %lu\n", + sbi_impl_version); break; case (SBI_IMPL_ID_KVM): rt_kprintf("SBI: Kernel-based Virtual Machine %lu\n", sbi_impl_version); @@ -138,8 +136,7 @@ void sbi_send_ipi(const unsigned long *hart_mask) /* Use the IPI legacy replacement extension, if available. */ if (has_ipi_extension) { - ret = SBI_CALL2(SBI_EXT_ID_IPI, SBI_IPI_SEND_IPI, - *hart_mask, 0); + ret = SBI_CALL2(SBI_EXT_ID_IPI, SBI_IPI_SEND_IPI, *hart_mask, 0); RT_ASSERT(ret.error == SBI_SUCCESS); } else @@ -155,8 +152,8 @@ void sbi_remote_fence_i(const unsigned long *hart_mask) /* Use the RFENCE legacy replacement extension, if available. */ if (has_rfnc_extension) { - ret = SBI_CALL2(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_FENCE_I, - *hart_mask, 0); + ret = + SBI_CALL2(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_FENCE_I, *hart_mask, 0); RT_ASSERT(ret.error == SBI_SUCCESS); } else @@ -165,25 +162,28 @@ void sbi_remote_fence_i(const unsigned long *hart_mask) } } -void sbi_remote_sfence_vma(const unsigned long *hart_mask, unsigned long start, unsigned long size) +int sbi_remote_sfence_vma(const unsigned long *hart_mask, + const unsigned long hart_mask_base, + unsigned long start, unsigned long size) { - struct sbi_ret ret; + struct sbi_ret ret = {.error = SBI_SUCCESS}; /* Use the RFENCE legacy replacement extension, if available. */ if (has_rfnc_extension) { - ret = SBI_CALL4(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_SFENCE_VMA, - *hart_mask, 0, start, size); - RT_ASSERT(ret.error == SBI_SUCCESS); + ret = SBI_CALL4(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_SFENCE_VMA, *hart_mask, + hart_mask_base, start, size); } else { - (void)SBI_CALL3(SBI_REMOTE_SFENCE_VMA, 0, (uint64_t)hart_mask, - start, size); + (void)SBI_CALL3(SBI_REMOTE_SFENCE_VMA, 0, (uint64_t)hart_mask, start, + size); } + return ret.error; } -void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start, unsigned long size, +void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, + unsigned long start, unsigned long size, unsigned long asid) { struct sbi_ret ret; @@ -197,12 +197,13 @@ void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long st } else { - (void)SBI_CALL4(SBI_REMOTE_SFENCE_VMA_ASID, 0, - (uint64_t)hart_mask, start, size, asid); + (void)SBI_CALL4(SBI_REMOTE_SFENCE_VMA_ASID, 0, (uint64_t)hart_mask, + start, size, asid); } } -int sbi_hsm_hart_start(unsigned long hart, unsigned long start_addr, unsigned long priv) +int sbi_hsm_hart_start(unsigned long hart, unsigned long start_addr, + unsigned long priv) { struct sbi_ret ret; diff --git a/libcpu/risc-v/virt64/sbi.h b/libcpu/risc-v/virt64/sbi.h index a5ab83d27d..be54dede72 100644 --- a/libcpu/risc-v/virt64/sbi.h +++ b/libcpu/risc-v/virt64/sbi.h @@ -185,7 +185,9 @@ void sbi_send_ipi(const unsigned long *hart_mask); /* RFENCE extension functions. */ void sbi_remote_fence_i(const unsigned long *hart_mask); -void sbi_remote_sfence_vma(const unsigned long *hart_mask, unsigned long start, unsigned long size); +int sbi_remote_sfence_vma(const unsigned long *hart_mask, + const unsigned long hart_mask_base, + unsigned long start, unsigned long size); void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start, unsigned long size, unsigned long asid); diff --git a/libcpu/risc-v/virt64/start.c b/libcpu/risc-v/virt64/start.c new file mode 100644 index 0000000000..a8c374f698 --- /dev/null +++ b/libcpu/risc-v/virt64/start.c @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2006-2022, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-12-07 RT-Thread the first version + */ + +#include <rthw.h> +#include <rtthread.h> +#include "board.h" + +void init_bss(void) +{ + unsigned int *dst; + + dst = &__bss_start; + while (dst < &__bss_end) + { + *dst++ = 0; + } +} diff --git a/libcpu/risc-v/virt64/startup_gcc.S b/libcpu/risc-v/virt64/startup_gcc.S index 700f31181d..c9ce4bcf1e 100644 --- a/libcpu/risc-v/virt64/startup_gcc.S +++ b/libcpu/risc-v/virt64/startup_gcc.S @@ -10,7 +10,7 @@ * 2020/6/12 Xim Port to QEMU and remove SMP support */ -#define SSTATUS_FS 0x00006000U /* initial state of FPU, clear to disable */ +#include <encoding.h> #include <cpuport.h> boot_hartid: .int @@ -72,7 +72,7 @@ _start: li x31,0 /* set to disable FPU */ - li t0, SSTATUS_FS + li t0, SSTATUS_FS + SSTATUS_VS csrc sstatus, t0 li t0, 0x40000 // SUM in sstatus csrs sstatus, t0 @@ -91,4 +91,6 @@ _start: * sscratch is always zero on kernel mode */ csrw sscratch, zero + call init_bss + call sbi_init j primary_cpu_entry diff --git a/libcpu/risc-v/virt64/syscall_c.c b/libcpu/risc-v/virt64/syscall_c.c index 9a946d98b9..8bda3e6278 100644 --- a/libcpu/risc-v/virt64/syscall_c.c +++ b/libcpu/risc-v/virt64/syscall_c.c @@ -21,7 +21,6 @@ #include <stdint.h> #include <mmu.h> #include <page.h> -#include <lwp_mm_area.h> #include <lwp_user_mm.h> #include "riscv_mmu.h" diff --git a/libcpu/risc-v/virt64/tlb.h b/libcpu/risc-v/virt64/tlb.h new file mode 100644 index 0000000000..5bdf18c189 --- /dev/null +++ b/libcpu/risc-v/virt64/tlb.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2006-2022, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-11-28 WangXiaoyao the first version + */ +#ifndef __TLB_H__ +#define __TLB_H__ + +#include "mm_aspace.h" +#include "riscv_mmu.h" +#include "rtdbg.h" +#include "rtthread.h" +#include <sbi.h> +#include <stddef.h> +#include <stdint.h> + +#define HANDLE_FAULT(ret) \ + if (__builtin_expect((ret) != SBI_SUCCESS, 0)) \ + LOG_W("%s failed", __FUNCTION__); + +static inline void rt_hw_tlb_invalidate_all(void) +{ + uintptr_t mask = -1ul; + HANDLE_FAULT(sbi_remote_sfence_vma(&mask, -1ul, 0, mask)); +} + +static inline void rt_hw_tlb_invalidate_all_local(void) +{ + __asm__ volatile("sfence.vma" ::: "memory"); +} + +static inline void rt_hw_tlb_invalidate_aspace(rt_aspace_t aspace) +{ + // TODO ASID + rt_hw_tlb_invalidate_all_local(); +} + +static inline void rt_hw_tlb_invalidate_page(rt_aspace_t aspace, void *start) +{ + __asm__ volatile("sfence.vma %0, zero" ::"r"(start) : "memory"); +} + +static inline void rt_hw_tlb_invalidate_range(rt_aspace_t aspace, void *start, + size_t size, size_t stride) +{ + // huge page is taking as normal page + if (size <= ARCH_PAGE_SIZE) + { + rt_hw_tlb_invalidate_page(aspace, start); + } + else + { + rt_hw_tlb_invalidate_aspace(aspace); + } +} + +#endif /* __TLB_H__ */ diff --git a/libcpu/risc-v/virt64/trap.c b/libcpu/risc-v/virt64/trap.c index 0a5c082f30..07d9894f4a 100644 --- a/libcpu/risc-v/virt64/trap.c +++ b/libcpu/risc-v/virt64/trap.c @@ -12,6 +12,7 @@ #include <stdint.h> #include "encoding.h" +#include "mm_fault.h" #include "stack.h" #include "sbi.h" #include "riscv.h" @@ -139,7 +140,17 @@ extern struct rt_irq_desc irq_desc[]; #include "rtdbg.h" #include "encoding.h" -void sys_exit(int value); + +#ifndef RT_USING_SMP +static volatile int nested = 0; +#define ENTER_TRAP \ + nested += 1 +#define EXIT_TRAP \ + nested -= 1 +#define CHECK_NESTED_PANIC(cause, tval, epc, eframe) \ + if (nested != 1) \ + handle_nested_trap_panic(cause, tval, epc, eframe) +#endif /* RT_USING_SMP */ static const char *get_exception_msg(int id) { @@ -161,10 +172,59 @@ void handle_user(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw rt_size_t id = __MASKVALUE(scause, __MASK(63UL)); /* user page fault */ - if (id == EP_LOAD_PAGE_FAULT || - id == EP_STORE_PAGE_FAULT) + enum rt_mm_fault_op fault_op; + enum rt_mm_fault_type fault_type; + switch (id) + { + case EP_LOAD_PAGE_FAULT: + fault_op = MM_FAULT_OP_READ; + fault_type = MM_FAULT_TYPE_PAGE_FAULT; + break; + case EP_LOAD_ACCESS_FAULT: + fault_op = MM_FAULT_OP_READ; + fault_type = MM_FAULT_TYPE_ACCESS_FAULT; + break; + case EP_LOAD_ADDRESS_MISALIGNED: + fault_op = MM_FAULT_OP_READ; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + case EP_STORE_PAGE_FAULT: + fault_op = MM_FAULT_OP_WRITE; + fault_type = MM_FAULT_TYPE_PAGE_FAULT; + break; + case EP_STORE_ACCESS_FAULT: + fault_op = MM_FAULT_OP_WRITE; + fault_type = MM_FAULT_TYPE_ACCESS_FAULT; + break; + case EP_STORE_ADDRESS_MISALIGNED: + fault_op = MM_FAULT_OP_WRITE; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + case EP_INSTRUCTION_PAGE_FAULT: + fault_op = MM_FAULT_OP_EXECUTE; + fault_type = MM_FAULT_TYPE_PAGE_FAULT; + break; + case EP_INSTRUCTION_ACCESS_FAULT: + fault_op = MM_FAULT_OP_EXECUTE; + fault_type = MM_FAULT_TYPE_ACCESS_FAULT; + break; + case EP_INSTRUCTION_ADDRESS_MISALIGNED: + fault_op = MM_FAULT_OP_EXECUTE; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + default: + fault_op = 0; + } + + if (fault_op) { - if (arch_expand_user_stack((void *)stval)) + struct rt_mm_fault_msg msg = { + .fault_op = fault_op, + .fault_type = fault_type, + .vaddr = (void *)stval, + }; + + if (rt_mm_fault_try_fix(&msg)) { return; } @@ -176,6 +236,8 @@ void handle_user(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw rt_hw_backtrace((uint32_t *)sp->s0_fp, sepc); LOG_E("User Fault, killing thread: %s", rt_thread_self()->name); + + EXIT_TRAP; sys_exit(-1); } #endif @@ -228,17 +290,6 @@ static void handle_nested_trap_panic( rt_hw_cpu_shutdown(); } -#ifndef RT_USING_SMP -static volatile int nested = 0; -#define ENTER_TRAP \ - nested += 1 -#define EXIT_TRAP \ - nested -= 1 -#define CHECK_NESTED_PANIC(cause, tval, epc, eframe) \ - if (nested != 1) \ - handle_nested_trap_panic(cause, tval, epc, eframe) -#endif /* RT_USING_SMP */ - /* Trap entry */ void handle_trap(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw_stack_frame *sp) { |
