/* * Copyright (c) 2006-2025 RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * * Change Logs: * Date Author Notes * 2021-01-30 lizhirui first version * 2022-12-13 WangXiaoyao Port to new mm * 2023-10-12 Shell Add permission control API * 2025-04-26 GuEe-GUI Follow the specific and vendor */ #include #include #define DBG_TAG "hw.mmu" #define DBG_LVL DBG_INFO #include #include #include #include #include #include #include #include #include #include #include #ifdef RT_USING_SMART #include #endif #ifndef ARCH_TEXT_OFFSET #if __riscv_xlen == 64 #define ARCH_TEXT_OFFSET 0x200000 #elif __riscv_xlen == 32 #define ARCH_TEXT_OFFSET 0x400000 #else #error "Unexpected __riscv_xlen" #endif #endif #ifndef ARCH_RAM_OFFSET #define ARCH_RAM_OFFSET 0 #endif #ifndef KERNEL_VADDR_START #ifdef KERNEL_ASPACE_START #define KERNEL_VADDR_START KERNEL_ASPACE_START #else #define KERNEL_VADDR_START (ARCH_RAM_OFFSET + ARCH_TEXT_OFFSET) #endif #endif #if defined(RT_USING_SMP) && RT_CPUS_NR > 1 #define MMU_PERCPU #endif #if __riscv_xlen == 32 #define VPN_BITS 10 #define VPN_MASK 0x3ffUL #define PPN_MASK 0xfff00000UL #define LEVEL_MAX 1 rt_weak rt_uint8_t rt_hw_arch_vaddr_width = 32; #else #define VPN_BITS 9 #define VPN_MASK 0x1ffUL #define PPN_MASK 0x3ffffffffffc00UL #define LEVEL_MAX 5 rt_weak rt_uint8_t rt_hw_arch_vaddr_width = 39; #endif struct mmu_level_info { rt_ubase_t *pos; void *page; }; #ifdef MMU_PERCPU struct mmu_percpu_level_info { rt_ubase_t offset; rt_ubase_t *page; }; #define PERCPU_OFFSET rt_hw_cpu_id() * ARCH_PAGE_SIZE static struct mmu_percpu_level_info percpu_level[RT_CPUS_NR][LEVEL_MAX + 1]; #else #define PERCPU_OFFSET 0 #endif rt_ubase_t MMUTable[(ARCH_PAGE_SIZE / sizeof(rt_ubase_t)) * RT_CPUS_NR] rt_align(ARCH_PAGE_SIZE); static rt_uint8_t satp_level_4k, satp_level_2M; static rt_ubase_t satp_mode; struct page_table { rt_ubase_t page[ARCH_PAGE_SIZE / sizeof(rt_ubase_t)]; }; static struct page_table *__init_page_array; rt_inline rt_ubase_t mmu_attr_fixup(rt_ubase_t attr) { #if !defined(__HAS_MMU_VENDOR__) && __riscv_xlen != 32 if (!riscv_isa_ext_test(SVPBMT)) { attr &= ~PTE_VENDOR_MTMASK; } #endif return attr; } void set_free_page(void *page_array) { __init_page_array = page_array; } rt_ubase_t get_free_page(void) { static rt_ubase_t page_off; if (page_off < ARCH_PAGE_SIZE / sizeof(rt_ubase_t)) { return (rt_ubase_t)__init_page_array[page_off++].page; } return 0; } static int mmu_map_early(rt_ubase_t *tbl, rt_uint8_t max_level, rt_ubase_t va, rt_ubase_t pa, rt_ubase_t attr, rt_bool_t flush) { rt_ubase_t page, *table, off, level_shift; attr = mmu_attr_fixup(attr); if (max_level == satp_level_2M) { if ((va & SUPPER_PAGE_MASK) || (pa & SUPPER_PAGE_MASK)) { return MMU_MAP_ERROR_VANOTALIGN; } } else { if ((va & ARCH_PAGE_MASK) || (pa & ARCH_PAGE_MASK)) { return MMU_MAP_ERROR_VANOTALIGN; } } table = tbl; level_shift = rt_hw_arch_vaddr_width - VPN_BITS; for (int level = 0; level < max_level; ++level) { off = (va >> level_shift) & VPN_MASK; if (!(table[off] & PTE_V)) { if (!(page = get_free_page())) { return MMU_MAP_ERROR_NOPAGE; } rt_memset((void *)page, 0, ARCH_PAGE_SIZE); table[off] = ((page >> ARCH_PAGE_SHIFT) << PTE_BITS) | PTE_V; if (flush) { rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *)); } } if ((table[off] & PTE_ATTR_RWX) != 0) { /* No a page! */ return MMU_MAP_ERROR_CONFLICT; } /* Step into the next level page table */ page = (table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT; table = (rt_ubase_t *)page; level_shift -= VPN_BITS; } off = (va >> level_shift) & VPN_MASK; table[off] = ((pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | attr; if (flush) { rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *)); } return 0; } #ifdef MMU_PERCPU static int mmu_map_percpu_early(rt_ubase_t *tbl, rt_ubase_t *tbl_shadow, rt_ubase_t va, rt_ubase_t pa) { rt_ubase_t page, off, level_shift; if (tbl == tbl_shadow) { return mmu_map_early(tbl, satp_level_4k, va, pa, MMU_MAP_K_RWCB, RT_FALSE); } level_shift = rt_hw_arch_vaddr_width - VPN_BITS; for (int level = 0; level < satp_level_4k; ++level) { off = (va >> level_shift) & VPN_MASK; if (!(tbl[off] & PTE_V) || (tbl[off] & PTE_ATTR_RWX)) { return MMU_MAP_ERROR_CONFLICT; } /* Copy a shared branch before changing its leaf mapping. */ if (tbl_shadow[off] == tbl[off]) { if (!(page = get_free_page())) { return MMU_MAP_ERROR_NOPAGE; } rt_memcpy((void *)page, (void *)((tbl[off] >> PTE_BITS) << ARCH_PAGE_SHIFT), ARCH_PAGE_SIZE); tbl_shadow[off] = ((page >> ARCH_PAGE_SHIFT) << PTE_BITS) | PTE_V; } else if (!(tbl_shadow[off] & PTE_V) || (tbl_shadow[off] & PTE_ATTR_RWX)) { return MMU_MAP_ERROR_CONFLICT; } /* Step into the next level page table */ tbl = (rt_ubase_t *)((tbl[off] >> PTE_BITS) << ARCH_PAGE_SHIFT); tbl_shadow = (rt_ubase_t *)((tbl_shadow[off] >> PTE_BITS) << ARCH_PAGE_SHIFT); level_shift -= VPN_BITS; } off = (va >> level_shift) & VPN_MASK; tbl_shadow[off] = ((pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | MMU_MAP_K_RWCB; return 0; } #endif /* MMU_PERCPU */ /** * @brief Switch the current address space to the specified one. * * This function is responsible for switching the address space by updating the page table * and related hardware state. The behavior depends on whether the architecture supports * Address Space Identifiers (ASIDs), devided by macro definition of ARCH_USING_ASID. * * @param aspace Pointer to the address space structure containing the new page table. * * @note If ASID is supported (`ARCH_USING_ASID` is defined), the ASID is encoded * into `satp`. Otherwise, switching address spaces invalidates the local TLB. */ #ifdef ARCH_USING_ASID /* Generation bits stay above the architectural ASID field. */ #define ASID_FIRST_GENERATION (_ULL(1) << SATP_ASID_BITS) rt_ubase_t riscv_asid_mask; static rt_spinlock_t asid_lock = RT_SPINLOCK_INIT; static void asid_init(void) { rt_ubase_t old_satp = csr_read(CSR_SATP); /* Probe with the early page table active, preserving MODE and PPN. */ csr_write(CSR_SATP, old_satp | (SATP_ASID_MASK << SATP_ASID_SHIFT)); riscv_asid_mask = (csr_read(CSR_SATP) >> SATP_ASID_SHIFT) & SATP_ASID_MASK; csr_write(CSR_SATP, old_satp); /* The probe may have populated translations under the temporary ASID. */ rt_hw_tlb_invalidate_all_local(); } static rt_uint64_t _asid_acquire(rt_aspace_t aspace) { static rt_uint32_t next_asid = 1; static rt_uint64_t global_asid_generation = ASID_FIRST_GENERATION; rt_ubase_t level; level = rt_spin_lock_irqsave(&asid_lock); if ((aspace->asid ^ global_asid_generation) >> SATP_ASID_BITS) { if (next_asid <= riscv_asid_mask) { aspace->asid = global_asid_generation | next_asid++; } else { global_asid_generation += ASID_FIRST_GENERATION; next_asid = 1; rt_hw_tlb_invalidate_all(); aspace->asid = global_asid_generation | next_asid++; } } rt_spin_unlock_irqrestore(&asid_lock, level); return aspace->asid & riscv_asid_mask; } void rt_hw_aspace_switch(rt_aspace_t aspace) { rt_ubase_t asid, page_table; page_table = (rt_ubase_t)rt_kmem_v2p(aspace->page_table + PERCPU_OFFSET); asid = riscv_asid_mask ? _asid_acquire(aspace) : 0; csr_write(CSR_SATP, satp_mode | (asid << SATP_ASID_SHIFT) | (page_table >> ARCH_PAGE_SHIFT)); #ifdef ARCH_RISCV_DISABLE_SFENCE_VMA_ASID rt_hw_tlb_invalidate_all_local(); #else if (riscv_asid_mask) { SFENCE_VMA_ASID(asid); } else { rt_hw_tlb_invalidate_all_local(); } #endif } #else /* !ARCH_USING_ASID */ void rt_hw_aspace_switch(rt_aspace_t aspace) { rt_ubase_t page_table; page_table = (rt_ubase_t)rt_kmem_v2p(aspace->page_table + PERCPU_OFFSET); csr_write(CSR_SATP, satp_mode | ((rt_ubase_t)page_table >> ARCH_PAGE_SHIFT)); rt_hw_tlb_invalidate_all_local(); } #endif /* ARCH_USING_ASID */ void rt_hw_mmu_tbl_set(rt_ubase_t tbl) { tbl += PERCPU_OFFSET; tbl += rt_kmem_pvoff(); csr_write(CSR_SATP, satp_mode | (tbl >> ARCH_PAGE_SHIFT)); rt_hw_tlb_invalidate_all_local(); } void *rt_hw_mmu_tbl_get(void) { rt_ubase_t satp = csr_read(CSR_SATP); return (void *)((satp & SATP_PPN) << ARCH_PAGE_SHIFT) - rt_kmem_pvoff(); } void *rt_ioremap_early(void *paddr, size_t size) { rt_ubase_t base; volatile size_t count; static void *tbl = RT_NULL; if (!size) { return RT_NULL; } if (!tbl) { tbl = rt_hw_mmu_tbl_get(); } base = RT_ALIGN_DOWN((rt_size_t)paddr, SUPPER_PAGE_SIZE); count = RT_ALIGN((rt_ubase_t)paddr + size - base, SUPPER_PAGE_SIZE) / SUPPER_PAGE_SIZE; while (count --> 0) { if (mmu_map_early(tbl, satp_level_2M, base, base, MMU_MAP_K_DEVICE, RT_TRUE)) { return RT_NULL; } base += SUPPER_PAGE_SIZE; } return paddr; } #ifdef MMU_PERCPU static int _map_percpu_area(rt_ubase_t *table, void *va, void *pa, int cpu_id) { rt_ubase_t page, off, level_shift; level_shift = rt_hw_arch_vaddr_width - VPN_BITS; for (int level = 0; level < satp_level_4k; ++level) { off = ((rt_ubase_t)va >> level_shift) & VPN_MASK; if (!(table[off] & PTE_V)) { if (!(page = (rt_ubase_t)rt_pages_alloc(0))) { return MMU_MAP_ERROR_NOPAGE; } rt_memset((void *)page, 0, ARCH_PAGE_SIZE); table[off] = (((page + rt_kmem_pvoff()) >> ARCH_PAGE_SHIFT) << PTE_BITS) | PTE_V; rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *)); } else if (table[off] & PTE_ATTR_RWX) { return MMU_MAP_ERROR_CONFLICT; } percpu_level[cpu_id][level].page = table; percpu_level[cpu_id][level].offset = off; /* Step into the next level page table */ table = (rt_ubase_t *)(((table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT) - rt_kmem_pvoff()); level_shift -= VPN_BITS; } off = ((rt_ubase_t)va >> level_shift) & VPN_MASK; table[off] = (((rt_ubase_t)pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | MMU_MAP_K_RWCB; rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *)); percpu_level[cpu_id][satp_level_4k].page = table; percpu_level[cpu_id][satp_level_4k].offset = off; return ARCH_PAGE_SIZE; } static void percpu_map_sync(struct rt_aspace *aspace, int level, rt_ubase_t *table, rt_ubase_t off) { if (aspace == &rt_kernel_space && percpu_level[0][level].page == table && percpu_level[0][level].offset != off) { rt_ubase_t *table_shadow; for (int i = 1; i < RT_CPUS_NR; ++i) { table_shadow = percpu_level[i][level].page; table_shadow[off] = table[off]; rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table_shadow + off, sizeof(void *)); } } } static void user_root_sync(struct rt_aspace *aspace, int level, rt_ubase_t *table, rt_ubase_t off) { if (aspace != &rt_kernel_space && level == 0 && table == aspace->page_table) { for (int i = 1; i < RT_CPUS_NR; ++i) { rt_ubase_t *table_shadow = (void *)aspace->page_table + i * ARCH_PAGE_SIZE; table_shadow[off] = table[off]; rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table_shadow + off, sizeof(*table_shadow)); } } } static void pgtbl_entry_sync(struct rt_aspace *aspace, int level, rt_ubase_t *table, rt_ubase_t off) { percpu_map_sync(aspace, level, table, off); user_root_sync(aspace, level, table, off); } static void percpu_mmu_init_check(void) { size_t mapped, size; void *page_table, *vaddr, *paddr; static rt_bool_t inited = RT_FALSE; if (inited) { return; } inited = RT_TRUE; page_table = rt_kernel_space.page_table; for (int cpu_id = 0; cpu_id < RT_CPUS_NR; ++cpu_id) { vaddr = &__percpu_start; paddr = vaddr + rt_kmem_pvoff(); size = (size_t)(&__percpu_end - &__percpu_start); /* Offset to per-CPU partition for current CPU */ paddr += size * cpu_id; while (size > 0) { MM_PGTBL_LOCK(&rt_kernel_space); mapped = _map_percpu_area(page_table, vaddr, paddr, cpu_id); MM_PGTBL_UNLOCK(&rt_kernel_space); RT_ASSERT(mapped > 0); size -= mapped; vaddr += mapped; paddr += mapped; } page_table += ARCH_PAGE_SIZE; } } #endif /* MMU_PERCPU */ static size_t _map_area(struct rt_aspace *aspace, void *va, void *pa, size_t attr, size_t max_size) { rt_uint8_t max_level; rt_ubase_t page, *table, off, level_shift; attr = mmu_attr_fixup(attr); if (max_size >= SUPPER_PAGE_SIZE && !((rt_ubase_t)va & SUPPER_PAGE_MASK) && !((rt_ubase_t)pa & SUPPER_PAGE_MASK)) { max_level = satp_level_2M; max_size = SUPPER_PAGE_SIZE; } else { max_level = satp_level_4k; max_size = ARCH_PAGE_SIZE; } table = aspace->page_table; level_shift = rt_hw_arch_vaddr_width - VPN_BITS; for (int level = 0; level < max_level; ++level) { off = ((rt_ubase_t)va >> level_shift) & VPN_MASK; if (!(table[off] & PTE_V)) { if (!(page = (rt_ubase_t)rt_pages_alloc(0))) { return MMU_MAP_ERROR_NOPAGE; } rt_memset((void *)page, 0, ARCH_PAGE_SIZE); table[off] = (((page + rt_kmem_pvoff()) >> ARCH_PAGE_SHIFT) << PTE_BITS) | PTE_V; rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *)); } else { page = (table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT; page -= rt_kmem_pvoff(); rt_page_ref_inc((void *)page, 0); } if ((table[off] & PTE_ATTR_RWX) != 0) { /* No a page! */ return MMU_MAP_ERROR_CONFLICT; } #ifdef MMU_PERCPU pgtbl_entry_sync(aspace, level, table, off); #endif /* Step into the next level page table */ page = (table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT; table = (rt_ubase_t *)(page - rt_kmem_pvoff()); level_shift -= VPN_BITS; } off = ((rt_ubase_t)va >> level_shift) & VPN_MASK; table[off] = (((rt_ubase_t)pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | attr; rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *)); #ifdef MMU_PERCPU pgtbl_entry_sync(aspace, max_level, table, off); #endif return max_size; } static size_t _unmap_area(struct rt_aspace *aspace, void *va, size_t max_size) { int level, ref; size_t unmapped; rt_ubase_t entry, page, *table, *pos, off, level_shift; struct mmu_level_info level_info[LEVEL_MAX + 1]; rt_memset(level_info, 0, sizeof(level_info)); table = aspace->page_table; level_shift = rt_hw_arch_vaddr_width - VPN_BITS; /* Max level is 4k */ for (level = 0; level <= satp_level_4k; ++level) { off = ((rt_ubase_t)va >> level_shift) & VPN_MASK; pos = table + off; entry = *pos; unmapped = 1UL << (ARCH_PAGE_SHIFT + VPN_BITS * (satp_level_4k - level)); if (!(entry & PTE_V)) { return max_size < unmapped ? max_size : unmapped; } level_info[level].pos = pos; if ((entry & PTE_ATTR_RWX) != 0) { break; } if (level == satp_level_4k) { return 0; } /* Step into the next level page table */ page = (entry >> PTE_BITS) << ARCH_PAGE_SHIFT; table = (rt_ubase_t *)(page - rt_kmem_pvoff()); level_info[level].page = table; level_shift -= VPN_BITS; } if (unmapped > max_size) { return 0; } pos = level_info[level].pos; *pos = 0; rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, pos, sizeof(void *)); #ifdef MMU_PERCPU pgtbl_entry_sync(aspace, level, pos - off, off); #endif --level; while (level >= 0) { pos = level_info[level].pos; if (pos) { void *cur_page = level_info[level].page; ref = rt_page_ref_get(cur_page, 0); if (ref == 1) { *pos = 0; rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, pos, sizeof(void *)); #ifdef MMU_PERCPU if (level == 0) { table = aspace->page_table; } else { table = percpu_level[0][level].page; } if (pos >= table && pos < table + (ARCH_PAGE_SIZE / sizeof(rt_ubase_t))) { pgtbl_entry_sync(aspace, level, table, pos - table); } #endif /* MMU_PERCPU */ } if (ref == 1) { /* Finish hardware page walks before recycling this table. */ rt_hw_tlb_invalidate_all(); } rt_pages_free(cur_page, 0); } else { break; } --level; } return unmapped; } /** * @brief Maps a virtual address space to a physical address space. * * This function maps a specified range of virtual addresses to a range of physical addresses * and sets the attributes of the page table entries (PTEs). If an error occurs during the * mapping process, the function will automatically roll back any partially completed mappings. * * @param aspace Pointer to the address space structure containing the page table information. * @param v_addr The starting virtual address to be mapped. * @param p_addr The starting physical address to be mapped. * @param size The size of the memory to be mapped (in bytes). * @param attr The attributes of the page table entries (e.g., read/write permissions, cache policies). * * @return On success, returns the starting virtual address `v_addr`; * On failure, returns `NULL`. * * @note This function will not override existing page table entries. * @warning The caller must ensure that `v_addr` and `p_addr` are page-aligned, * and `size` is a multiple of the page size. * */ void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr, size_t size, size_t attr) { size_t mapped; void *unmap_va = v_addr; #ifdef MMU_PERCPU percpu_mmu_init_check(); #endif while (size > 0) { #ifdef MMU_PERCPU if (v_addr < (void *)&__percpu_start || v_addr >= (void *)&__percpu_real_end) #endif { MM_PGTBL_LOCK(aspace); mapped = _map_area(aspace, v_addr, p_addr, attr, size); MM_PGTBL_UNLOCK(aspace); } #ifdef MMU_PERCPU else { /* Make the kernel skip the per-CPU as soon as possible */ mapped = size > SUPPER_PAGE_SIZE ? SUPPER_PAGE_SIZE : size; } #endif /* MMU_PERCPU */ if (mapped && mapped <= size) { size -= mapped; v_addr += mapped; p_addr += mapped; continue; } /* Error, undo map */ while (unmap_va != v_addr) { MM_PGTBL_LOCK(aspace); mapped = _unmap_area(aspace, unmap_va, (rt_ubase_t)v_addr - (rt_ubase_t)unmap_va); MM_PGTBL_UNLOCK(aspace); if (!mapped) { RT_ASSERT(mapped != 0); break; } unmap_va += mapped; } rt_hw_tlb_invalidate_all(); return RT_NULL; } return unmap_va; } /** * @brief Unmaps a range of virtual memory addresses from the specified address space. * * This function is responsible for unmapping a contiguous region of virtual memory * from the given address space. It handles multiple pages and ensures thread safety * by locking the page table during the unmapping operation. * * @param aspace Pointer to the address space structure from which the memory will be unmapped. * @param v_addr Starting virtual address to unmap. Must be page-aligned. * @param size Size of the memory region to unmap. Must be page-aligned. * * @note The caller must ensure that both `v_addr` and `size` are page-aligned. * * @details The function operates in a loop, unmapping memory in chunks. It uses the * `_unmap_area` function to perform the actual unmapping, which is called within a * locked section to ensure thread safety. The loop continues until the entire region * is unmapped. * * @see _unmap_area * @note unmap is different from map that it can handle multiple pages */ void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size) { size_t unmapped; /* caller guarantee that v_addr & size are page aligned */ if (!aspace->page_table) { return; } while (size > 0) { MM_PGTBL_LOCK(aspace); unmapped = _unmap_area(aspace, v_addr, size); MM_PGTBL_UNLOCK(aspace); if (unmapped && unmapped <= size) { size -= unmapped; v_addr += unmapped; continue; } /* Region not exist in pgtbl */ break; } } rt_inline void _init_region(void *vaddr, size_t size) { rt_ioremap_start = vaddr; rt_ioremap_size = size; rt_mpr_start = rt_ioremap_start - rt_mpr_size; LOG_D("rt_ioremap_start: %p, rt_mpr_start: %p", rt_ioremap_start, rt_mpr_start); } rt_inline rt_ubase_t get_top_vpn(rt_ubase_t va, rt_uint8_t satp_level) { return (va >> (VPN_BITS * satp_level + 12)) & VPN_MASK; } /** * @brief Initialize the MMU (Memory Management Unit) mapping. * * This function initializes the MMU mapping, incluing these steps as follows: * 1. Check the validity of the input parameters, * 2. Calculate the start and end virtual addresses based on the input virtual address and size. * 3. Convert the virtual addresses to PPN2 indices. * 4. Check the initialization of the page table. If any entry in the page table within * the specified range is non-zero, it returns -1. * 5. It initializes the kernel address space using rt_aspace_init() and initializes the specified region * using _init_region. * * @param aspace Pointer to the address space. Must not be NULL. * @param v_address The starting virtual address. * @param size The size of the virtual address space. * @param vtable Pointer to the page table. Must not be NULL. * @param pv_off The page table offset. * * @return Returns 0 if the initialization is successful. Returns -1 if any input parameter is invalid * or the page table initialization check fails. */ int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_ubase_t size, rt_ubase_t *vtable, rt_ubase_t pv_off) { rt_size_t off, va_s, va_e; rt_ubase_t aspace_start = KERNEL_VADDR_START; #if !defined(RT_USING_SMART) && ARCH_RAM_OFFSET aspace_start = ARCH_RAM_OFFSET; #endif if ((!aspace) || (!vtable)) { return -1; } va_s = (rt_ubase_t)v_address; va_e = (rt_ubase_t)v_address + size - 1; if (va_e < va_s) { return -1; } /* convert address to top-level index */ va_s = get_top_vpn(va_s, satp_level_4k); va_e = get_top_vpn(va_e, satp_level_4k); if (va_s == 0) { return -1; } /* vtable initialization check */ for (off = va_s; off <= va_e; off++) { rt_size_t v = vtable[off]; if (v) { return -1; } } #if __riscv_xlen == 32 && !defined(RT_USING_SMART) /* The RV32 ioremap and MPR regions are placed below the kernel image. */ aspace_start = (rt_ubase_t)v_address - rt_mpr_size; #endif rt_aspace_init(aspace, (void *)aspace_start, 0UL - aspace_start, vtable); _init_region(v_address, size); return 0; } static void *mmu_v2p(void *tbl, void *vaddr, int *out_level, rt_base_t **out_pte) { rt_ubase_t page, entry, off, level_shift, pa; rt_ubase_t va = (rt_ubase_t)vaddr; rt_ubase_t *table = tbl; #ifdef MMU_PERCPU if (tbl == rt_kernel_space.page_table) { table = (rt_ubase_t *)(tbl + PERCPU_OFFSET); } #endif /* MMU_PERCPU */ /* Calculate initial shift based on virtual address width */ level_shift = rt_hw_arch_vaddr_width - VPN_BITS; /* 4k is the max level, translate to the last level */ for (*out_level = 0; *out_level <= satp_level_4k; ++(*out_level)) { /* Extract VPN (Virtual Page Number) for this level */ off = (va >> level_shift) & VPN_MASK; /* Fetch the page table entry */ entry = table[off]; /* If entry is invalid (V bit not set), address translation fails */ if (!(entry & PTE_V)) { goto _translate_fail; } /* If entry has any RWX permission bits set, it is a leaf entry */ if (entry & PTE_ATTR_RWX) { if (out_pte) { *out_pte = (rt_base_t *)table + off; } /* Extract physical page base address and add page offset */ pa = ((entry & PTE_PPN_MASK) >> PTE_BITS) << ARCH_PAGE_SHIFT; if (*out_level >= satp_level_4k) { pa |= va & ARCH_PAGE_MASK; } else { pa |= va & ((1UL << level_shift) - 1); } return (void *)pa; } /* Otherwise, walk to the next level page table */ page = (entry >> PTE_BITS) << ARCH_PAGE_SHIFT; table = (rt_ubase_t *)(page - rt_kmem_pvoff()); level_shift -= VPN_BITS; } _translate_fail: /* If reached here, it is not a valid mapped address */ LOG_D("%s: failed at %p", __func__, vaddr); return ARCH_MAP_FAILED; } /** * @brief Translate a virtual address to a physical address. * * This function translates a given virtual address (`vaddr`) to its corresponding * physical address (`paddr`) using the page table in the specified address space (`aspace`). * * @param aspace Pointer to the address space structure containing the page table. * @param vaddr The virtual address to be translated. * * @return The translated physical address. If the translation fails, `ARCH_MAP_FAILED` is returned. * * @note The function queries the page table entry (PTE) for the virtual address using `mmu_v2p`. * If a valid PTE is found, the physical address is extracted and combined with the offset * from the virtual address. If no valid PTE is found, a debug log is recorded, and * `ARCH_MAP_FAILED` is returned. */ void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr) { int level; return mmu_v2p(aspace->page_table, vaddr, &level, RT_NULL); } static int _noncache(rt_base_t *pte) { rt_ubase_t entry; if (!riscv_isa_ext_test(SVPBMT)) { return -RT_ENOSYS; } entry = *pte; *pte = (entry & ~PTE_VENDOR_MTMASK) | PTE_VENDOR_NOCACHE; return 0; } static int _cache(rt_base_t *pte) { rt_ubase_t entry; if (!riscv_isa_ext_test(SVPBMT)) { return -RT_ENOSYS; } entry = *pte; *pte = (entry & ~PTE_VENDOR_MTMASK) | PTE_VENDOR_PMA; return 0; } static int (*control_handler[MMU_CNTL_DUMMY_END])(rt_base_t *pte)= { [MMU_CNTL_CACHE] = _cache, [MMU_CNTL_NONCACHE] = _noncache, }; /** * @brief Control the page table entries (PTEs) for a specified virtual address range. * * This function applies a control command (e.g., cache control) to the page table entries * (PTEs) corresponding to the specified virtual address range (`vaddr` to `vaddr + size`). * * @param aspace Pointer to the address space structure containing the page table. * @param vaddr The starting virtual address of the range. * @param size The size of the virtual address range. * @param cmd The control command to apply (e.g., `MMU_CNTL_CACHE`, `MMU_CNTL_NONCACHE`.etc.). * * @return `RT_EOK` on success, or an error code (`-RT_EINVAL` or `-RT_ENOSYS`) on failure. * * @note The function uses the `control_handler` array to map the command to a handler function. * It iterates over the virtual address range, queries the PTEs, and applies the handler * to each valid PTE. If the command is invalid, `-RT_ENOSYS` is returned. */ int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, enum rt_mmu_cntl cmd) { int level; int err = -RT_EINVAL; void *vend = vaddr + size; int (*handler)(rt_base_t *pte); if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END) { handler = control_handler[cmd]; while (vaddr < vend) { void *range_end; rt_base_t *pte = RT_NULL; mmu_v2p(aspace->page_table, vaddr, &level, &pte); if (level == satp_level_4k) { range_end = vaddr + ARCH_PAGE_SIZE; } else if (level == satp_level_2M) { range_end = vaddr + SUPPER_PAGE_SIZE; } else { /* 1G or larger is not supported. */ RT_ASSERT(0); } if (pte) { err = handler(pte); RT_ASSERT(err == RT_EOK); } vaddr = range_end; } } else { err = -RT_ENOSYS; } return err; } /** * @brief setup Page Table for kernel space. It's a fixed map * and all mappings cannot be changed after initialization. * * Memory region in struct mem_desc must be page aligned, * otherwise is a failure and no report will be * returned. * * @param aspace Pointer to the address space structure. * @param mdesc Pointer to the array of memory descriptors. * @param desc_nr Number of memory descriptors in the array. */ void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr) { void *err; for (size_t i = 0; i < desc_nr; ++i) { size_t attr; struct rt_mm_va_hint hint = { .flags = MMF_MAP_FIXED, .limit_start = aspace->start, .limit_range_size = aspace->size, .map_size = mdesc->vaddr_end - mdesc->vaddr_start + 1, .prefer = (void *)mdesc->vaddr_start }; switch (mdesc->attr) { case NORMAL_MEM: attr = MMU_MAP_K_RWCB; break; case NORMAL_NOCACHE_MEM: attr = MMU_MAP_K_RWCB; break; case DEVICE_MEM: attr = MMU_MAP_K_DEVICE; break; default: attr = MMU_MAP_K_DEVICE; } if (mdesc->paddr_start == (rt_uintptr_t)ARCH_MAP_FAILED) { mdesc->paddr_start = mdesc->vaddr_start + rt_kmem_pvoff(); } rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr, mdesc->paddr_start >> MM_PAGE_SHIFT, &err); mdesc++; } #ifdef ARCH_USING_ASID asid_init(); #endif rt_hw_mmu_tbl_set((rt_ubase_t)rt_kernel_space.page_table); rt_page_cleanup(); } /** * @brief Early memory setup function for hardware initialization. * * This function performs early memory setup tasks, including: * - Calculating the physical-to-virtual (PV) offset. * - Setting up initial page tables for identity mapping and text region relocation. * - Applying new memory mappings by updating the SATP register. * * @note This function is typically called during the early stages of system initialization (startup_gcc.S), * before the memory management system is fully operational. */ void rt_hw_mem_setup_early(rt_ubase_t vaddr_start, rt_ubase_t paddr_start, void *pgtbl) { extern char __text_start, __text_end; rt_size_t size = ARCH_EARLY_MAP_SIZE; rt_size_t map_pages; rt_ubase_t *early_pgtbl = pgtbl, paddr_end; rt_ubase_t kernel_paddr; rt_kmem_pvoff_set(paddr_start - vaddr_start); switch (rt_hw_arch_vaddr_width) { case 0: satp_mode = SATP_MODE_BARE; satp_level_4k = 0; satp_level_2M = 0; #ifdef RT_USING_SMART RT_ASSERT(0); #else RT_ASSERT(paddr_start == vaddr_start); #endif /* Do nothing */ return; #if __riscv_xlen != 32 case 64: satp_mode = SATP_MODE_64; satp_level_4k = 5; satp_level_2M = 4; break; case 57: satp_mode = SATP_MODE_57; satp_level_4k = 4; satp_level_2M = 3; break; case 48: satp_mode = SATP_MODE_48; satp_level_4k = 3; satp_level_2M = 2; break; case 39: default: satp_mode = SATP_MODE_39; satp_level_4k = 2; satp_level_2M = 1; rt_hw_arch_vaddr_width = 39; #else case 32: default: satp_mode = SATP_MODE_32; satp_level_4k = 1; satp_level_2M = 0; rt_hw_arch_vaddr_width = 32; #endif /* __riscv_xlen != 32 */ break; } map_pages = (size + (paddr_start & SUPPER_PAGE_MASK) + SUPPER_PAGE_MASK) / SUPPER_PAGE_SIZE; vaddr_start = RT_ALIGN_DOWN(vaddr_start, SUPPER_PAGE_SIZE); paddr_start = RT_ALIGN_DOWN(paddr_start, SUPPER_PAGE_SIZE); kernel_paddr = paddr_start; rt_memset(early_pgtbl, 0, RT_CPUS_NR * ARCH_PAGE_SIZE); if (!rt_kmem_pvoff()) { goto _map_vaddress; } /* Identical mapping, only map for 'text' */ size = RT_ALIGN((rt_ubase_t)(&__text_end - &__text_start), SUPPER_PAGE_SIZE); /* Check whether the virtual address area and identical area overlap */ RT_ASSERT(size < rt_kmem_pvoff()); for (rt_ssize_t pages = size / SUPPER_PAGE_SIZE; pages > 0; --pages) { mmu_map_early(early_pgtbl, satp_level_2M, paddr_start, paddr_start, MMU_MAP_K_RWCB, RT_FALSE); paddr_start += SUPPER_PAGE_SIZE; } /* Page array for map early */ paddr_start = RT_ALIGN_DOWN((rt_ubase_t)early_pgtbl, SUPPER_PAGE_SIZE); mmu_map_early(early_pgtbl, satp_level_2M, paddr_start, paddr_start, MMU_MAP_K_RWCB, RT_FALSE); if ((rt_ubase_t)__init_page_array - paddr_start >= SUPPER_PAGE_SIZE) { paddr_start = RT_ALIGN_DOWN((rt_ubase_t)__init_page_array, SUPPER_PAGE_SIZE); mmu_map_early(early_pgtbl, satp_level_2M, paddr_start, paddr_start, MMU_MAP_K_RWCB, RT_FALSE); } _map_vaddress: /* Virtual mapping */ paddr_start = kernel_paddr; while (map_pages--) { #ifdef MMU_PERCPU if (paddr_start < (rt_ubase_t)&__percpu_start || paddr_start >= (rt_ubase_t)&__percpu_real_end) #endif { mmu_map_early(early_pgtbl, satp_level_2M, vaddr_start, paddr_start, MMU_MAP_K_RWCB, RT_FALSE); } rt_ubase_t next_paddr = paddr_start + SUPPER_PAGE_SIZE; rt_ubase_t next_vaddr = vaddr_start + SUPPER_PAGE_SIZE; /* RV32 cannot represent an exclusive end address at 4 GiB. */ if (next_paddr < paddr_start || next_vaddr < vaddr_start) { break; } paddr_start = next_paddr; vaddr_start = next_vaddr; } #ifdef MMU_PERCPU for (int cpu_id = 0; cpu_id < RT_CPUS_NR; ++cpu_id) { rt_ubase_t *tbl_shadow = early_pgtbl + cpu_id * (ARCH_PAGE_SIZE / sizeof(*early_pgtbl)); if (cpu_id) { rt_memcpy(tbl_shadow, early_pgtbl, ARCH_PAGE_SIZE); } paddr_start = (rt_ubase_t)&__percpu_start; vaddr_start = paddr_start - rt_kmem_pvoff(); size = (rt_size_t)(&__percpu_end - &__percpu_start); /* Offset to per-CPU partition for current CPU */ paddr_start += cpu_id * size; paddr_end = paddr_start + size; while (paddr_start < paddr_end) { /* Map physical address per-CPU partition */ RT_ASSERT(mmu_map_percpu_early(early_pgtbl, tbl_shadow, vaddr_start, paddr_start) == 0); paddr_start += ARCH_PAGE_SIZE; vaddr_start += ARCH_PAGE_SIZE; } } #endif /* MMU_PERCPU */ } rt_ubase_t mmu_satp_load(void *pgtbl, rt_ubase_t hartid) { #ifdef MMU_PERCPU pgtbl += riscv_hartid_to_cpu_id(hartid) * ARCH_PAGE_SIZE; #endif return satp_mode | ((rt_ubase_t)pgtbl >> ARCH_PAGE_SHIFT); } /** * @brief Creates and initializes a new MMU page table. * * This function allocates a new MMU page table, copies the kernel space * page table into it, and flushes the data cache to ensure consistency. * * @return * - A pointer to the newly allocated MMU page table on success. * - RT_NULL if the allocation fails. */ void *rt_hw_mmu_pgtbl_create(void) { rt_size_t alloc_bits = 0; rt_ubase_t *mmu_table; #ifdef MMU_PERCPU alloc_bits = rt_page_bits(RT_CPUS_NR * ARCH_PAGE_SIZE); #endif mmu_table = (rt_ubase_t *)rt_pages_alloc_ext(alloc_bits, PAGE_ANY_AVAILABLE); if (!mmu_table) { return RT_NULL; } #ifdef MMU_PERCPU for (int i = 0; i < RT_CPUS_NR; ++i) { rt_memcpy((void *)mmu_table + i * ARCH_PAGE_SIZE, (void *)rt_kernel_space.page_table + i * ARCH_PAGE_SIZE, ARCH_PAGE_SIZE); } rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_table, RT_CPUS_NR * ARCH_PAGE_SIZE); #else rt_memcpy(mmu_table, rt_kernel_space.page_table, ARCH_PAGE_SIZE); rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_table, ARCH_PAGE_SIZE); #endif return mmu_table; } /** * @brief Deletes an MMU page table. * * This function frees the memory allocated for the given MMU page table. * * @param pgtbl Pointer to the MMU page table to be deleted. */ void rt_hw_mmu_pgtbl_delete(void *pgtbl) { rt_size_t alloc_bits = 0; #ifdef MMU_PERCPU alloc_bits = rt_page_bits(RT_CPUS_NR * ARCH_PAGE_SIZE); #endif rt_pages_free(pgtbl, alloc_bits); }