path: root/libcpu/risc-v/common/mmu.c

/*
 * Copyright (c) 2006-2025 RT-Thread Development Team
 *
 * SPDX-License-Identifier: Apache-2.0
 *
 * Change Logs:
 * Date           Author       Notes
 * 2021-01-30     lizhirui     first version
 * 2022-12-13     WangXiaoyao  Port to new mm
 * 2023-10-12     Shell        Add permission control API
 * 2025-04-26     GuEe-GUI     Follow the specific and vendor
 */

#include <rthw.h>
#include <rtthread.h>

#define DBG_TAG "hw.mmu"
#define DBG_LVL DBG_INFO
#include <rtdbg.h>

#include <cpu.h>
#include <csr.h>
#include <mmu.h>
#include <tlb.h>
#include <cache.h>
#include <ioremap.h>
#include <isa_ext.h>
#include <mm_page.h>
#include <mm_aspace.h>
#include <board.h>
#ifdef RT_USING_SMART
#include <lwp_user_mm.h>
#endif

#ifndef ARCH_TEXT_OFFSET
#if __riscv_xlen == 64
#define ARCH_TEXT_OFFSET 0x200000
#elif __riscv_xlen == 32
#define ARCH_TEXT_OFFSET 0x400000
#else
#error "Unexpected __riscv_xlen"
#endif
#endif

#ifndef ARCH_RAM_OFFSET
#define ARCH_RAM_OFFSET 0
#endif

#ifndef KERNEL_VADDR_START
#ifdef KERNEL_ASPACE_START
#define KERNEL_VADDR_START KERNEL_ASPACE_START
#else
#define KERNEL_VADDR_START (ARCH_RAM_OFFSET + ARCH_TEXT_OFFSET)
#endif
#endif

#if defined(RT_USING_SMP) && RT_CPUS_NR > 1
#define MMU_PERCPU
#endif

#if __riscv_xlen == 32
#define VPN_BITS    10
#define VPN_MASK    0x3ffUL
#define PPN_MASK    0xfff00000UL
#define LEVEL_MAX   1
rt_weak rt_uint8_t rt_hw_arch_vaddr_width = 32;
#else
#define VPN_BITS    9
#define VPN_MASK    0x1ffUL
#define PPN_MASK    0x3ffffffffffc00UL
#define LEVEL_MAX   5
rt_weak rt_uint8_t rt_hw_arch_vaddr_width = 39;
#endif

struct mmu_level_info
{
    rt_ubase_t *pos;
    void *page;
};

#ifdef MMU_PERCPU
struct mmu_percpu_level_info
{
    rt_ubase_t offset;
    rt_ubase_t *page;
};

#define PERCPU_OFFSET rt_hw_cpu_id() * ARCH_PAGE_SIZE
static struct mmu_percpu_level_info percpu_level[RT_CPUS_NR][LEVEL_MAX + 1];
#else
#define PERCPU_OFFSET 0
#endif

rt_ubase_t MMUTable[(ARCH_PAGE_SIZE / sizeof(rt_ubase_t)) * RT_CPUS_NR]
        rt_align(ARCH_PAGE_SIZE);

static rt_uint8_t satp_level_4k, satp_level_2M;
static rt_ubase_t satp_mode;

struct page_table
{
    rt_ubase_t page[ARCH_PAGE_SIZE / sizeof(rt_ubase_t)];
};
static struct page_table *__init_page_array;

rt_inline rt_ubase_t mmu_attr_fixup(rt_ubase_t attr)
{
#if !defined(__HAS_MMU_VENDOR__) && __riscv_xlen != 32
    if (!riscv_isa_ext_test(SVPBMT))
    {
        attr &= ~PTE_VENDOR_MTMASK;
    }
#endif

    return attr;
}

void set_free_page(void *page_array)
{
    __init_page_array = page_array;
}

rt_ubase_t get_free_page(void)
{
    static rt_ubase_t page_off;

    if (page_off < ARCH_PAGE_SIZE / sizeof(rt_ubase_t))
    {
        return (rt_ubase_t)__init_page_array[page_off++].page;
    }

    return 0;
}

static int mmu_map_early(rt_ubase_t *tbl, rt_uint8_t max_level,
        rt_ubase_t va, rt_ubase_t pa, rt_ubase_t attr, rt_bool_t flush)
{
    rt_ubase_t page, *table, off, level_shift;

    attr = mmu_attr_fixup(attr);

    if (max_level == satp_level_2M)
    {
        if ((va & SUPPER_PAGE_MASK) || (pa & SUPPER_PAGE_MASK))
        {
            return MMU_MAP_ERROR_VANOTALIGN;
        }
    }
    else
    {
        if ((va & ARCH_PAGE_MASK) || (pa & ARCH_PAGE_MASK))
        {
            return MMU_MAP_ERROR_VANOTALIGN;
        }
    }

    table = tbl;
    level_shift = rt_hw_arch_vaddr_width - VPN_BITS;

    for (int level = 0; level < max_level; ++level)
    {
        off = (va >> level_shift) & VPN_MASK;

        if (!(table[off] & PTE_V))
        {
            if (!(page = get_free_page()))
            {
                return MMU_MAP_ERROR_NOPAGE;
            }

            rt_memset((void *)page, 0, ARCH_PAGE_SIZE);
            table[off] = ((page >> ARCH_PAGE_SHIFT) << PTE_BITS) | PTE_V;

            if (flush)
            {
                rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *));
            }
        }

        if ((table[off] & PTE_ATTR_RWX) != 0)
        {
            /* No a page! */
            return MMU_MAP_ERROR_CONFLICT;
        }

        /* Step into the next level page table */
        page = (table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT;
        table = (rt_ubase_t *)page;

        level_shift -= VPN_BITS;
    }

    off = (va >> level_shift) & VPN_MASK;
    table[off] = ((pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | attr;

    if (flush)
    {
        rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *));
    }

    return 0;
}

#ifdef MMU_PERCPU
static int mmu_map_percpu_early(rt_ubase_t *tbl, rt_ubase_t *tbl_shadow,
                                rt_ubase_t va, rt_ubase_t pa)
{
    rt_ubase_t page, off, level_shift;

    if (tbl == tbl_shadow)
    {
        return mmu_map_early(tbl, satp_level_4k, va, pa, MMU_MAP_K_RWCB, RT_FALSE);
    }

    level_shift = rt_hw_arch_vaddr_width - VPN_BITS;

    for (int level = 0; level < satp_level_4k; ++level)
    {
        off = (va >> level_shift) & VPN_MASK;

        if (!(tbl[off] & PTE_V) || (tbl[off] & PTE_ATTR_RWX))
        {
            return MMU_MAP_ERROR_CONFLICT;
        }

        /* Copy a shared branch before changing its leaf mapping. */
        if (tbl_shadow[off] == tbl[off])
        {
            if (!(page = get_free_page()))
            {
                return MMU_MAP_ERROR_NOPAGE;
            }

            rt_memcpy((void *)page,
                      (void *)((tbl[off] >> PTE_BITS) << ARCH_PAGE_SHIFT),
                      ARCH_PAGE_SIZE);
            tbl_shadow[off] = ((page >> ARCH_PAGE_SHIFT) << PTE_BITS) | PTE_V;
        }
        else if (!(tbl_shadow[off] & PTE_V) || (tbl_shadow[off] & PTE_ATTR_RWX))
        {
            return MMU_MAP_ERROR_CONFLICT;
        }

        /* Step into the next level page table */
        tbl = (rt_ubase_t *)((tbl[off] >> PTE_BITS) << ARCH_PAGE_SHIFT);
        tbl_shadow = (rt_ubase_t *)((tbl_shadow[off] >> PTE_BITS) << ARCH_PAGE_SHIFT);

        level_shift -= VPN_BITS;
    }

    off = (va >> level_shift) & VPN_MASK;
    tbl_shadow[off] = ((pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | MMU_MAP_K_RWCB;

    return 0;
}
#endif /* MMU_PERCPU */

/**
 * @brief Switch the current address space to the specified one.
 *
 * This function is responsible for switching the address space by updating the page table
 * and related hardware state. The behavior depends on whether the architecture supports
 * Address Space Identifiers (ASIDs), devided by macro definition of ARCH_USING_ASID.
 *
 * @param aspace Pointer to the address space structure containing the new page table.
 *
 * @note If ASID is supported (`ARCH_USING_ASID` is defined), the ASID is encoded
 *       into `satp`. Otherwise, switching address spaces invalidates the local TLB.
 */
#ifdef ARCH_USING_ASID
/* Generation bits stay above the architectural ASID field. */
#define ASID_FIRST_GENERATION   (_ULL(1) << SATP_ASID_BITS)

rt_ubase_t riscv_asid_mask;

static rt_spinlock_t asid_lock = RT_SPINLOCK_INIT;

static void asid_init(void)
{
    rt_ubase_t old_satp = csr_read(CSR_SATP);

    /* Probe with the early page table active, preserving MODE and PPN. */
    csr_write(CSR_SATP, old_satp | (SATP_ASID_MASK << SATP_ASID_SHIFT));
    riscv_asid_mask = (csr_read(CSR_SATP) >> SATP_ASID_SHIFT) & SATP_ASID_MASK;
    csr_write(CSR_SATP, old_satp);

    /* The probe may have populated translations under the temporary ASID. */
    rt_hw_tlb_invalidate_all_local();
}

static rt_uint64_t _asid_acquire(rt_aspace_t aspace)
{
    static rt_uint32_t next_asid = 1;
    static rt_uint64_t global_asid_generation = ASID_FIRST_GENERATION;
    rt_ubase_t level;

    level = rt_spin_lock_irqsave(&asid_lock);

    if ((aspace->asid ^ global_asid_generation) >> SATP_ASID_BITS)
    {
        if (next_asid <= riscv_asid_mask)
        {
            aspace->asid = global_asid_generation | next_asid++;
        }
        else
        {
            global_asid_generation += ASID_FIRST_GENERATION;
            next_asid = 1;
            rt_hw_tlb_invalidate_all();

            aspace->asid = global_asid_generation | next_asid++;
        }
    }

    rt_spin_unlock_irqrestore(&asid_lock, level);

    return aspace->asid & riscv_asid_mask;
}

void rt_hw_aspace_switch(rt_aspace_t aspace)
{
    rt_ubase_t asid, page_table;

    page_table = (rt_ubase_t)rt_kmem_v2p(aspace->page_table + PERCPU_OFFSET);

    asid = riscv_asid_mask ? _asid_acquire(aspace) : 0;

    csr_write(CSR_SATP, satp_mode | (asid << SATP_ASID_SHIFT) |
            (page_table >> ARCH_PAGE_SHIFT));

#ifdef ARCH_RISCV_DISABLE_SFENCE_VMA_ASID
    rt_hw_tlb_invalidate_all_local();
#else
    if (riscv_asid_mask)
    {
        SFENCE_VMA_ASID(asid);
    }
    else
    {
        rt_hw_tlb_invalidate_all_local();
    }
#endif
}

#else /* !ARCH_USING_ASID */
void rt_hw_aspace_switch(rt_aspace_t aspace)
{
    rt_ubase_t page_table;

    page_table = (rt_ubase_t)rt_kmem_v2p(aspace->page_table + PERCPU_OFFSET);

    csr_write(CSR_SATP, satp_mode | ((rt_ubase_t)page_table >> ARCH_PAGE_SHIFT));

    rt_hw_tlb_invalidate_all_local();
}
#endif /* ARCH_USING_ASID */

void rt_hw_mmu_tbl_set(rt_ubase_t tbl)
{
    tbl += PERCPU_OFFSET;
    tbl += rt_kmem_pvoff();

    csr_write(CSR_SATP, satp_mode | (tbl >> ARCH_PAGE_SHIFT));

    rt_hw_tlb_invalidate_all_local();
}

void *rt_hw_mmu_tbl_get(void)
{
    rt_ubase_t satp = csr_read(CSR_SATP);

    return (void *)((satp & SATP_PPN) << ARCH_PAGE_SHIFT) - rt_kmem_pvoff();
}

void *rt_ioremap_early(void *paddr, size_t size)
{
    rt_ubase_t base;
    volatile size_t count;
    static void *tbl = RT_NULL;

    if (!size)
    {
        return RT_NULL;
    }

    if (!tbl)
    {
        tbl = rt_hw_mmu_tbl_get();
    }

    base = RT_ALIGN_DOWN((rt_size_t)paddr, SUPPER_PAGE_SIZE);
    count = RT_ALIGN((rt_ubase_t)paddr + size - base, SUPPER_PAGE_SIZE) / SUPPER_PAGE_SIZE;

    while (count --> 0)
    {
        if (mmu_map_early(tbl, satp_level_2M, base, base, MMU_MAP_K_DEVICE, RT_TRUE))
        {
            return RT_NULL;
        }

        base += SUPPER_PAGE_SIZE;
    }

    return paddr;
}

#ifdef MMU_PERCPU
static int _map_percpu_area(rt_ubase_t *table, void *va, void *pa, int cpu_id)
{
    rt_ubase_t page, off, level_shift;

    level_shift = rt_hw_arch_vaddr_width - VPN_BITS;

    for (int level = 0; level < satp_level_4k; ++level)
    {
        off = ((rt_ubase_t)va >> level_shift) & VPN_MASK;

        if (!(table[off] & PTE_V))
        {
            if (!(page = (rt_ubase_t)rt_pages_alloc(0)))
            {
                return MMU_MAP_ERROR_NOPAGE;
            }

            rt_memset((void *)page, 0, ARCH_PAGE_SIZE);

            table[off] = (((page + rt_kmem_pvoff()) >> ARCH_PAGE_SHIFT) << PTE_BITS) | PTE_V;

            rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *));
        }
        else if (table[off] & PTE_ATTR_RWX)
        {
            return MMU_MAP_ERROR_CONFLICT;
        }

        percpu_level[cpu_id][level].page = table;
        percpu_level[cpu_id][level].offset = off;

        /* Step into the next level page table */
        table = (rt_ubase_t *)(((table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT) - rt_kmem_pvoff());

        level_shift -= VPN_BITS;
    }

    off = ((rt_ubase_t)va >> level_shift) & VPN_MASK;
    table[off] = (((rt_ubase_t)pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | MMU_MAP_K_RWCB;

    rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *));

    percpu_level[cpu_id][satp_level_4k].page = table;
    percpu_level[cpu_id][satp_level_4k].offset = off;

    return ARCH_PAGE_SIZE;
}

static void percpu_map_sync(struct rt_aspace *aspace, int level, rt_ubase_t *table, rt_ubase_t off)
{
    if (aspace == &rt_kernel_space &&
        percpu_level[0][level].page == table &&
        percpu_level[0][level].offset != off)
    {
        rt_ubase_t *table_shadow;

        for (int i = 1; i < RT_CPUS_NR; ++i)
        {
            table_shadow = percpu_level[i][level].page;
            table_shadow[off] = table[off];

            rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table_shadow + off, sizeof(void *));
        }
    }
}

static void user_root_sync(struct rt_aspace *aspace, int level,
                           rt_ubase_t *table, rt_ubase_t off)
{
    if (aspace != &rt_kernel_space && level == 0 && table == aspace->page_table)
    {
        for (int i = 1; i < RT_CPUS_NR; ++i)
        {
            rt_ubase_t *table_shadow = (void *)aspace->page_table + i * ARCH_PAGE_SIZE;

            table_shadow[off] = table[off];
            rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table_shadow + off,
                                 sizeof(*table_shadow));
        }
    }
}

static void pgtbl_entry_sync(struct rt_aspace *aspace, int level,
                             rt_ubase_t *table, rt_ubase_t off)
{
    percpu_map_sync(aspace, level, table, off);
    user_root_sync(aspace, level, table, off);
}

static void percpu_mmu_init_check(void)
{
    size_t mapped, size;
    void *page_table, *vaddr, *paddr;
    static rt_bool_t inited = RT_FALSE;

    if (inited)
    {
        return;
    }
    inited = RT_TRUE;

    page_table = rt_kernel_space.page_table;

    for (int cpu_id = 0; cpu_id < RT_CPUS_NR; ++cpu_id)
    {
        vaddr = &__percpu_start;
        paddr = vaddr + rt_kmem_pvoff();
        size = (size_t)(&__percpu_end - &__percpu_start);
        /* Offset to per-CPU partition for current CPU */
        paddr += size * cpu_id;

        while (size > 0)
        {
            MM_PGTBL_LOCK(&rt_kernel_space);
            mapped = _map_percpu_area(page_table, vaddr, paddr, cpu_id);
            MM_PGTBL_UNLOCK(&rt_kernel_space);

            RT_ASSERT(mapped > 0);

            size -= mapped;
            vaddr += mapped;
            paddr += mapped;
        }

        page_table += ARCH_PAGE_SIZE;
    }
}
#endif /* MMU_PERCPU */

static size_t _map_area(struct rt_aspace *aspace, void *va, void *pa,
                        size_t attr, size_t max_size)
{
    rt_uint8_t max_level;
    rt_ubase_t page, *table, off, level_shift;

    attr = mmu_attr_fixup(attr);

    if (max_size >= SUPPER_PAGE_SIZE &&
        !((rt_ubase_t)va & SUPPER_PAGE_MASK) &&
        !((rt_ubase_t)pa & SUPPER_PAGE_MASK))
    {
        max_level = satp_level_2M;
        max_size = SUPPER_PAGE_SIZE;
    }
    else
    {
        max_level = satp_level_4k;
        max_size = ARCH_PAGE_SIZE;
    }

    table = aspace->page_table;
    level_shift = rt_hw_arch_vaddr_width - VPN_BITS;

    for (int level = 0; level < max_level; ++level)
    {
        off = ((rt_ubase_t)va >> level_shift) & VPN_MASK;

        if (!(table[off] & PTE_V))
        {
            if (!(page = (rt_ubase_t)rt_pages_alloc(0)))
            {
                return MMU_MAP_ERROR_NOPAGE;
            }

            rt_memset((void *)page, 0, ARCH_PAGE_SIZE);
            table[off] = (((page + rt_kmem_pvoff()) >> ARCH_PAGE_SHIFT) << PTE_BITS) | PTE_V;

            rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *));
        }
        else
        {
            page = (table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT;
            page -= rt_kmem_pvoff();

            rt_page_ref_inc((void *)page, 0);
        }

        if ((table[off] & PTE_ATTR_RWX) != 0)
        {
            /* No a page! */
            return MMU_MAP_ERROR_CONFLICT;
        }

    #ifdef MMU_PERCPU
        pgtbl_entry_sync(aspace, level, table, off);
    #endif

        /* Step into the next level page table */
        page = (table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT;
        table = (rt_ubase_t *)(page - rt_kmem_pvoff());

        level_shift -= VPN_BITS;
    }

    off = ((rt_ubase_t)va >> level_shift) & VPN_MASK;
    table[off] = (((rt_ubase_t)pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | attr;

    rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *));

#ifdef MMU_PERCPU
    pgtbl_entry_sync(aspace, max_level, table, off);
#endif

    return max_size;
}

static size_t _unmap_area(struct rt_aspace *aspace, void *va, size_t max_size)
{
    int level, ref;
    size_t unmapped;
    rt_ubase_t entry, page, *table, *pos, off, level_shift;
    struct mmu_level_info level_info[LEVEL_MAX + 1];

    rt_memset(level_info, 0, sizeof(level_info));

    table = aspace->page_table;
    level_shift = rt_hw_arch_vaddr_width - VPN_BITS;

    /* Max level is 4k */
    for (level = 0; level <= satp_level_4k; ++level)
    {
        off = ((rt_ubase_t)va >> level_shift) & VPN_MASK;
        pos = table + off;
        entry = *pos;
        unmapped = 1UL << (ARCH_PAGE_SHIFT +
                           VPN_BITS * (satp_level_4k - level));

        if (!(entry & PTE_V))
        {
            return max_size < unmapped ? max_size : unmapped;
        }

        level_info[level].pos = pos;

        if ((entry & PTE_ATTR_RWX) != 0)
        {
            break;
        }

        if (level == satp_level_4k)
        {
            return 0;
        }

        /* Step into the next level page table */
        page = (entry >> PTE_BITS) << ARCH_PAGE_SHIFT;
        table = (rt_ubase_t *)(page - rt_kmem_pvoff());

        level_info[level].page = table;

        level_shift -= VPN_BITS;
    }

    if (unmapped > max_size)
    {
        return 0;
    }

    pos = level_info[level].pos;
    *pos = 0;
    rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, pos, sizeof(void *));
#ifdef MMU_PERCPU
    pgtbl_entry_sync(aspace, level, pos - off, off);
#endif
    --level;

    while (level >= 0)
    {
        pos = level_info[level].pos;

        if (pos)
        {
            void *cur_page = level_info[level].page;

            ref = rt_page_ref_get(cur_page, 0);

            if (ref == 1)
            {
                *pos = 0;
                rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, pos, sizeof(void *));

            #ifdef MMU_PERCPU
                if (level == 0)
                {
                    table = aspace->page_table;
                }
                else
                {
                    table = percpu_level[0][level].page;
                }

                if (pos >= table && pos < table + (ARCH_PAGE_SIZE / sizeof(rt_ubase_t)))
                {
                    pgtbl_entry_sync(aspace, level, table, pos - table);
                }
            #endif /* MMU_PERCPU */
            }

            if (ref == 1)
            {
                /* Finish hardware page walks before recycling this table. */
                rt_hw_tlb_invalidate_all();
            }
            rt_pages_free(cur_page, 0);
        }
        else
        {
            break;
        }

        --level;
    }

    return unmapped;
}

/**
 * @brief Maps a virtual address space to a physical address space.
 *
 * This function maps a specified range of virtual addresses to a range of physical addresses
 * and sets the attributes of the page table entries (PTEs). If an error occurs during the
 * mapping process, the function will automatically roll back any partially completed mappings.
 *
 * @param aspace Pointer to the address space structure containing the page table information.
 * @param v_addr The starting virtual address to be mapped.
 * @param p_addr The starting physical address to be mapped.
 * @param size The size of the memory to be mapped (in bytes).
 * @param attr The attributes of the page table entries (e.g., read/write permissions, cache policies).
 *
 * @return On success, returns the starting virtual address `v_addr`;
 *         On failure, returns `NULL`.
 *
 * @note This function will not override existing page table entries.
 * @warning The caller must ensure that `v_addr` and `p_addr` are page-aligned,
 *          and `size` is a multiple of the page size.
 *
 */
void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr,
                    size_t size, size_t attr)
{
    size_t mapped;
    void *unmap_va = v_addr;

#ifdef MMU_PERCPU
    percpu_mmu_init_check();
#endif

    while (size > 0)
    {
    #ifdef MMU_PERCPU
        if (v_addr < (void *)&__percpu_start ||
            v_addr >= (void *)&__percpu_real_end)
    #endif
        {
            MM_PGTBL_LOCK(aspace);
            mapped = _map_area(aspace, v_addr, p_addr, attr, size);
            MM_PGTBL_UNLOCK(aspace);
        }
    #ifdef MMU_PERCPU
        else
        {
            /* Make the kernel skip the per-CPU as soon as possible */
            mapped = size > SUPPER_PAGE_SIZE ? SUPPER_PAGE_SIZE : size;
        }
    #endif /* MMU_PERCPU */

        if (mapped && mapped <= size)
        {
            size -= mapped;
            v_addr += mapped;
            p_addr += mapped;

            continue;
        }

        /* Error, undo map */
        while (unmap_va != v_addr)
        {
            MM_PGTBL_LOCK(aspace);
            mapped = _unmap_area(aspace, unmap_va,
                    (rt_ubase_t)v_addr - (rt_ubase_t)unmap_va);
            MM_PGTBL_UNLOCK(aspace);

            if (!mapped)
            {
                RT_ASSERT(mapped != 0);
                break;
            }
            unmap_va += mapped;
        }

        rt_hw_tlb_invalidate_all();
        return RT_NULL;
    }

    return unmap_va;
}

/**
 * @brief Unmaps a range of virtual memory addresses from the specified address space.
 *
 * This function is responsible for unmapping a contiguous region of virtual memory
 * from the given address space. It handles multiple pages and ensures thread safety
 * by locking the page table during the unmapping operation.
 *
 * @param aspace Pointer to the address space structure from which the memory will be unmapped.
 * @param v_addr Starting virtual address to unmap. Must be page-aligned.
 * @param size Size of the memory region to unmap. Must be page-aligned.
 *
 * @note The caller must ensure that both `v_addr` and `size` are page-aligned.
 *
 * @details The function operates in a loop, unmapping memory in chunks. It uses the
 * `_unmap_area` function to perform the actual unmapping, which is called within a
 * locked section to ensure thread safety. The loop continues until the entire region
 * is unmapped.
 *
 * @see _unmap_area
 * @note unmap is different from map that it can handle multiple pages
 */
void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size)
{
    size_t unmapped;

    /* caller guarantee that v_addr & size are page aligned */
    if (!aspace->page_table)
    {
        return;
    }

    while (size > 0)
    {
        MM_PGTBL_LOCK(aspace);
        unmapped = _unmap_area(aspace, v_addr, size);
        MM_PGTBL_UNLOCK(aspace);

        if (unmapped && unmapped <= size)
        {
            size -= unmapped;
            v_addr += unmapped;

            continue;
        }

        /* Region not exist in pgtbl */
        break;
    }
}

rt_inline void _init_region(void *vaddr, size_t size)
{
    rt_ioremap_start = vaddr;
    rt_ioremap_size = size;
    rt_mpr_start = rt_ioremap_start - rt_mpr_size;
    LOG_D("rt_ioremap_start: %p, rt_mpr_start: %p", rt_ioremap_start,
          rt_mpr_start);
}

rt_inline rt_ubase_t get_top_vpn(rt_ubase_t va, rt_uint8_t satp_level)
{
    return (va >> (VPN_BITS * satp_level + 12)) & VPN_MASK;
}

/**
 * @brief Initialize the MMU (Memory Management Unit) mapping.
 *
 * This function initializes the MMU mapping, incluing these steps as follows:
 * 1. Check the validity of the input parameters,
 * 2. Calculate the start and end virtual addresses based on the input virtual address and size.
 * 3. Convert the virtual addresses to PPN2 indices.
 * 4. Check the initialization of the page table. If any entry in the page table within
 *    the specified range is non-zero, it returns -1.
 * 5. It initializes the kernel address space using rt_aspace_init() and initializes the specified region
 *    using _init_region.
 *
 * @param aspace Pointer to the address space. Must not be NULL.
 * @param v_address The starting virtual address.
 * @param size The size of the virtual address space.
 * @param vtable Pointer to the page table. Must not be NULL.
 * @param pv_off The page table offset.
 *
 * @return Returns 0 if the initialization is successful. Returns -1 if any input parameter is invalid
 *         or the page table initialization check fails.
 */
int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_ubase_t size,
                       rt_ubase_t *vtable, rt_ubase_t pv_off)
{
    rt_size_t off, va_s, va_e;
    rt_ubase_t aspace_start = KERNEL_VADDR_START;

#if !defined(RT_USING_SMART) && ARCH_RAM_OFFSET
    aspace_start = ARCH_RAM_OFFSET;
#endif

    if ((!aspace) || (!vtable))
    {
        return -1;
    }

    va_s = (rt_ubase_t)v_address;
    va_e = (rt_ubase_t)v_address + size - 1;

    if (va_e < va_s)
    {
        return -1;
    }

    /* convert address to top-level index */
    va_s = get_top_vpn(va_s, satp_level_4k);
    va_e = get_top_vpn(va_e, satp_level_4k);

    if (va_s == 0)
    {
        return -1;
    }

    /* vtable initialization check */
    for (off = va_s; off <= va_e; off++)
    {
        rt_size_t v = vtable[off];

        if (v)
        {
            return -1;
        }
    }

#if __riscv_xlen == 32 && !defined(RT_USING_SMART)
    /* The RV32 ioremap and MPR regions are placed below the kernel image. */
    aspace_start = (rt_ubase_t)v_address - rt_mpr_size;
#endif

    rt_aspace_init(aspace, (void *)aspace_start, 0UL - aspace_start, vtable);

    _init_region(v_address, size);

    return 0;
}

static void *mmu_v2p(void *tbl, void *vaddr, int *out_level, rt_base_t **out_pte)
{
    rt_ubase_t page, entry, off, level_shift, pa;
    rt_ubase_t va = (rt_ubase_t)vaddr;
    rt_ubase_t *table = tbl;

#ifdef MMU_PERCPU
    if (tbl == rt_kernel_space.page_table)
    {
        table = (rt_ubase_t *)(tbl + PERCPU_OFFSET);
    }
#endif /* MMU_PERCPU */

    /* Calculate initial shift based on virtual address width */
    level_shift = rt_hw_arch_vaddr_width - VPN_BITS;

    /* 4k is the max level, translate to the last level */
    for (*out_level = 0; *out_level <= satp_level_4k; ++(*out_level))
    {
        /* Extract VPN (Virtual Page Number) for this level */
        off = (va >> level_shift) & VPN_MASK;
        /* Fetch the page table entry */
        entry = table[off];

        /* If entry is invalid (V bit not set), address translation fails */
        if (!(entry & PTE_V))
        {
            goto _translate_fail;
        }

        /* If entry has any RWX permission bits set, it is a leaf entry */
        if (entry & PTE_ATTR_RWX)
        {
            if (out_pte)
            {
                *out_pte = (rt_base_t *)table + off;
            }

            /* Extract physical page base address and add page offset */
            pa = ((entry & PTE_PPN_MASK) >> PTE_BITS) << ARCH_PAGE_SHIFT;

            if (*out_level >= satp_level_4k)
            {
                pa |= va & ARCH_PAGE_MASK;
            }
            else
            {
                pa |= va & ((1UL << level_shift) - 1);
            }

            return (void *)pa;
        }

        /* Otherwise, walk to the next level page table */
        page = (entry >> PTE_BITS) << ARCH_PAGE_SHIFT;
        table = (rt_ubase_t *)(page - rt_kmem_pvoff());

        level_shift -= VPN_BITS;
    }

_translate_fail:
    /* If reached here, it is not a valid mapped address */
    LOG_D("%s: failed at %p", __func__, vaddr);
    return ARCH_MAP_FAILED;
}

/**
 * @brief Translate a virtual address to a physical address.
 *
 * This function translates a given virtual address (`vaddr`) to its corresponding
 * physical address (`paddr`) using the page table in the specified address space (`aspace`).
 *
 * @param aspace Pointer to the address space structure containing the page table.
 * @param vaddr The virtual address to be translated.
 *
 * @return The translated physical address. If the translation fails, `ARCH_MAP_FAILED` is returned.
 *
 * @note The function queries the page table entry (PTE) for the virtual address using `mmu_v2p`.
 *       If a valid PTE is found, the physical address is extracted and combined with the offset
 *       from the virtual address. If no valid PTE is found, a debug log is recorded, and
 *       `ARCH_MAP_FAILED` is returned.
 */
void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr)
{
    int level;

    return mmu_v2p(aspace->page_table, vaddr, &level, RT_NULL);
}

static int _noncache(rt_base_t *pte)
{
    rt_ubase_t entry;

    if (!riscv_isa_ext_test(SVPBMT))
    {
        return -RT_ENOSYS;
    }

    entry = *pte;
    *pte = (entry & ~PTE_VENDOR_MTMASK) | PTE_VENDOR_NOCACHE;

    return 0;
}

static int _cache(rt_base_t *pte)
{
    rt_ubase_t entry;

    if (!riscv_isa_ext_test(SVPBMT))
    {
        return -RT_ENOSYS;
    }

    entry = *pte;
    *pte = (entry & ~PTE_VENDOR_MTMASK) | PTE_VENDOR_PMA;

    return 0;
}

static int (*control_handler[MMU_CNTL_DUMMY_END])(rt_base_t *pte)=
{
    [MMU_CNTL_CACHE] = _cache,
    [MMU_CNTL_NONCACHE] = _noncache,
};

/**
 * @brief Control the page table entries (PTEs) for a specified virtual address range.
 *
 * This function applies a control command (e.g., cache control) to the page table entries
 * (PTEs) corresponding to the specified virtual address range (`vaddr` to `vaddr + size`).
 *
 * @param aspace Pointer to the address space structure containing the page table.
 * @param vaddr The starting virtual address of the range.
 * @param size The size of the virtual address range.
 * @param cmd The control command to apply (e.g., `MMU_CNTL_CACHE`, `MMU_CNTL_NONCACHE`.etc.).
 *
 * @return `RT_EOK` on success, or an error code (`-RT_EINVAL` or `-RT_ENOSYS`) on failure.
 *
 * @note The function uses the `control_handler` array to map the command to a handler function.
 *       It iterates over the virtual address range, queries the PTEs, and applies the handler
 *       to each valid PTE. If the command is invalid, `-RT_ENOSYS` is returned.
 */
int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
                      enum rt_mmu_cntl cmd)
{
    int level;
    int err = -RT_EINVAL;
    void *vend = vaddr + size;

    int (*handler)(rt_base_t *pte);
    if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END)
    {
        handler = control_handler[cmd];

        while (vaddr < vend)
        {
            void *range_end;
            rt_base_t *pte = RT_NULL;

            mmu_v2p(aspace->page_table, vaddr, &level, &pte);

            if (level == satp_level_4k)
            {
                range_end = vaddr + ARCH_PAGE_SIZE;
            }
            else if (level == satp_level_2M)
            {
                range_end = vaddr + SUPPER_PAGE_SIZE;
            }
            else
            {
                /* 1G or larger is not supported. */
                RT_ASSERT(0);
            }

            if (pte)
            {
                err = handler(pte);
                RT_ASSERT(err == RT_EOK);
            }
            vaddr = range_end;
        }
    }
    else
    {
        err = -RT_ENOSYS;
    }

    return err;
}

/**
 * @brief setup Page Table for kernel space. It's a fixed map
 * and all mappings cannot be changed after initialization.
 *
 * Memory region in struct mem_desc must be page aligned,
 * otherwise is a failure and no report will be
 * returned.
 *
 * @param aspace Pointer to the address space structure.
 * @param mdesc Pointer to the array of memory descriptors.
 * @param desc_nr Number of memory descriptors in the array.
 */
void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr)
{
    void *err;

    for (size_t i = 0; i < desc_nr; ++i)
    {
        size_t attr;
        struct rt_mm_va_hint hint =
        {
            .flags = MMF_MAP_FIXED,
            .limit_start = aspace->start,
            .limit_range_size = aspace->size,
            .map_size = mdesc->vaddr_end - mdesc->vaddr_start + 1,
            .prefer = (void *)mdesc->vaddr_start
        };

        switch (mdesc->attr)
        {
        case NORMAL_MEM:
            attr = MMU_MAP_K_RWCB;
            break;
        case NORMAL_NOCACHE_MEM:
            attr = MMU_MAP_K_RWCB;
            break;
        case DEVICE_MEM:
            attr = MMU_MAP_K_DEVICE;
            break;
        default:
            attr = MMU_MAP_K_DEVICE;
        }

        if (mdesc->paddr_start == (rt_uintptr_t)ARCH_MAP_FAILED)
        {
            mdesc->paddr_start = mdesc->vaddr_start + rt_kmem_pvoff();
        }

        rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr,
                                 mdesc->paddr_start >> MM_PAGE_SHIFT, &err);
        mdesc++;
    }

#ifdef ARCH_USING_ASID
    asid_init();
#endif
    rt_hw_mmu_tbl_set((rt_ubase_t)rt_kernel_space.page_table);
    rt_page_cleanup();
}

/**
 * @brief Early memory setup function for hardware initialization.
 *
 * This function performs early memory setup tasks, including:
 * - Calculating the physical-to-virtual (PV) offset.
 * - Setting up initial page tables for identity mapping and text region relocation.
 * - Applying new memory mappings by updating the SATP register.
 *
 * @note This function is typically called during the early stages of system initialization (startup_gcc.S),
 *       before the memory management system is fully operational.
 */
void rt_hw_mem_setup_early(rt_ubase_t vaddr_start, rt_ubase_t paddr_start, void *pgtbl)
{
    extern char __text_start, __text_end;

    rt_size_t size = ARCH_EARLY_MAP_SIZE;
    rt_size_t map_pages;
    rt_ubase_t *early_pgtbl = pgtbl, paddr_end;
    rt_ubase_t kernel_paddr;

    rt_kmem_pvoff_set(paddr_start - vaddr_start);

    switch (rt_hw_arch_vaddr_width)
    {
    case 0:
        satp_mode = SATP_MODE_BARE;
        satp_level_4k = 0;
        satp_level_2M = 0;
    #ifdef RT_USING_SMART
        RT_ASSERT(0);
    #else
        RT_ASSERT(paddr_start == vaddr_start);
    #endif
        /* Do nothing */
        return;
#if __riscv_xlen != 32
    case 64:
        satp_mode = SATP_MODE_64;
        satp_level_4k = 5;
        satp_level_2M = 4;
        break;
    case 57:
        satp_mode = SATP_MODE_57;
        satp_level_4k = 4;
        satp_level_2M = 3;
        break;
    case 48:
        satp_mode = SATP_MODE_48;
        satp_level_4k = 3;
        satp_level_2M = 2;
        break;
    case 39:
    default:
        satp_mode = SATP_MODE_39;
        satp_level_4k = 2;
        satp_level_2M = 1;
        rt_hw_arch_vaddr_width = 39;
#else
    case 32:
    default:
        satp_mode = SATP_MODE_32;
        satp_level_4k = 1;
        satp_level_2M = 0;
        rt_hw_arch_vaddr_width = 32;
#endif /* __riscv_xlen != 32 */
        break;
    }

    map_pages = (size + (paddr_start & SUPPER_PAGE_MASK) + SUPPER_PAGE_MASK) /
            SUPPER_PAGE_SIZE;
    vaddr_start = RT_ALIGN_DOWN(vaddr_start, SUPPER_PAGE_SIZE);
    paddr_start = RT_ALIGN_DOWN(paddr_start, SUPPER_PAGE_SIZE);
    kernel_paddr = paddr_start;

    rt_memset(early_pgtbl, 0, RT_CPUS_NR * ARCH_PAGE_SIZE);

    if (!rt_kmem_pvoff())
    {
        goto _map_vaddress;
    }

    /* Identical mapping, only map for 'text' */
    size = RT_ALIGN((rt_ubase_t)(&__text_end - &__text_start), SUPPER_PAGE_SIZE);

    /* Check whether the virtual address area and identical area overlap */
    RT_ASSERT(size < rt_kmem_pvoff());

    for (rt_ssize_t pages = size / SUPPER_PAGE_SIZE; pages > 0; --pages)
    {
        mmu_map_early(early_pgtbl, satp_level_2M, paddr_start, paddr_start, MMU_MAP_K_RWCB, RT_FALSE);

        paddr_start += SUPPER_PAGE_SIZE;
    }

    /* Page array for map early */
    paddr_start = RT_ALIGN_DOWN((rt_ubase_t)early_pgtbl, SUPPER_PAGE_SIZE);
    mmu_map_early(early_pgtbl, satp_level_2M, paddr_start, paddr_start, MMU_MAP_K_RWCB, RT_FALSE);

    if ((rt_ubase_t)__init_page_array - paddr_start >= SUPPER_PAGE_SIZE)
    {
        paddr_start = RT_ALIGN_DOWN((rt_ubase_t)__init_page_array, SUPPER_PAGE_SIZE);
        mmu_map_early(early_pgtbl, satp_level_2M, paddr_start, paddr_start, MMU_MAP_K_RWCB, RT_FALSE);
    }

_map_vaddress:
    /* Virtual mapping */
    paddr_start = kernel_paddr;
    while (map_pages--)
    {
    #ifdef MMU_PERCPU
        if (paddr_start < (rt_ubase_t)&__percpu_start ||
            paddr_start >= (rt_ubase_t)&__percpu_real_end)
    #endif
        {
            mmu_map_early(early_pgtbl, satp_level_2M, vaddr_start, paddr_start, MMU_MAP_K_RWCB, RT_FALSE);
        }

        rt_ubase_t next_paddr = paddr_start + SUPPER_PAGE_SIZE;
        rt_ubase_t next_vaddr = vaddr_start + SUPPER_PAGE_SIZE;

        /* RV32 cannot represent an exclusive end address at 4 GiB. */
        if (next_paddr < paddr_start || next_vaddr < vaddr_start)
        {
            break;
        }

        paddr_start = next_paddr;
        vaddr_start = next_vaddr;
    }

#ifdef MMU_PERCPU
    for (int cpu_id = 0; cpu_id < RT_CPUS_NR; ++cpu_id)
    {
        rt_ubase_t *tbl_shadow = early_pgtbl +
            cpu_id * (ARCH_PAGE_SIZE / sizeof(*early_pgtbl));

        if (cpu_id)
        {
            rt_memcpy(tbl_shadow, early_pgtbl, ARCH_PAGE_SIZE);
        }

        paddr_start = (rt_ubase_t)&__percpu_start;
        vaddr_start = paddr_start - rt_kmem_pvoff();
        size = (rt_size_t)(&__percpu_end - &__percpu_start);
        /* Offset to per-CPU partition for current CPU */
        paddr_start += cpu_id * size;
        paddr_end = paddr_start + size;

        while (paddr_start < paddr_end)
        {
            /* Map physical address per-CPU partition */
            RT_ASSERT(mmu_map_percpu_early(early_pgtbl, tbl_shadow,
                                            vaddr_start, paddr_start) == 0);

            paddr_start += ARCH_PAGE_SIZE;
            vaddr_start += ARCH_PAGE_SIZE;
        }
    }
#endif /* MMU_PERCPU */
}

rt_ubase_t mmu_satp_load(void *pgtbl, rt_ubase_t hartid)
{
#ifdef MMU_PERCPU
    pgtbl += riscv_hartid_to_cpu_id(hartid) * ARCH_PAGE_SIZE;
#endif

    return satp_mode | ((rt_ubase_t)pgtbl >> ARCH_PAGE_SHIFT);
}

/**
 * @brief Creates and initializes a new MMU page table.
 *
 * This function allocates a new MMU page table, copies the kernel space
 * page table into it, and flushes the data cache to ensure consistency.
 *
 * @return
 * - A pointer to the newly allocated MMU page table on success.
 * - RT_NULL if the allocation fails.
 */
void *rt_hw_mmu_pgtbl_create(void)
{
    rt_size_t alloc_bits = 0;
    rt_ubase_t *mmu_table;

#ifdef MMU_PERCPU
    alloc_bits = rt_page_bits(RT_CPUS_NR * ARCH_PAGE_SIZE);
#endif
    mmu_table = (rt_ubase_t *)rt_pages_alloc_ext(alloc_bits, PAGE_ANY_AVAILABLE);
    if (!mmu_table)
    {
        return RT_NULL;
    }

#ifdef MMU_PERCPU
    for (int i = 0; i < RT_CPUS_NR; ++i)
    {
        rt_memcpy((void *)mmu_table + i * ARCH_PAGE_SIZE,
                  (void *)rt_kernel_space.page_table + i * ARCH_PAGE_SIZE,
                  ARCH_PAGE_SIZE);
    }
    rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_table,
                         RT_CPUS_NR * ARCH_PAGE_SIZE);
#else
    rt_memcpy(mmu_table, rt_kernel_space.page_table, ARCH_PAGE_SIZE);
    rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_table, ARCH_PAGE_SIZE);
#endif

    return mmu_table;
}

/**
 * @brief Deletes an MMU page table.
 *
 * This function frees the memory allocated for the given MMU page table.
 *
 * @param pgtbl Pointer to the MMU page table to be deleted.
 */
void rt_hw_mmu_pgtbl_delete(void *pgtbl)
{
    rt_size_t alloc_bits = 0;

#ifdef MMU_PERCPU
    alloc_bits = rt_page_bits(RT_CPUS_NR * ARCH_PAGE_SIZE);
#endif
    rt_pages_free(pgtbl, alloc_bits);
}