diff options
| author | guanxi.cgh <[email protected]> | 2026-08-25 21:25:54 +0800 |
|---|---|---|
| committer | Rbb666 <[email protected]> | 2026-09-15 22:57:55 +0800 |
| commit | a5d9dc884b6f5f61b06664bbf745356416de98cf (patch) | |
| tree | 2677c6d5707957c60fedb8ec5dd99349b6c06ae1 /libcpu | |
| parent | dc3da6d87fa980b5e4fe2f73a828708e69182b90 (diff) | |
[libcpu][risc-v]Feat: Add mmu SV48/SV57 mode support and add mmu_demo for C908.
Diffstat (limited to 'libcpu')
| -rw-r--r-- | libcpu/risc-v/common64/mmu.c | 343 | ||||
| -rw-r--r-- | libcpu/risc-v/common64/mmu.h | 62 | ||||
| -rw-r--r-- | libcpu/risc-v/t-head/c908/riscv_mmu.h | 103 |
3 files changed, 311 insertions, 197 deletions
diff --git a/libcpu/risc-v/common64/mmu.c b/libcpu/risc-v/common64/mmu.c index 2dc82320f5..f09100c3d4 100644 --- a/libcpu/risc-v/common64/mmu.c +++ b/libcpu/risc-v/common64/mmu.c @@ -8,6 +8,7 @@ * 2021-01-30 lizhirui first version * 2022-12-13 WangXiaoyao Port to new mm * 2023-10-12 Shell Add permission control API + * 2026-08-25 chenguohao Add SV48/SV57 support */ #include <rtthread.h> @@ -52,7 +53,7 @@ static void *current_mmu_table[RT_CPUS_NR] = { RT_NULL }; #endif /* RT_USING_SMP */ volatile __attribute__((aligned(4 * 1024))) -rt_ubase_t MMUTable[__SIZE(VPN2_BIT) * RT_CPUS_NR]; +rt_ubase_t MMUTable[ARCH_INDEX_SIZE * RT_CPUS_NR]; /** * @brief Switch the current address space to the specified one. @@ -114,16 +115,12 @@ void *rt_hw_mmu_tbl_get() static int _map_one_page(struct rt_aspace *aspace, void *va, void *pa, size_t attr) { - rt_ubase_t l1_off, l2_off, l3_off; - rt_ubase_t *mmu_l1, *mmu_l2, *mmu_l3; + rt_ubase_t *table, *next_table; + rt_ubase_t off, leaf_off; int nr_cpus = RT_CPUS_NR; - l1_off = GET_L1((size_t)va); - l2_off = GET_L2((size_t)va); - l3_off = GET_L3((size_t)va); - #if defined(RT_USING_SMP) - /* For user address spaces, all harts share the same L2/L3 page tables. + /* For user address spaces, all harts share the same L2/L3/L4 page tables. * Only build the page table tree for hart 0, then sync L1 to other harts. */ if (aspace != &rt_kernel_space) { @@ -131,71 +128,58 @@ static int _map_one_page(struct rt_aspace *aspace, void *va, void *pa, } #endif - /* Create a separate page table for each hart to facilitate access to the .percpu section. */ for (int hartid = 0; hartid < nr_cpus; hartid++) { - mmu_l1 = (rt_ubase_t *)((rt_ubase_t)aspace->page_table + (rt_ubase_t)(hartid * ARCH_PAGE_SIZE)) + l1_off; + table = (rt_ubase_t *)((rt_ubase_t)aspace->page_table + (rt_ubase_t)(hartid * ARCH_PAGE_SIZE)); - if (PTE_USED(*mmu_l1)) - { - mmu_l2 = (rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET); - } - else + for (int level = 1; level < ARCH_PAGE_TBL_LEVELS; level++) { - mmu_l2 = (rt_ubase_t *)rt_pages_alloc(0); + off = GET_LVL_INDEX((rt_ubase_t)va, level); - if (mmu_l2) + if (PTE_USED(*(table + off))) { - rt_memset(mmu_l2, 0, PAGE_SIZE); - rt_hw_cpu_dcache_clean(mmu_l2, PAGE_SIZE); - *mmu_l1 = COMBINEPTE((rt_ubase_t)VPN_TO_PPN(mmu_l2, PV_OFFSET), - PAGE_DEFAULT_ATTR_NEXT); - rt_hw_cpu_dcache_clean(mmu_l1, sizeof(*mmu_l1)); + RT_ASSERT(!PAGE_IS_LEAF(*(table + off))); + table = (rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*(table + off)), PV_OFFSET); } else { - return -1; - } - } + next_table = (rt_ubase_t *)rt_pages_alloc(0); - if (PTE_USED(*(mmu_l2 + l2_off))) - { - RT_ASSERT(!PAGE_IS_LEAF(*(mmu_l2 + l2_off))); - mmu_l3 = - (rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), PV_OFFSET); - } - else - { - mmu_l3 = (rt_ubase_t *)rt_pages_alloc(0); - - if (mmu_l3) - { - rt_memset(mmu_l3, 0, PAGE_SIZE); - rt_hw_cpu_dcache_clean(mmu_l3, PAGE_SIZE); - *(mmu_l2 + l2_off) = - COMBINEPTE((rt_ubase_t)VPN_TO_PPN(mmu_l3, PV_OFFSET), - PAGE_DEFAULT_ATTR_NEXT); - rt_hw_cpu_dcache_clean(mmu_l2, sizeof(*mmu_l2)); - /* declares a reference to parent page table */ - rt_page_ref_inc((void *)mmu_l2, 0); - } - else - { - return -1; + if (next_table) + { + rt_memset(next_table, 0, PAGE_SIZE); + rt_hw_cpu_dcache_clean(next_table, PAGE_SIZE); + *(table + off) = COMBINEPTE((rt_ubase_t)VPN_TO_PPN(next_table, PV_OFFSET), + PAGE_DEFAULT_ATTR_NEXT); + rt_hw_cpu_dcache_clean(table + off, sizeof(*(table + off))); + /* declares a reference to parent page table */ + if (level > 1) + { + rt_page_ref_inc((void *)table, 0); + } + table = next_table; + } + else + { + return -1; + } } } - RT_ASSERT(!PTE_USED(*(mmu_l3 + l3_off))); - /* declares a reference to parent page table */ - rt_page_ref_inc((void *)mmu_l3, 0); - *(mmu_l3 + l3_off) = COMBINEPTE((rt_ubase_t)pa, attr); - rt_hw_cpu_dcache_clean(mmu_l3 + l3_off, sizeof(*(mmu_l3 + l3_off))); + /* Write the leaf PTE at the last level */ + leaf_off = GET_LVL_INDEX((rt_ubase_t)va, ARCH_PAGE_TBL_LEVELS); + RT_ASSERT(!PTE_USED(*(table + leaf_off))); + /* declares a reference to leaf page table */ + rt_page_ref_inc((void *)table, 0); + *(table + leaf_off) = COMBINEPTE((rt_ubase_t)pa, attr); + rt_hw_cpu_dcache_clean(table + leaf_off, sizeof(*(table + leaf_off))); } #if defined(RT_USING_SMP) /* For user address spaces, sync L1 entry from hart 0 to all other harts */ if (aspace != &rt_kernel_space) { + rt_ubase_t l1_off = GET_LVL_INDEX((rt_ubase_t)va, 1); rt_ubase_t *mmu_l1_0 = (rt_ubase_t *)aspace->page_table + l1_off; for (int h = 1; h < RT_CPUS_NR; h++) { @@ -217,16 +201,16 @@ static int _map_percpu_area(rt_ubase_t *table, void *va, void *pa, int cpu_id) unsigned long page; rt_ubase_t off, level_shift; - level_shift = PPN2_SHIFT; + level_shift = ARCH_TOP_LEVEL_SHIFT; // map pages - 4KB - for (int level = 0; level < 2; ++level) + for (int level = 1; level < ARCH_PAGE_TBL_LEVELS; ++level) { off = ((rt_ubase_t)va >> level_shift) & VPN_MASK; if (table[off] & PTE_V) { /* Step into the next level page table */ - table = (unsigned long *)((table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT); + table = (unsigned long *)((table[off] >> PTE_PPN_SHIFT) << ARCH_PAGE_SHIFT); level_shift -= VPN_BITS; continue; } @@ -235,18 +219,18 @@ static int _map_percpu_area(rt_ubase_t *table, void *va, void *pa, int cpu_id) return MMU_MAP_ERROR_NOPAGE; } rt_memset((void *)page, 0, ARCH_PAGE_SIZE); - table[off] = ((page >> ARCH_PAGE_SHIFT) << PTE_BITS) | PTE_V; + table[off] = ((page >> ARCH_PAGE_SHIFT) << PTE_PPN_SHIFT) | PTE_V; rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *)); /* Step into the next level page table */ - table = (unsigned long *)((table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT); + table = (unsigned long *)((table[off] >> PTE_PPN_SHIFT) << ARCH_PAGE_SHIFT); level_shift -= VPN_BITS; } off = ((rt_ubase_t)va >> level_shift) & VPN_MASK; - table[off] = (((rt_ubase_t)pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | MMU_MAP_K_RWCB; + table[off] = (((rt_ubase_t)pa >> ARCH_PAGE_SHIFT) << PTE_PPN_SHIFT) | MMU_MAP_K_RWCB; rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *)); @@ -387,19 +371,21 @@ unsigned long get_free_page(void) // Perform early mapping for the .percpu section static int rt_hw_mmu_map_percpu_early(rt_ubase_t *tbl, rt_ubase_t va, rt_ubase_t pa) { - unsigned long page; rt_ubase_t off, level_shift; - level_shift = PPN2_SHIFT; + level_shift = ARCH_TOP_LEVEL_SHIFT; // page size 2MB - off = (va >> level_shift) & VPN_MASK; - // Step into the next level page table - tbl = (rt_ubase_t *)((tbl[off] >> PTE_BITS) << ARCH_PAGE_SHIFT); - level_shift -= VPN_BITS; + for (int level = 1; level < ARCH_PAGE_TBL_LEVELS - 1; ++level) + { + off = (va >> level_shift) & VPN_MASK; + // Step into the next level page table + tbl = (rt_ubase_t *)((tbl[off] >> PTE_PPN_SHIFT) << ARCH_PAGE_SHIFT); + level_shift -= VPN_BITS; + } off = (va >> level_shift) & VPN_MASK; - tbl[off] = ((pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | MMU_MAP_K_RWCB; + tbl[off] = ((pa >> ARCH_PAGE_SHIFT) << PTE_PPN_SHIFT) | MMU_MAP_K_RWCB; asm volatile("sfence.vma x0, x0"); return 0; @@ -412,16 +398,16 @@ static int rt_hw_mmu_map_early(rt_ubase_t *tbl, rt_ubase_t va, rt_ubase_t pa, unsigned long page, *table; rt_ubase_t off, level_shift; - if ((va & (L2_PAGE_SIZE - 1)) || (pa & (L2_PAGE_SIZE - 1))) + if ((va & (EARLY_MAP_PAGE_SIZE_2M - 1)) || (pa & (EARLY_MAP_PAGE_SIZE_2M - 1))) { return MMU_MAP_ERROR_VANOTALIGN; } table = tbl; - level_shift = PPN2_SHIFT; + level_shift = ARCH_TOP_LEVEL_SHIFT; // page size 2MB - for (int level = 0; level < 1; ++level) + for (int level = 1; level < ARCH_PAGE_TBL_LEVELS - 1; ++level) { off = (va >> level_shift) & VPN_MASK; @@ -443,14 +429,14 @@ static int rt_hw_mmu_map_early(rt_ubase_t *tbl, rt_ubase_t va, rt_ubase_t pa, } /* Step into the next level page table */ - page = (table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT; + page = (table[off] >> PTE_PPN_SHIFT) << ARCH_PAGE_SHIFT; table = (unsigned long *)page; level_shift -= VPN_BITS; } off = (va >> level_shift) & VPN_MASK; - table[off] = ((pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | attr; + table[off] = ((pa >> ARCH_PAGE_SHIFT) << PTE_PPN_SHIFT) | attr; return 0; } @@ -485,22 +471,27 @@ static void _unmap_pte(rt_ubase_t *pentry, rt_ubase_t *lvl_entry[], int level) } } -/* Unmaps a virtual address range (1GB/2MB/4KB according to actual page level) from the page table. */ -static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr) +/* Unmaps a virtual address range (1GB/2MB/4KB according to actual page level) + * from one per-hart copy of the top level page table. */ +static size_t _unmap_area_one(rt_ubase_t *top_table, void *v_addr) { rt_ubase_t loop_va = __UMASKVALUE((rt_ubase_t)v_addr, PAGE_OFFSET_MASK); size_t unmapped = 0; int i = 0; - rt_ubase_t lvl_off[3]; - rt_ubase_t *lvl_entry[3]; - lvl_off[0] = (rt_ubase_t)GET_L1(loop_va); - lvl_off[1] = (rt_ubase_t)GET_L2(loop_va); - lvl_off[2] = (rt_ubase_t)GET_L3(loop_va); - unmapped = 1 << (ARCH_PAGE_SHIFT + ARCH_INDEX_WIDTH * 2ul); + rt_ubase_t lvl_off[ARCH_PAGE_TBL_LEVELS]; + rt_ubase_t *lvl_entry[ARCH_PAGE_TBL_LEVELS]; + + /* Pre-compute index at each level */ + for (int lvl = 0; lvl < ARCH_PAGE_TBL_LEVELS; lvl++) + { + lvl_off[lvl] = GET_LVL_INDEX(loop_va, lvl + 1); + } + + unmapped = 1ul << (ARCH_PAGE_SHIFT + ARCH_INDEX_WIDTH * (ARCH_PAGE_TBL_LEVELS - 1)); rt_ubase_t *pentry; - lvl_entry[i] = ((rt_ubase_t *)aspace->page_table + lvl_off[i]); + lvl_entry[i] = (top_table + lvl_off[i]); pentry = lvl_entry[i]; /* check if lvl_entry[0] is valid. if no, return 0 directly. */ @@ -514,7 +505,7 @@ static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr) { i += 1; - if (i >= 3) + if (i >= ARCH_PAGE_TBL_LEVELS) { unmapped = 0; break; @@ -536,18 +527,50 @@ static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr) unmapped = 0; /* invalid pte, return 0. */ } + return unmapped; +} + +/* Unmaps a virtual address range from the page table. */ +static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr) +{ + size_t unmapped = 0; + int nr_cpus = RT_CPUS_NR; + +#if defined(RT_USING_SMP) + /* For user address spaces, all harts share the same L2/L3/L4 page tables, as + * built by _map_one_page(). Only unmap hart 0's tree, otherwise the shared + * lower level page tables would be freed once per hart. */ + if (aspace != &rt_kernel_space) + { + nr_cpus = 1; + } +#endif + + for (int hartid = 0; hartid < nr_cpus; hartid++) + { + rt_ubase_t *top_table = + (rt_ubase_t *)((rt_ubase_t)aspace->page_table + (rt_ubase_t)(hartid * ARCH_PAGE_SIZE)); + size_t hart_unmapped = _unmap_area_one(top_table, v_addr); + + if (hartid == 0) + { + unmapped = hart_unmapped; + } + } + #if defined(RT_USING_SMP) /* For user address spaces, sync L1 entry from hart 0 to all other harts. * This ensures that if L1 was cleared (due to L2 being freed), all harts * see the updated L1 consistently. */ if (aspace != &rt_kernel_space && unmapped > 0) { - rt_ubase_t *mmu_l1_0 = (rt_ubase_t *)aspace->page_table + lvl_off[0]; + rt_ubase_t l1_off = GET_LVL_INDEX((rt_ubase_t)v_addr, 1); + rt_ubase_t *mmu_l1_0 = (rt_ubase_t *)aspace->page_table + l1_off; for (int h = 1; h < RT_CPUS_NR; h++) { rt_ubase_t *mmu_l1_h = (rt_ubase_t *)((rt_ubase_t)aspace->page_table + (rt_ubase_t)(h * ARCH_PAGE_SIZE)) + - lvl_off[0]; + l1_off; *mmu_l1_h = *mmu_l1_0; rt_hw_cpu_dcache_clean(mmu_l1_h, sizeof(*mmu_l1_h)); } @@ -634,7 +657,7 @@ static inline void _init_region(void *vaddr, size_t size) * This function initializes the MMU mapping, incluing these steps as follows: * 1. Check the validity of the input parameters, * 2. Calculate the start and end virtual addresses based on the input virtual address and size. - * 3. Convert the virtual addresses to PPN2 indices. + * 3. Convert the virtual addresses to top-level page table indices. * 4. Check the initialization of the page table. If any entry in the page table within * the specified range is non-zero, it returns -1. * 5. It initializes the kernel address space using rt_aspace_init() and initializes the specified region @@ -667,7 +690,7 @@ int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_ubase_t size, return -1; } - /* convert address to PPN2 index */ + /* convert address to top-level page table index */ va_s = GET_L1(va_s); va_e = GET_L1(va_e); @@ -693,8 +716,7 @@ int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_ubase_t size, return 0; } -const static int max_level = - (ARCH_VADDR_WIDTH - ARCH_PAGE_SHIFT) / ARCH_INDEX_WIDTH; +const static int max_level = ARCH_PAGE_TBL_LEVELS; static inline uintptr_t _get_level_size(int level) { @@ -703,13 +725,8 @@ static inline uintptr_t _get_level_size(int level) static rt_ubase_t *_query(struct rt_aspace *aspace, void *vaddr, int *level) { - rt_ubase_t l1_off, l2_off, l3_off; - rt_ubase_t *mmu_l1, *mmu_l2, *mmu_l3; - rt_ubase_t pa; - - l1_off = GET_L1((rt_uintptr_t)vaddr); - l2_off = GET_L2((rt_uintptr_t)vaddr); - l3_off = GET_L3((rt_uintptr_t)vaddr); + rt_ubase_t *table; + rt_ubase_t off; if (!aspace) { @@ -717,37 +734,29 @@ static rt_ubase_t *_query(struct rt_aspace *aspace, void *vaddr, int *level) return RT_NULL; } - mmu_l1 = ((rt_ubase_t *)((rt_ubase_t)aspace->page_table + (rt_ubase_t)(rt_hw_cpu_id() * ARCH_PAGE_SIZE))) + l1_off; + table = (rt_ubase_t *)((rt_ubase_t)aspace->page_table + (rt_ubase_t)(rt_hw_cpu_id() * ARCH_PAGE_SIZE)); - if (PTE_USED(*mmu_l1)) + for (int lvl = 1; lvl <= ARCH_PAGE_TBL_LEVELS; lvl++) { - if (*mmu_l1 & PTE_XWR_MASK) - { - *level = 1; - return mmu_l1; - } - - mmu_l2 = (rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET); + off = GET_LVL_INDEX((rt_uintptr_t)vaddr, lvl); - if (PTE_USED(*(mmu_l2 + l2_off))) + if (PTE_USED(*(table + off))) { - if (*(mmu_l2 + l2_off) & PTE_XWR_MASK) + if (PAGE_IS_LEAF(*(table + off))) { - *level = 2; - return mmu_l2 + l2_off; + *level = lvl; + return table + off; } - mmu_l3 = (rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), - PV_OFFSET); - - if (PTE_USED(*(mmu_l3 + l3_off))) - { - *level = 3; - return mmu_l3 + l3_off; - } + /* Not a leaf, step into next level page table */ + table = (rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*(table + off)), PV_OFFSET); + } + else + { + /* hole in the page table, no mapping for this address */ + break; } } - return RT_NULL; } @@ -956,19 +965,39 @@ void rt_hw_mem_setup_early(void *pgtbl, rt_uint64_t hartid) /** * identical mapping, - * PC are still at lower region before relocating to high memory + * PC are still at lower region before relocating to high memory. + * Create a 1GB mapping at the correct page table level for all SV modes. */ - rt_ubase_t pg_idx; - /* Round down symb_pc to L1_PAGE_SIZE boundary to ensure proper page alignment. - * This is necessary because MMU operations work with page-aligned addresses, and - * make sure all the text region is mapped.*/ - ps = (rt_ubase_t)symb_pc & (~(L1_PAGE_SIZE - 1)); - pg_idx = GET_L1(ps); - early_pgtbl[pg_idx] = COMBINEPTE(ps, MMU_MAP_EARLY); + { + unsigned long page, *table; + rt_ubase_t off, level_shift; + + ps = (rt_ubase_t)symb_pc & (~(EARLY_MAP_PAGE_SIZE_1G - 1)); + table = (unsigned long *)early_pgtbl; + level_shift = ARCH_TOP_LEVEL_SHIFT; + + /* Traverse to the 1GB leaf level */ + for (int level = 1; level < ARCH_PAGE_TBL_LEVELS - 2; ++level) + { + off = (ps >> level_shift) & VPN_MASK; + if (!(table[off] & PTE_V)) + { + page = get_free_page(); + rt_memset((void *)page, 0, ARCH_PAGE_SIZE); + table[off] = ((page >> ARCH_PAGE_SHIFT) << PTE_PPN_SHIFT) | PTE_V; + } + page = (table[off] >> PTE_PPN_SHIFT) << ARCH_PAGE_SHIFT; + table = (unsigned long *)page; + level_shift -= VPN_BITS; + } + + off = (ps >> level_shift) & VPN_MASK; + table[off] = ((ps >> ARCH_PAGE_SHIFT) << PTE_PPN_SHIFT) | MMU_MAP_EARLY; + } /* relocate text region */ __asm__ volatile("la %0, _start\n" : "=r"(ps)); - ps &= ~(L1_PAGE_SIZE - 1); + ps &= ~(EARLY_MAP_PAGE_SIZE_1G - 1); vs = ps - pv_off; /* relocate region */ @@ -977,14 +1006,37 @@ void rt_hw_mem_setup_early(void *pgtbl, rt_uint64_t hartid) while (vs < ve) { rt_hw_mmu_map_early(early_pgtbl, vs, ps, MMU_MAP_EARLY); - vs += L2_PAGE_SIZE; - ps += L2_PAGE_SIZE; + vs += EARLY_MAP_PAGE_SIZE_2M; + ps += EARLY_MAP_PAGE_SIZE_2M; } #else - for (int i = GET_L1(vs); i < GET_L1(ve); i++) + while (vs < ve) { - early_pgtbl[i] = COMBINEPTE(ps, MMU_MAP_EARLY); - ps += L1_PAGE_SIZE; + unsigned long page, *table; + rt_ubase_t off, level_shift; + + table = (unsigned long *)early_pgtbl; + level_shift = ARCH_TOP_LEVEL_SHIFT; + + for (int level = 1; level < ARCH_PAGE_TBL_LEVELS - 2; ++level) + { + off = (vs >> level_shift) & VPN_MASK; + if (!(table[off] & PTE_V)) + { + page = get_free_page(); + rt_memset((void *)page, 0, ARCH_PAGE_SIZE); + table[off] = ((page >> ARCH_PAGE_SHIFT) << PTE_PPN_SHIFT) | PTE_V; + } + page = (table[off] >> PTE_PPN_SHIFT) << ARCH_PAGE_SHIFT; + table = (unsigned long *)page; + level_shift -= VPN_BITS; + } + + off = (vs >> level_shift) & VPN_MASK; + table[off] = ((ps >> ARCH_PAGE_SHIFT) << PTE_PPN_SHIFT) | MMU_MAP_EARLY; + + vs += EARLY_MAP_PAGE_SIZE_1G; + ps += EARLY_MAP_PAGE_SIZE_1G; } #endif @@ -1001,8 +1053,8 @@ void rt_hw_mem_setup_early(void *pgtbl, rt_uint64_t hartid) { /* Map physical address per-CPU partition */ rt_hw_mmu_map_percpu_early(early_pgtbl, vs, ps); - ps += L2_PAGE_SIZE; - vs += L2_PAGE_SIZE; + ps += EARLY_MAP_PAGE_SIZE_2M; + vs += EARLY_MAP_PAGE_SIZE_2M; } #endif /* apply new mapping */ @@ -1013,6 +1065,19 @@ void rt_hw_mem_setup_early(void *pgtbl, rt_uint64_t hartid) /* return to lower text section */ } +/* Each aspace owns RT_CPUS_NR consecutive top-level page table pages (one + * per hart), as assumed by _map_one_page() and rt_hw_aspace_switch(). */ +rt_inline int _pgtbl_order(void) +{ + int order = 0; + + while ((1 << order) < RT_CPUS_NR) + { + order++; + } + return order; +} + /** * @brief Creates and initializes a new MMU page table. * @@ -1031,11 +1096,7 @@ void *rt_hw_mmu_pgtbl_create(void) /* Each hart needs its own L1 page table (for per-hart kernel .percpu mappings), * so allocate RT_CPUS_NR pages. Compute allocation order: smallest order where * 2^order >= RT_CPUS_NR. */ - int order = 0; - while ((1 << order) < RT_CPUS_NR) - { - order++; - } + int order = _pgtbl_order(); mmu_table = (rt_ubase_t *)rt_pages_alloc_ext(order, PAGE_ANY_AVAILABLE); if (!mmu_table) { @@ -1076,11 +1137,7 @@ void *rt_hw_mmu_pgtbl_create(void) void rt_hw_mmu_pgtbl_delete(void *pgtbl) { #if defined(RT_USING_SMP) - int order = 0; - while ((1 << order) < RT_CPUS_NR) - { - order++; - } + int order = _pgtbl_order(); rt_pages_free(pgtbl, order); #else rt_pages_free(pgtbl, 0); diff --git a/libcpu/risc-v/common64/mmu.h b/libcpu/risc-v/common64/mmu.h index eb4dd9a629..cdae6ad847 100644 --- a/libcpu/risc-v/common64/mmu.h +++ b/libcpu/risc-v/common64/mmu.h @@ -7,6 +7,7 @@ * Date Author Notes * 2021-01-30 lizhirui first version * 2023-10-12 Shell Add permission control API + * 2026-08-25 chenguohao Add SV48/SV57 support */ #ifndef __MMU_H__ @@ -37,18 +38,58 @@ struct mem_desc #define GET_PF_ID(addr) ((addr) >> PAGE_OFFSET_BIT) #define GET_PF_OFFSET(addr) __MASKVALUE(addr, PAGE_OFFSET_MASK) -#define GET_L1(addr) __PARTBIT(addr, VPN2_SHIFT, VPN2_BIT) -#define GET_L2(addr) __PARTBIT(addr, VPN1_SHIFT, VPN1_BIT) -#define GET_L3(addr) __PARTBIT(addr, VPN0_SHIFT, VPN0_BIT) -#define GET_PPN(pte) \ + +#ifndef ARCH_PAGE_TBL_LEVELS +#if (SATP_MODE == SATP_MODE_SV57) +#define ARCH_PAGE_TBL_LEVELS 5 +#elif (SATP_MODE == SATP_MODE_SV48) +#define ARCH_PAGE_TBL_LEVELS 4 +#else +#define ARCH_PAGE_TBL_LEVELS 3 +#endif +#endif /* ARCH_PAGE_TBL_LEVELS */ + +/* Top-level page table shift (PGD index extraction shift) */ +#ifndef ARCH_TOP_LEVEL_SHIFT +#define ARCH_TOP_LEVEL_SHIFT \ + (ARCH_PAGE_SHIFT + (ARCH_PAGE_TBL_LEVELS - 1) * ARCH_INDEX_WIDTH) +#endif + +/* Early mapping uses the fixed 1G/2M leaf size, whatever the paging mode is */ +#ifndef EARLY_MAP_PAGE_SIZE_1G +#define EARLY_MAP_PAGE_SIZE_1G __SIZE(ARCH_PAGE_SHIFT + 2 * ARCH_INDEX_WIDTH) +#endif +#ifndef EARLY_MAP_PAGE_SIZE_2M +#define EARLY_MAP_PAGE_SIZE_2M __SIZE(ARCH_PAGE_SHIFT + ARCH_INDEX_WIDTH) +#endif + +#if (ARCH_PAGE_TBL_LEVELS == 5) +#define GET_L1(addr) __PARTBIT(addr, VPN4_SHIFT, VPN4_BIT) +#define GET_L2(addr) __PARTBIT(addr, VPN3_SHIFT, VPN3_BIT) +#define GET_L3(addr) __PARTBIT(addr, VPN2_SHIFT, VPN2_BIT) +#define GET_L4(addr) __PARTBIT(addr, VPN1_SHIFT, VPN1_BIT) +#define GET_L5(addr) __PARTBIT(addr, VPN0_SHIFT, VPN0_BIT) +#elif (ARCH_PAGE_TBL_LEVELS == 4) +#define GET_L1(addr) __PARTBIT(addr, VPN3_SHIFT, VPN3_BIT) +#define GET_L2(addr) __PARTBIT(addr, VPN2_SHIFT, VPN2_BIT) +#define GET_L3(addr) __PARTBIT(addr, VPN1_SHIFT, VPN1_BIT) +#define GET_L4(addr) __PARTBIT(addr, VPN0_SHIFT, VPN0_BIT) +#elif (ARCH_PAGE_TBL_LEVELS == 3) +#define GET_L1(addr) __PARTBIT(addr, VPN2_SHIFT, VPN2_BIT) +#define GET_L2(addr) __PARTBIT(addr, VPN1_SHIFT, VPN1_BIT) +#define GET_L3(addr) __PARTBIT(addr, VPN0_SHIFT, VPN0_BIT) +#endif + +/* Runtime level index extraction: get the page table index at given level (1-based) */ +#define GET_LVL_INDEX(addr, level) \ + (((rt_ubase_t)(addr) >> (ARCH_PAGE_SHIFT + (ARCH_PAGE_TBL_LEVELS - (level)) * ARCH_INDEX_WIDTH)) & ((1UL << ARCH_INDEX_WIDTH) - 1)) + +#define GET_PPN(pte) \ (__PARTBIT(pte, PTE_PPN_SHIFT, PHYSICAL_ADDRESS_WIDTH_BITS - PAGE_OFFSET_BIT)) #define GET_PADDR(pte) (GET_PPN(pte) << PAGE_OFFSET_BIT) #define VPN_TO_PPN(vaddr, pv_off) (((rt_uintptr_t)(vaddr)) + (pv_off)) #define PPN_TO_VPN(paddr, pv_off) (((rt_uintptr_t)(paddr)) - (pv_off)) -#define COMBINEVADDR(l1_off, l2_off, l3_off) \ - (((l1_off) << VPN2_SHIFT) | ((l2_off) << VPN1_SHIFT) | \ - ((l3_off) << VPN0_SHIFT)) -#define COMBINEPTE(paddr, attr) \ +#define COMBINEPTE(paddr, attr) \ ((((paddr) >> PAGE_OFFSET_BIT) << PTE_PPN_SHIFT) | (attr)) #define MMU_MAP_ERROR_VANOTALIGN -1 @@ -56,9 +97,8 @@ struct mem_desc #define MMU_MAP_ERROR_NOPAGE -3 #define MMU_MAP_ERROR_CONFLICT -4 -#define VPN_MASK 0x1ffUL -#define PTE_BITS 10 -#define VPN_BITS 9 +#define VPN_MASK ((1UL << ARCH_INDEX_WIDTH) - 1) +#define VPN_BITS 9 #if defined(RT_USING_SMP) && defined(ARCH_MM_MMU) extern unsigned int __percpu_end, __percpu_start; diff --git a/libcpu/risc-v/t-head/c908/riscv_mmu.h b/libcpu/risc-v/t-head/c908/riscv_mmu.h index c457d104c1..84d764bd73 100644 --- a/libcpu/risc-v/t-head/c908/riscv_mmu.h +++ b/libcpu/risc-v/t-head/c908/riscv_mmu.h @@ -9,6 +9,7 @@ * 2021-05-03 lizhirui porting to C906 * 2023-10-12 Shell Add permission control API * 2026-02-25 Steven Porting to Standard Svpbmt + * 2026-08-25 chenguohao Add SV48/SV57 support */ #ifndef __RISCV_MMU_H__ @@ -34,11 +35,11 @@ #define PTE_PBMT_MASK (3UL << 61) #else /* XuanTie Extension (Bit 59-63) */ -#define PTE_SEC (1UL << 59) /* Security */ -#define PTE_SHARE (1UL << 60) /* Shareable */ -#define PTE_BUF (1UL << 61) /* Bufferable */ -#define PTE_CACHE (1UL << 62) /* Cacheable */ -#define PTE_SO (1UL << 63) /* Strong Order */ +#define PTE_SEC (1UL << 59) /* Security */ +#define PTE_SHARE (1UL << 60) /* Shareable */ +#define PTE_BUF (1UL << 61) /* Bufferable */ +#define PTE_CACHE (1UL << 62) /* Cacheable */ +#define PTE_SO (1UL << 63) /* Strong Order */ /* Compatible with Standard Svpbmt */ #define PTE_PBMT_PMA (PTE_CACHE | PTE_BUF | PTE_SHARE) #define PTE_PBMT_NC (PTE_BUF | PTE_SHARE) @@ -56,17 +57,22 @@ #define VPN1_BIT 9 #define VPN2_SHIFT (VPN1_SHIFT + VPN1_BIT) #define VPN2_BIT 9 +#define VPN3_SHIFT (VPN2_SHIFT + VPN2_BIT) +#define VPN3_BIT 9 +#define VPN4_SHIFT (VPN3_SHIFT + VPN3_BIT) +#define VPN4_BIT 9 #define PPN0_SHIFT (PAGE_OFFSET_SHIFT + PAGE_OFFSET_BIT) #define PPN0_BIT 9 #define PPN1_SHIFT (PPN0_SHIFT + PPN0_BIT) #define PPN1_BIT 9 #define PPN2_SHIFT (PPN1_SHIFT + PPN1_BIT) -#define PPN2_BIT 26 - -#define L1_PAGE_SIZE __SIZE(PAGE_OFFSET_BIT + VPN0_BIT + VPN1_BIT) -#define L2_PAGE_SIZE __SIZE(PAGE_OFFSET_BIT + VPN0_BIT) -#define L3_PAGE_SIZE __SIZE(PAGE_OFFSET_BIT) +#define PPN2_BIT 9 +#define PPN3_SHIFT (PPN2_SHIFT + PPN2_BIT) +#define PPN3_BIT 9 +#define PPN4_SHIFT (PPN3_SHIFT + PPN3_BIT) +#define PPN4_BIT 8 +#define PPN_BITS (PPN0_BIT + PPN1_BIT + PPN2_BIT + PPN3_BIT + PPN4_BIT) #define ARCH_ADDRESS_WIDTH_BITS 64 @@ -88,13 +94,13 @@ #define PAGE_DEFAULT_ATTR_LEAF \ (PAGE_ATTR_RWX | PAGE_ATTR_USER | PTE_V | PTE_G | PTE_PBMT_PMA | PTE_A | PTE_D) #else -#define PAGE_DEFAULT_ATTR_LEAF \ - (PAGE_ATTR_RWX | PAGE_ATTR_USER | PTE_V | PTE_G | PTE_SHARE | PTE_BUF | \ - PTE_CACHE | PTE_A | PTE_D) +#define PAGE_DEFAULT_ATTR_LEAF \ + (PAGE_ATTR_RWX | PAGE_ATTR_USER | PTE_V | PTE_G | PTE_SHARE | PTE_BUF | \ + PTE_CACHE | PTE_A | PTE_D) #endif #define PAGE_DEFAULT_ATTR_NEXT (PAGE_ATTR_NEXT_LEVEL | PTE_V | PTE_G) -#define PAGE_IS_LEAF(pte) __MASKVALUE(pte, PAGE_ATTR_RWX) +#define PAGE_IS_LEAF(pte) __MASKVALUE(pte, PAGE_ATTR_RWX) #define PTE_USED(pte) __MASKVALUE(pte, PTE_V) #define PTE_WRAP(attr) (attr | PTE_A | PTE_D) @@ -109,8 +115,18 @@ #define SATP_MODE_SV57 10 #define SATP_MODE_SV64 11 -#define ARCH_VADDR_WIDTH 39 -#define SATP_MODE SATP_MODE_SV39 +/* default to Sv39 */ +#ifndef SATP_MODE +#define SATP_MODE SATP_MODE_SV39 +#endif + +#if (SATP_MODE == SATP_MODE_SV57) +#define ARCH_VADDR_WIDTH 57 +#elif (SATP_MODE == SATP_MODE_SV48) +#define ARCH_VADDR_WIDTH 48 +#else +#define ARCH_VADDR_WIDTH 39 +#endif #if !CONFIG_XUANTIE_SVPBMT /* @@ -120,16 +136,16 @@ #define MMU_MAP_K_DEVICE (PTE_PBMT_IO | PTE_A | PTE_D | PTE_G | PTE_W | PTE_R | PTE_V) /* RW: Non-Cacheable (NC Mode) */ -#define MMU_MAP_K_RW (PTE_PBMT_NC | PTE_A | PTE_D | PTE_G | PAGE_ATTR_RWX | PTE_V) +#define MMU_MAP_K_RW (PTE_PBMT_NC | PTE_A | PTE_D | PTE_G | PAGE_ATTR_RWX | PTE_V) /* RWCB: Cacheable (PMA Mode) - Normal RAM */ -#define MMU_MAP_K_RWCB (PTE_PBMT_PMA | PTE_A | PTE_D | PTE_G | PAGE_ATTR_RWX | PTE_V) +#define MMU_MAP_K_RWCB (PTE_PBMT_PMA | PTE_A | PTE_D | PTE_G | PAGE_ATTR_RWX | PTE_V) /* * User Mappings */ /* User RW: Non-Cacheable */ -#define MMU_MAP_U_RW (PTE_PBMT_NC | PTE_U | PTE_A | PTE_D | PAGE_ATTR_RWX | PTE_V) +#define MMU_MAP_U_RW (PTE_PBMT_NC | PTE_U | PTE_A | PTE_D | PAGE_ATTR_RWX | PTE_V) /* User RWCB: Cacheable */ #define MMU_MAP_U_RWCB (PTE_PBMT_PMA | PTE_U | PTE_A | PTE_D | PAGE_ATTR_RWX | PTE_V) @@ -156,6 +172,7 @@ #define ARCH_PAGE_MASK (ARCH_PAGE_SIZE - 1) #define ARCH_PAGE_SHIFT PAGE_OFFSET_BIT #define ARCH_INDEX_WIDTH 9 +#define ARCH_INDEX_SIZE (1ul << ARCH_INDEX_WIDTH) #define ARCH_MAP_FAILED ((void *)0x8000000000000000) @@ -186,15 +203,15 @@ rt_inline size_t rt_hw_mmu_attr_rm_perm(size_t attr, rt_base_t prot) switch (prot) { /* remove write permission for user */ - case RT_HW_MMU_PROT_WRITE | RT_HW_MMU_PROT_USER: - attr &= ~PTE_W; - break; + case RT_HW_MMU_PROT_WRITE | RT_HW_MMU_PROT_USER: + attr &= ~PTE_W; + break; /* remove write permission for kernel */ - case RT_HW_MMU_PROT_WRITE | RT_HW_MMU_PROT_KERNEL: - attr &= ~PTE_W; - break; - default: - RT_ASSERT(0); + case RT_HW_MMU_PROT_WRITE | RT_HW_MMU_PROT_KERNEL: + attr &= ~PTE_W; + break; + default: + RT_ASSERT(0); } return attr; } @@ -211,11 +228,11 @@ rt_inline size_t rt_hw_mmu_attr_add_perm(size_t attr, rt_base_t prot) switch (prot) { /* add write permission for user */ - case RT_HW_MMU_PROT_WRITE | RT_HW_MMU_PROT_USER: - attr |= (PTE_R | PTE_W | PTE_U); - break; - default: - RT_ASSERT(0); + case RT_HW_MMU_PROT_WRITE | RT_HW_MMU_PROT_USER: + attr |= (PTE_R | PTE_W | PTE_U); + break; + default: + RT_ASSERT(0); } return attr; } @@ -233,17 +250,17 @@ rt_inline rt_bool_t rt_hw_mmu_attr_test_perm(size_t attr, rt_base_t prot) switch (prot & ~RT_HW_MMU_PROT_USER) { /* test write permission for user */ - case RT_HW_MMU_PROT_WRITE: - rc = ((attr & PTE_W) && (attr & PTE_R)); - break; - case RT_HW_MMU_PROT_READ: - rc = !!(attr & PTE_R); - break; - case RT_HW_MMU_PROT_EXECUTE: - rc = !!(attr & PTE_X); - break; - default: - RT_ASSERT(0); + case RT_HW_MMU_PROT_WRITE: + rc = ((attr & PTE_W) && (attr & PTE_R)); + break; + case RT_HW_MMU_PROT_READ: + rc = !!(attr & PTE_R); + break; + case RT_HW_MMU_PROT_EXECUTE: + rc = !!(attr & PTE_X); + break; + default: + RT_ASSERT(0); } if (rc && (prot & RT_HW_MMU_PROT_USER)) |
