summaryrefslogtreecommitdiff
path: root/libcpu
diff options
context:
space:
mode:
authorShell <[email protected]>2023-01-09 10:08:55 +0800
committerGitHub <[email protected]>2023-01-08 21:08:55 -0500
commit7450ef6c4d71f992444d27f4a26c16f813ed8e55 (patch)
tree1d9b450d85c2c048cae2410e97ef9edfc2868323 /libcpu
parent7f9ccd3c809afa6b54f0d453f2edcc8367a6aeb8 (diff)
[rt-smart] kernel virtual memory management layer (#6809)
synchronize virtual memory system works. adding kernel virtual memory management layer for page-based MMU enabled architecture porting libcpu MMU codes porting lwp memory related codes
Diffstat (limited to 'libcpu')
-rw-r--r--libcpu/aarch64/common/context_gcc.S27
-rw-r--r--libcpu/aarch64/common/cpu_ops_common.h2
-rw-r--r--libcpu/aarch64/common/cpuport.h15
-rw-r--r--libcpu/aarch64/common/interrupt.c14
-rw-r--r--libcpu/aarch64/common/mmu.c1056
-rw-r--r--libcpu/aarch64/common/mmu.h129
-rw-r--r--libcpu/aarch64/common/tlb.h80
-rw-r--r--libcpu/aarch64/common/trap.c49
-rw-r--r--libcpu/aarch64/cortex-a/entry_point.S19
-rw-r--r--libcpu/arm/cortex-a/context_gcc.S4
-rw-r--r--libcpu/arm/cortex-a/cpuport.h2
-rw-r--r--libcpu/arm/cortex-a/interrupt.c7
-rw-r--r--libcpu/arm/cortex-a/mmu.c665
-rw-r--r--libcpu/arm/cortex-a/mmu.h51
-rw-r--r--libcpu/arm/cortex-a/start_gcc.S25
-rw-r--r--libcpu/arm/cortex-a/tlb.h48
-rw-r--r--libcpu/arm/cortex-a/trap.c28
-rw-r--r--libcpu/risc-v/t-head/c906/backtrace.c137
-rw-r--r--libcpu/risc-v/t-head/c906/context_gcc.S4
-rw-r--r--libcpu/risc-v/t-head/c906/cpuport.c7
-rw-r--r--libcpu/risc-v/t-head/c906/cpuport.h8
-rw-r--r--libcpu/risc-v/t-head/c906/encoding.h2
-rw-r--r--libcpu/risc-v/t-head/c906/interrupt_gcc.S2
-rw-r--r--libcpu/risc-v/t-head/c906/mmu.c818
-rw-r--r--libcpu/risc-v/t-head/c906/mmu.h96
-rw-r--r--libcpu/risc-v/t-head/c906/riscv_mmu.c12
-rw-r--r--libcpu/risc-v/t-head/c906/riscv_mmu.h45
-rw-r--r--libcpu/risc-v/t-head/c906/sbi.c85
-rw-r--r--libcpu/risc-v/t-head/c906/sbi.h6
-rw-r--r--libcpu/risc-v/t-head/c906/tlb.h61
-rw-r--r--libcpu/risc-v/t-head/c906/trap.c77
-rw-r--r--libcpu/risc-v/virt64/backtrace.c7
-rw-r--r--libcpu/risc-v/virt64/context_gcc.S4
-rw-r--r--libcpu/risc-v/virt64/mmu.c726
-rw-r--r--libcpu/risc-v/virt64/mmu.h84
-rw-r--r--libcpu/risc-v/virt64/riscv_mmu.c14
-rw-r--r--libcpu/risc-v/virt64/riscv_mmu.h13
-rw-r--r--libcpu/risc-v/virt64/sbi.c47
-rw-r--r--libcpu/risc-v/virt64/sbi.h4
-rw-r--r--libcpu/risc-v/virt64/start.c24
-rw-r--r--libcpu/risc-v/virt64/startup_gcc.S6
-rw-r--r--libcpu/risc-v/virt64/syscall_c.c1
-rw-r--r--libcpu/risc-v/virt64/tlb.h61
-rw-r--r--libcpu/risc-v/virt64/trap.c81
44 files changed, 2095 insertions, 2558 deletions
diff --git a/libcpu/aarch64/common/context_gcc.S b/libcpu/aarch64/common/context_gcc.S
index 344d7317c6..0a611cbf48 100644
--- a/libcpu/aarch64/common/context_gcc.S
+++ b/libcpu/aarch64/common/context_gcc.S
@@ -183,7 +183,7 @@ rt_hw_get_gtimer_frq:
#ifdef RT_USING_LWP
BL rt_thread_self
MOV X19, X0
- BL lwp_mmu_switch
+ BL lwp_aspace_switch
MOV X0, X19
BL lwp_user_setting_restore
#endif
@@ -514,28 +514,3 @@ vector_serror:
STP X0, X1, [SP, #-0x10]!
BL rt_hw_trap_serror
b .
-
-.global rt_hw_mmu_switch
-rt_hw_mmu_switch:
- MSR TTBR0_EL1, X0
- MRS X1, TCR_EL1
- CMP X0, XZR
- ORR X1, X1, #(1 << 7)
- BEQ 1f
- BIC X1, X1, #(1 << 7)
-1:
- MSR TCR_EL1, X1
- DSB SY
- ISB
- TLBI VMALLE1
- DSB SY
- ISB
- IC IALLUIS
- DSB SY
- ISB
- RET
-
-.global rt_hw_mmu_tbl_get
-rt_hw_mmu_tbl_get:
- MRS X0, TTBR0_EL1
- RET
diff --git a/libcpu/aarch64/common/cpu_ops_common.h b/libcpu/aarch64/common/cpu_ops_common.h
index 3ac0a363c3..5e47584c7b 100644
--- a/libcpu/aarch64/common/cpu_ops_common.h
+++ b/libcpu/aarch64/common/cpu_ops_common.h
@@ -8,7 +8,7 @@
static inline rt_uint64_t get_secondary_entry_pa(void)
{
- rt_uint64_t secondary_entry_pa = (rt_uint64_t)rt_hw_mmu_v2p(&mmu_info, _secondary_cpu_entry);
+ rt_uint64_t secondary_entry_pa = (rt_uint64_t)rt_kmem_v2p(_secondary_cpu_entry);
if (!secondary_entry_pa)
{
diff --git a/libcpu/aarch64/common/cpuport.h b/libcpu/aarch64/common/cpuport.h
index 9f209069ac..cf767b9cfb 100644
--- a/libcpu/aarch64/common/cpuport.h
+++ b/libcpu/aarch64/common/cpuport.h
@@ -11,6 +11,7 @@
#define CPUPORT_H__
#include <armv8.h>
+#include <rtdef.h>
#ifdef RT_USING_SMP
typedef union {
@@ -29,12 +30,22 @@ rt_inline void rt_hw_isb(void)
rt_inline void rt_hw_dmb(void)
{
- asm volatile ("dmb sy":::"memory");
+ asm volatile ("dmb ish":::"memory");
+}
+
+rt_inline void rt_hw_wmb(void)
+{
+ asm volatile ("dmb ishst":::"memory");
+}
+
+rt_inline void rt_hw_rmb(void)
+{
+ asm volatile ("dmb ishld":::"memory");
}
rt_inline void rt_hw_dsb(void)
{
- asm volatile ("dsb sy":::"memory");
+ asm volatile ("dsb ish":::"memory");
}
#endif /*CPUPORT_H__*/
diff --git a/libcpu/aarch64/common/interrupt.c b/libcpu/aarch64/common/interrupt.c
index 3e527640b2..5a0b67160c 100644
--- a/libcpu/aarch64/common/interrupt.c
+++ b/libcpu/aarch64/common/interrupt.c
@@ -15,6 +15,12 @@
#include "gic.h"
#include "gicv3.h"
+#ifdef RT_USING_SMART
+#include "ioremap.h"
+#else
+#define rt_ioremap(x, ...) (x)
+#endif
+
/* exception and interrupt handler table */
struct rt_irq_desc isr_table[MAX_HANDLERS];
@@ -103,11 +109,11 @@ void rt_hw_interrupt_init(void)
/* initialize ARM GIC */
#ifdef RT_USING_SMART
- gic_dist_base = (rt_uint64_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_dist_base(), 0x2000, MMU_MAP_K_DEVICE);
- gic_cpu_base = (rt_uint64_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_cpu_base(), 0x1000, MMU_MAP_K_DEVICE);
+ gic_dist_base = (rt_uint64_t)rt_ioremap((void*)platform_get_gic_dist_base(), 0x2000);
+ gic_cpu_base = (rt_uint64_t)rt_ioremap((void*)platform_get_gic_cpu_base(), 0x1000);
#ifdef BSP_USING_GICV3
- gic_rdist_base = (rt_uint64_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_redist_base(),
- RT_CPUS_NR * (2 << 16), MMU_MAP_K_DEVICE);
+ gic_rdist_base = (rt_uint64_t)rt_ioremap((void*)platform_get_gic_redist_base(),
+ RT_CPUS_NR * (2 << 16));
#endif
#else
gic_dist_base = platform_get_gic_dist_base();
diff --git a/libcpu/aarch64/common/mmu.c b/libcpu/aarch64/common/mmu.c
index 0197e23f0f..d567d111d9 100644
--- a/libcpu/aarch64/common/mmu.c
+++ b/libcpu/aarch64/common/mmu.c
@@ -8,245 +8,44 @@
* 2012-01-10 bernard porting to AM1808
*/
-#include <rtthread.h>
-#include <rthw.h>
#include <board.h>
+#include <rthw.h>
+#include <rtthread.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include "mm_aspace.h"
+#include "mm_page.h"
#include "mmu.h"
+#include "tlb.h"
#ifdef RT_USING_SMART
+#include "ioremap.h"
#include <lwp_mm.h>
-#include <page.h>
#endif
-#define MMU_LEVEL_MASK 0x1ffUL
-#define MMU_LEVEL_SHIFT 9
-#define MMU_ADDRESS_BITS 39
-#define MMU_ADDRESS_MASK 0x0000fffffffff000UL
-#define MMU_ATTRIB_MASK 0xfff0000000000ffcUL
+#define DBG_TAG "hw.mmu"
+#define DBG_LVL DBG_LOG
+#include <rtdbg.h>
+
+#define MMU_LEVEL_MASK 0x1ffUL
+#define MMU_LEVEL_SHIFT 9
+#define MMU_ADDRESS_BITS 39
+#define MMU_ADDRESS_MASK 0x0000fffffffff000UL
+#define MMU_ATTRIB_MASK 0xfff0000000000ffcUL
-#define MMU_TYPE_MASK 3UL
-#define MMU_TYPE_USED 1UL
-#define MMU_TYPE_BLOCK 1UL
-#define MMU_TYPE_TABLE 3UL
-#define MMU_TYPE_PAGE 3UL
+#define MMU_TYPE_MASK 3UL
+#define MMU_TYPE_USED 1UL
+#define MMU_TYPE_BLOCK 1UL
+#define MMU_TYPE_TABLE 3UL
+#define MMU_TYPE_PAGE 3UL
#define MMU_TBL_BLOCK_2M_LEVEL 2
#define MMU_TBL_PAGE_4k_LEVEL 3
#define MMU_TBL_LEVEL_NR 4
-void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr);
-
-struct page_table
-{
- unsigned long page[512];
-};
-
-#ifndef RT_USING_SMART
-#define MMU_TBL_PAGE_NR_MAX 32
-
-static rt_mutex_t mm_lock = RT_NULL;
-
-void rt_mm_lock(void)
-{
- if (rt_thread_self())
- {
- if (!mm_lock)
- {
- mm_lock = rt_mutex_create("mm_lock", RT_IPC_FLAG_FIFO);
- }
- if (mm_lock)
- {
- rt_mutex_take(mm_lock, RT_WAITING_FOREVER);
- }
- }
-}
-
-void rt_mm_unlock(void)
-{
- if (rt_thread_self())
- {
- if (mm_lock)
- {
- rt_mutex_release(mm_lock);
- }
- }
-}
-
-static volatile struct page_table MMUPage[MMU_TBL_PAGE_NR_MAX] __attribute__((aligned(4096)));
-
-#define rt_page_ref_inc(...)
-
-unsigned long rt_pages_alloc(rt_size_t size_bits)
-{
- static unsigned long i = 0;
-
- if (i >= MMU_TBL_PAGE_NR_MAX)
- {
- return RT_NULL;
- }
-
- ++i;
-
- return (unsigned long)&MMUPage[i - 1].page;
-}
-#endif
-
-static struct page_table *__init_page_array;
-static unsigned long __page_off = 0UL;
-unsigned long get_free_page(void)
-{
- if (!__init_page_array)
- {
- unsigned long temp_page_start;
- asm volatile("mov %0, sp":"=r"(temp_page_start));
- __init_page_array = (struct page_table *)(temp_page_start & ~(ARCH_SECTION_MASK));
- __page_off = 2; /* 0, 1 for ttbr0, ttrb1 */
- }
- __page_off++;
- return (unsigned long)(__init_page_array[__page_off - 1].page);
-}
-
-void mmu_memset(char *dst, char v, size_t len)
-{
- while (len--)
- {
- *dst++ = v;
- }
-}
-
-static int _map_single_page_2M(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long attr)
-{
- int level;
- unsigned long *cur_lv_tbl = lv0_tbl;
- unsigned long page;
- unsigned long off;
- int level_shift = MMU_ADDRESS_BITS;
-
- if (va & ARCH_SECTION_MASK)
- {
- return MMU_MAP_ERROR_VANOTALIGN;
- }
- if (pa & ARCH_SECTION_MASK)
- {
- return MMU_MAP_ERROR_PANOTALIGN;
- }
- for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++)
- {
- off = (va >> level_shift);
- off &= MMU_LEVEL_MASK;
- if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
- {
- page = get_free_page();
- if (!page)
- {
- return MMU_MAP_ERROR_NOPAGE;
- }
- mmu_memset((char *)page, 0, ARCH_PAGE_SIZE);
- cur_lv_tbl[off] = page | MMU_TYPE_TABLE;
- }
- page = cur_lv_tbl[off];
- if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
- {
- /* is block! error! */
- return MMU_MAP_ERROR_CONFLICT;
- }
- cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
- level_shift -= MMU_LEVEL_SHIFT;
- }
- attr &= MMU_ATTRIB_MASK;
- pa |= (attr | MMU_TYPE_BLOCK); /* block */
- off = (va >> ARCH_SECTION_SHIFT);
- off &= MMU_LEVEL_MASK;
- cur_lv_tbl[off] = pa;
- return 0;
-}
-
-int armv8_init_map_2M(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long count, unsigned long attr)
-{
- unsigned long i;
- int ret;
-
- if (va & ARCH_SECTION_MASK)
- {
- return -1;
- }
- if (pa & ARCH_SECTION_MASK)
- {
- return -1;
- }
- for (i = 0; i < count; i++)
- {
- ret = _map_single_page_2M(lv0_tbl, va, pa, attr);
- va += ARCH_SECTION_SIZE;
- pa += ARCH_SECTION_SIZE;
- if (ret != 0)
- {
- return ret;
- }
- }
- return 0;
-}
-
-static int _kenrel_map_2M(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long attr)
-{
- int level;
- unsigned long *cur_lv_tbl = lv0_tbl;
- unsigned long page;
- unsigned long off;
- int level_shift = MMU_ADDRESS_BITS;
-
- if (va & ARCH_SECTION_MASK)
- {
- return MMU_MAP_ERROR_VANOTALIGN;
- }
- if (pa & ARCH_SECTION_MASK)
- {
- return MMU_MAP_ERROR_PANOTALIGN;
- }
- for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++)
- {
- off = (va >> level_shift);
- off &= MMU_LEVEL_MASK;
- if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
- {
- page = (unsigned long)rt_pages_alloc(0);
- if (!page)
- {
- return MMU_MAP_ERROR_NOPAGE;
- }
- rt_memset((char *)page, 0, ARCH_PAGE_SIZE);
- rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE);
- cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE;
- rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
- }
- else
- {
- page = cur_lv_tbl[off];
- page &= MMU_ADDRESS_MASK;
- /* page to va */
- page -= PV_OFFSET;
- rt_page_ref_inc((void *)page, 0);
- }
- page = cur_lv_tbl[off];
- if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
- {
- /* is block! error! */
- return MMU_MAP_ERROR_CONFLICT;
- }
- cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
- cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
- level_shift -= MMU_LEVEL_SHIFT;
- }
- attr &= MMU_ATTRIB_MASK;
- pa |= (attr | MMU_TYPE_BLOCK); /* block */
- off = (va >> ARCH_SECTION_SHIFT);
- off &= MMU_LEVEL_MASK;
- cur_lv_tbl[off] = pa;
- rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
-
- return 0;
-}
+volatile unsigned long MMUTable[512] __attribute__((aligned(4 * 1024)));
struct mmu_level_info
{
@@ -254,7 +53,6 @@ struct mmu_level_info
void *page;
};
-#ifdef RT_USING_LWP
static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr)
{
int level;
@@ -313,17 +111,20 @@ static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr)
}
level--;
}
- asm volatile("tlbi vae1, %0\ndsb sy"::"r"(v_addr):"memory");
+
return;
}
-static int _kenrel_map_4K(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long attr)
+static int _kenrel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr,
+ unsigned long attr)
{
int ret = 0;
int level;
unsigned long *cur_lv_tbl = lv0_tbl;
unsigned long page;
unsigned long off;
+ intptr_t va = (intptr_t)vaddr;
+ intptr_t pa = (intptr_t)paddr;
int level_shift = MMU_ADDRESS_BITS;
if (va & ARCH_PAGE_MASK)
@@ -347,9 +148,11 @@ static int _kenrel_map_4K(unsigned long *lv0_tbl, unsigned long va, unsigned lon
goto err;
}
rt_memset((void *)page, 0, ARCH_PAGE_SIZE);
- rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE);
+ rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page,
+ ARCH_PAGE_SIZE);
cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE;
- rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
+ rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off,
+ sizeof(void *));
}
else
{
@@ -382,139 +185,156 @@ err:
_kenrel_unmap_4K(lv0_tbl, (void *)va);
return ret;
}
-#endif
-static int _kernel_map_fixed(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long count, unsigned long attr)
+void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
+ size_t attr)
{
- unsigned long i;
- int ret;
- unsigned long _attr = MMU_MAP_CUSTOM(MMU_AP_KAUN, attr);
+ int ret = -1;
- if (va & ARCH_SECTION_MASK)
- {
- return -1;
- }
- if (pa & ARCH_SECTION_MASK)
- {
- return -1;
- }
- for (i = 0; i < count; i++)
+ void *unmap_va = v_addr;
+ size_t npages = size >> ARCH_PAGE_SHIFT;
+
+ // TODO trying with HUGEPAGE here
+ while (npages--)
{
- ret = _kenrel_map_2M(lv0_tbl, va, pa, _attr);
- va += ARCH_SECTION_SIZE;
- pa += ARCH_SECTION_SIZE;
+ ret = _kenrel_map_4K(aspace->page_table, v_addr, p_addr, attr);
if (ret != 0)
{
- return ret;
+ /* error, undo map */
+ while (unmap_va != v_addr)
+ {
+ _kenrel_unmap_4K(aspace->page_table, (void *)unmap_va);
+ unmap_va += ARCH_PAGE_SIZE;
+ }
+ break;
}
+ v_addr += ARCH_PAGE_SIZE;
+ p_addr += ARCH_PAGE_SIZE;
}
- return 0;
-}
-/************ setting el1 mmu register**************
- MAIR_EL1
- index 0 : memory outer writeback, write/read alloc
- index 1 : memory nocache
- index 2 : device nGnRnE
- *****************************************************/
-void mmu_tcr_init(void)
-{
- unsigned long val64;
-
- val64 = 0x00447fUL;
- __asm__ volatile("msr MAIR_EL1, %0\n dsb sy\n"::"r"(val64));
-
- /* TCR_EL1 */
- val64 = (16UL << 0) /* t0sz 48bit */
- | (0x0UL << 6) /* reserved */
- | (0x0UL << 7) /* epd0 */
- | (0x3UL << 8) /* t0 wb cacheable */
- | (0x3UL << 10) /* inner shareable */
- | (0x2UL << 12) /* t0 outer shareable */
- | (0x0UL << 14) /* t0 4K */
- | (16UL << 16) /* t1sz 48bit */
- | (0x0UL << 22) /* define asid use ttbr0.asid */
- | (0x0UL << 23) /* epd1 */
- | (0x3UL << 24) /* t1 inner wb cacheable */
- | (0x3UL << 26) /* t1 outer wb cacheable */
- | (0x2UL << 28) /* t1 outer shareable */
- | (0x2UL << 30) /* t1 4k */
- | (0x1UL << 32) /* 001b 64GB PA */
- | (0x0UL << 35) /* reserved */
- | (0x1UL << 36) /* as: 0:8bit 1:16bit */
- | (0x0UL << 37) /* tbi0 */
- | (0x0UL << 38); /* tbi1 */
- __asm__ volatile("msr TCR_EL1, %0\n"::"r"(val64));
-}
-
-/* dump 2nd level page table */
-void rt_hw_cpu_dump_page_table_2nd(rt_uint32_t *ptb)
-{
-}
+ if (ret == 0)
+ {
+ return unmap_va;
+ }
-void rt_hw_cpu_dump_page_table(rt_uint32_t *ptb)
-{
+ return NULL;
}
-volatile unsigned long MMUTable[512] __attribute__((aligned(4 * 1024)));
-void rt_hw_mmu_setmtt(unsigned long vaddrStart,
- unsigned long vaddrEnd,
- unsigned long paddrStart,
- unsigned long attr)
+void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size)
{
- unsigned long count;
+ // caller guarantee that v_addr & size are page aligned
+ size_t npages = size >> ARCH_PAGE_SHIFT;
- if (vaddrStart & ARCH_SECTION_MASK)
+ if (!aspace->page_table)
{
- while (1);
- }
- if (paddrStart & ARCH_SECTION_MASK)
- {
- while (1);
- }
- if (vaddrStart > vaddrEnd)
- {
- while (1);
+ return;
}
- count = vaddrEnd + 1;
- if (count & ARCH_SECTION_MASK)
+
+ while (npages--)
{
- while (1);
+ _kenrel_unmap_4K(aspace->page_table, v_addr);
+ v_addr += ARCH_PAGE_SIZE;
}
- count -= vaddrStart;
- if (count == 0)
+}
+
+void rt_hw_aspace_switch(rt_aspace_t aspace)
+{
+ if (aspace != &rt_kernel_space)
{
- while (1);
+ void *pgtbl = aspace->page_table;
+ pgtbl = _rt_kmem_v2p(pgtbl);
+ uintptr_t tcr;
+
+ __asm__ volatile("msr ttbr0_el1, %0" ::"r"(pgtbl) : "memory");
+
+ __asm__ volatile("mrs %0, tcr_el1" : "=r"(tcr));
+ tcr &= ~(1ul << 7);
+ __asm__ volatile("msr tcr_el1, %0\n"
+ "isb" ::"r"(tcr)
+ : "memory");
+
+ rt_hw_tlb_invalidate_all_local();
}
- count >>= ARCH_SECTION_SHIFT;
- _kernel_map_fixed((unsigned long *)MMUTable, vaddrStart, paddrStart, count, attr);
}
void rt_hw_mmu_ktbl_set(unsigned long tbl)
{
#ifdef RT_USING_SMART
tbl += PV_OFFSET;
- __asm__ volatile("msr TTBR1_EL1, %0\n dsb sy\nisb"::"r"(tbl):"memory");
+ __asm__ volatile("msr TTBR1_EL1, %0\n dsb sy\nisb" ::"r"(tbl) : "memory");
#else
- __asm__ volatile("msr TTBR0_EL1, %0\n dsb sy\nisb"::"r"(tbl):"memory");
+ __asm__ volatile("msr TTBR0_EL1, %0\n dsb sy\nisb" ::"r"(tbl) : "memory");
#endif
- __asm__ volatile("tlbi vmalle1\n dsb sy\nisb":::"memory");
- __asm__ volatile("ic ialluis\n dsb sy\nisb":::"memory");
+ __asm__ volatile("tlbi vmalle1\n dsb sy\nisb" ::: "memory");
+ __asm__ volatile("ic ialluis\n dsb sy\nisb" ::: "memory");
}
-void rt_hw_mmu_setup(struct mem_desc *mdesc, int desc_nr)
+/**
+ * @brief setup Page Table for kernel space. It's a fixed map
+ * and all mappings cannot be changed after initialization.
+ *
+ * Memory region in struct mem_desc must be page aligned,
+ * otherwise is a failure and no report will be
+ * returned.
+ *
+ * @param mmu_info
+ * @param mdesc
+ * @param desc_nr
+ */
+void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr)
{
- /* set page table */
- for (; desc_nr > 0; desc_nr--)
+ void *err;
+ for (size_t i = 0; i < desc_nr; i++)
{
- rt_hw_mmu_setmtt(mdesc->vaddr_start, mdesc->vaddr_end,
- mdesc->paddr_start, mdesc->attr);
+ size_t attr;
+ switch (mdesc->attr)
+ {
+ case NORMAL_MEM:
+ attr = MMU_MAP_K_RWCB;
+ break;
+ case NORMAL_NOCACHE_MEM:
+ attr = MMU_MAP_K_RWCB;
+ break;
+ case DEVICE_MEM:
+ attr = MMU_MAP_K_DEVICE;
+ break;
+ default:
+ attr = MMU_MAP_K_DEVICE;
+ }
+
+ struct rt_mm_va_hint hint = {.flags = MMF_MAP_FIXED,
+ .limit_start = aspace->start,
+ .limit_range_size = aspace->size,
+ .map_size = mdesc->vaddr_end -
+ mdesc->vaddr_start + 1,
+ .prefer = (void *)mdesc->vaddr_start};
+
+ rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr,
+ mdesc->paddr_start >> MM_PAGE_SHIFT, &err);
mdesc++;
}
- rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)MMUTable, sizeof MMUTable);
- rt_hw_mmu_ktbl_set((unsigned long)MMUTable);
+
+ rt_hw_mmu_ktbl_set((unsigned long)rt_kernel_space.page_table);
+ rt_page_cleanup();
}
+
+#ifdef RT_USING_SMART
+static inline void _init_region(void *vaddr, size_t size)
+{
+ rt_ioremap_start = vaddr;
+ rt_ioremap_size = size;
+ rt_mpr_start = rt_ioremap_start - rt_mpr_size;
+}
+#else
+
+#define RTOS_VEND ((void *)0xfffffffff000UL)
+static inline void _init_region(void *vaddr, size_t size)
+{
+ rt_mpr_start = RTOS_VEND - rt_mpr_size;
+}
+#endif
+
/**
* This function will initialize rt_mmu_info structure.
*
@@ -526,11 +346,12 @@ void rt_hw_mmu_setup(struct mem_desc *mdesc, int desc_nr)
*
* @return 0 on successful and -1 for fail
*/
-int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void *v_address, size_t size, size_t *vtable, size_t pv_off)
+int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, size_t size,
+ size_t *vtable, size_t pv_off)
{
size_t va_s, va_e;
- if (!mmu_info || !vtable)
+ if (!aspace || !vtable)
{
return -1;
}
@@ -551,463 +372,292 @@ int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void *v_address, size_t size, size
return -1;
}
- mmu_info->vtable = vtable;
- mmu_info->vstart = va_s;
- mmu_info->vend = va_e;
- mmu_info->pv_off = pv_off;
+#ifdef RT_USING_SMART
+ rt_aspace_init(aspace, (void *)KERNEL_VADDR_START, 0 - KERNEL_VADDR_START,
+ vtable);
+#else
+ rt_aspace_init(aspace, (void *)0x1000, RTOS_VEND - (void *)0x1000, vtable);
+#endif
+
+ _init_region(v_address, size);
return 0;
}
-int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void *v_address, size_t size)
+/************ setting el1 mmu register**************
+ MAIR_EL1
+ index 0 : memory outer writeback, write/read alloc
+ index 1 : memory nocache
+ index 2 : device nGnRnE
+ *****************************************************/
+void mmu_tcr_init(void)
{
- return 0;
+ unsigned long val64;
+
+ val64 = 0x00447fUL;
+ __asm__ volatile("msr MAIR_EL1, %0\n dsb sy\n" ::"r"(val64));
+
+ /* TCR_EL1 */
+ val64 = (16UL << 0) /* t0sz 48bit */
+ | (0x0UL << 6) /* reserved */
+ | (0x0UL << 7) /* epd0 */
+ | (0x3UL << 8) /* t0 wb cacheable */
+ | (0x3UL << 10) /* inner shareable */
+ | (0x2UL << 12) /* t0 outer shareable */
+ | (0x0UL << 14) /* t0 4K */
+ | (16UL << 16) /* t1sz 48bit */
+ | (0x0UL << 22) /* define asid use ttbr0.asid */
+ | (0x0UL << 23) /* epd1 */
+ | (0x3UL << 24) /* t1 inner wb cacheable */
+ | (0x3UL << 26) /* t1 outer wb cacheable */
+ | (0x2UL << 28) /* t1 outer shareable */
+ | (0x2UL << 30) /* t1 4k */
+ | (0x1UL << 32) /* 001b 64GB PA */
+ | (0x0UL << 35) /* reserved */
+ | (0x1UL << 36) /* as: 0:8bit 1:16bit */
+ | (0x0UL << 37) /* tbi0 */
+ | (0x0UL << 38); /* tbi1 */
+ __asm__ volatile("msr TCR_EL1, %0\n" ::"r"(val64));
}
-#ifdef RT_USING_SMART
-static size_t find_vaddr(rt_mmu_info *mmu_info, int pages)
+struct page_table
{
- size_t loop_pages;
- size_t va;
- size_t find_va = 0;
- int n = 0;
- size_t i;
+ unsigned long page[512];
+};
- if (!pages)
+static struct page_table *__init_page_array;
+static unsigned long __page_off = 0UL;
+unsigned long get_free_page(void)
+{
+ if (!__init_page_array)
{
- return 0;
+ unsigned long temp_page_start;
+ asm volatile("mov %0, sp" : "=r"(temp_page_start));
+ __init_page_array =
+ (struct page_table *)(temp_page_start & ~(ARCH_SECTION_MASK));
+ __page_off = 2; /* 0, 1 for ttbr0, ttrb1 */
}
+ __page_off++;
+ return (unsigned long)(__init_page_array[__page_off - 1].page);
+}
- if (!mmu_info)
+static int _map_single_page_2M(unsigned long *lv0_tbl, unsigned long va,
+ unsigned long pa, unsigned long attr)
+{
+ int level;
+ unsigned long *cur_lv_tbl = lv0_tbl;
+ unsigned long page;
+ unsigned long off;
+ int level_shift = MMU_ADDRESS_BITS;
+
+ if (va & ARCH_SECTION_MASK)
{
- return 0;
+ return MMU_MAP_ERROR_VANOTALIGN;
}
-
- loop_pages = mmu_info->vend - mmu_info->vstart + 1;
- loop_pages <<= (ARCH_SECTION_SHIFT - ARCH_PAGE_SHIFT);
- va = mmu_info->vstart;
- va <<= ARCH_SECTION_SHIFT;
- for (i = 0; i < loop_pages; i++, va += ARCH_PAGE_SIZE)
+ if (pa & ARCH_SECTION_MASK)
{
- if (_rt_hw_mmu_v2p(mmu_info, (void *)va))
- {
- n = 0;
- find_va = 0;
- continue;
- }
- if (!find_va)
+ return MMU_MAP_ERROR_PANOTALIGN;
+ }
+ for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++)
+ {
+ off = (va >> level_shift);
+ off &= MMU_LEVEL_MASK;
+ if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
{
- find_va = va;
+ page = get_free_page();
+ if (!page)
+ {
+ return MMU_MAP_ERROR_NOPAGE;
+ }
+ rt_memset((char *)page, 0, ARCH_PAGE_SIZE);
+ cur_lv_tbl[off] = page | MMU_TYPE_TABLE;
}
- n++;
- if (n >= pages)
+ page = cur_lv_tbl[off];
+ if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
{
- return find_va;
+ /* is block! error! */
+ return MMU_MAP_ERROR_CONFLICT;
}
+ cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
+ level_shift -= MMU_LEVEL_SHIFT;
}
+ attr &= MMU_ATTRIB_MASK;
+ pa |= (attr | MMU_TYPE_BLOCK); /* block */
+ off = (va >> ARCH_SECTION_SHIFT);
+ off &= MMU_LEVEL_MASK;
+ cur_lv_tbl[off] = pa;
return 0;
}
-static int check_vaddr(rt_mmu_info *mmu_info, void *va, int pages)
+static int _init_map_2M(unsigned long *lv0_tbl, unsigned long va,
+ unsigned long pa, unsigned long count,
+ unsigned long attr)
{
- size_t loop_va;
-
- if (!pages)
- {
- return -1;
- }
-
- if (!mmu_info)
- {
- return -1;
- }
+ unsigned long i;
+ int ret;
- loop_va = ((size_t)va >> ARCH_SECTION_SHIFT);
- if (loop_va < mmu_info->vstart || loop_va > mmu_info->vend)
+ if (va & ARCH_SECTION_MASK)
{
return -1;
}
- loop_va += ((pages << ARCH_PAGE_SHIFT) >> ARCH_SECTION_SHIFT);
- if (loop_va < mmu_info->vstart || loop_va > mmu_info->vend + 1)
+ if (pa & ARCH_SECTION_MASK)
{
return -1;
}
-
- loop_va = (size_t)va & ~ARCH_PAGE_MASK;
- while (pages--)
+ for (i = 0; i < count; i++)
{
- if (_rt_hw_mmu_v2p(mmu_info, (void *)loop_va))
+ ret = _map_single_page_2M(lv0_tbl, va, pa, attr);
+ va += ARCH_SECTION_SIZE;
+ pa += ARCH_SECTION_SIZE;
+ if (ret != 0)
{
- return -1;
+ return ret;
}
- loop_va += ARCH_PAGE_SIZE;
}
return 0;
}
-static void __rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, size_t npages)
+static unsigned long *_query(rt_aspace_t aspace, void *vaddr, int *plvl_shf)
{
- size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK;
+ int level;
+ unsigned long va = (unsigned long)vaddr;
+ unsigned long *cur_lv_tbl;
+ unsigned long page;
+ unsigned long off;
+ int level_shift = MMU_ADDRESS_BITS;
- if (!mmu_info || !mmu_info->vtable)
- {
- return;
- }
+ cur_lv_tbl = aspace->page_table;
+ RT_ASSERT(cur_lv_tbl);
- while (npages--)
+ for (level = 0; level < MMU_TBL_PAGE_4k_LEVEL; level++)
{
- _kenrel_unmap_4K(mmu_info->vtable, (void *)loop_va);
- loop_va += ARCH_PAGE_SIZE;
- }
-}
+ off = (va >> level_shift);
+ off &= MMU_LEVEL_MASK;
-static int __rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, size_t npages, size_t attr)
-{
- int ret = -1;
- size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK;
- size_t loop_pa = (size_t)p_addr & ~ARCH_PAGE_MASK;
- size_t unmap_va = loop_va;
+ if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
+ {
+ return (void *)0;
+ }
- if (mmu_info)
- {
- while (npages--)
+ page = cur_lv_tbl[off];
+ if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
{
- ret = _kenrel_map_4K(mmu_info->vtable, loop_va, loop_pa, attr);
- if (ret != 0)
- {
- /* error, undo map */
- while (unmap_va != loop_va)
- {
- _kenrel_unmap_4K(mmu_info->vtable, (void *)unmap_va);
- unmap_va += ARCH_PAGE_SIZE;
- }
- break;
- }
- loop_va += ARCH_PAGE_SIZE;
- loop_pa += ARCH_PAGE_SIZE;
+ *plvl_shf = level_shift;
+ return &cur_lv_tbl[off];
}
+
+ cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
+ cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
+ level_shift -= MMU_LEVEL_SHIFT;
}
- return ret;
-}
-#endif
+ /* now is level MMU_TBL_PAGE_4k_LEVEL */
+ off = (va >> ARCH_PAGE_SHIFT);
+ off &= MMU_LEVEL_MASK;
+ page = cur_lv_tbl[off];
-static void rt_hw_cpu_tlb_invalidate(void)
-{
- __asm__ volatile("tlbi vmalle1\n dsb sy\n isb sy\n");
+ if (!(page & MMU_TYPE_USED))
+ {
+ return (void *)0;
+ }
+ *plvl_shf = level_shift;
+ return &cur_lv_tbl[off];
}
-#ifdef RT_USING_SMART
-void *_rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, size_t size, size_t attr)
+void *rt_hw_mmu_v2p(rt_aspace_t aspace, void *v_addr)
{
- size_t pa_s, pa_e;
- size_t vaddr;
- int pages;
- int ret;
+ int level_shift;
+ unsigned long paddr;
+ unsigned long *pte = _query(aspace, v_addr, &level_shift);
- if (!size)
+ if (pte)
{
- return 0;
- }
- pa_s = (size_t)p_addr;
- pa_e = (size_t)p_addr + size - 1;
- pa_s >>= ARCH_PAGE_SHIFT;
- pa_e >>= ARCH_PAGE_SHIFT;
- pages = pa_e - pa_s + 1;
- if (v_addr)
- {
- vaddr = (size_t)v_addr;
- pa_s = (size_t)p_addr;
- if ((vaddr & ARCH_PAGE_MASK) != (pa_s & ARCH_PAGE_MASK))
- {
- return 0;
- }
- vaddr &= ~ARCH_PAGE_MASK;
- if (check_vaddr(mmu_info, (void *)vaddr, pages) != 0)
- {
- return 0;
- }
+ paddr = *pte & MMU_ADDRESS_MASK;
+ paddr |= (uintptr_t)v_addr & ((1ul << level_shift) - 1);
}
else
{
- vaddr = find_vaddr(mmu_info, pages);
+ paddr = (unsigned long)ARCH_MAP_FAILED;
}
- if (vaddr)
- {
- rt_enter_critical();
- ret = __rt_hw_mmu_map(mmu_info, (void *)vaddr, p_addr, pages, attr);
- if (ret == 0)
- {
- rt_hw_cpu_tlb_invalidate();
- rt_exit_critical();
- return (void *)(vaddr + ((size_t)p_addr & ARCH_PAGE_MASK));
- }
- rt_exit_critical();
- }
- return 0;
+ return (void *)paddr;
}
-#else
-void *_rt_hw_mmu_map(rt_mmu_info *mmu_info, void* p_addr, size_t size, size_t attr)
-{
- return p_addr;
-}
-#endif
-#ifdef RT_USING_SMART
-static int __rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t npages, size_t attr)
+static int _noncache(uintptr_t *pte)
{
- size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK;
- size_t loop_pa;
-
- if (!mmu_info)
+ int err = 0;
+ const uintptr_t idx_shift = 2;
+ const uintptr_t idx_mask = 0x7 << idx_shift;
+ uintptr_t entry = *pte;
+ if ((entry & idx_mask) == (NORMAL_MEM << idx_shift))
{
- return -1;
- }
-
- while (npages--)
- {
- loop_pa = (size_t)rt_pages_alloc(0);
- if (!loop_pa)
- {
- goto err;
- }
- loop_pa += mmu_info->pv_off;
- _kenrel_map_4K(mmu_info->vtable, loop_va, loop_pa, attr);
- loop_va += ARCH_PAGE_SIZE;
+ *pte = (entry & ~idx_mask) | (NORMAL_NOCACHE_MEM << idx_shift);
}
- return 0;
-err:
+ else
{
- /* error, unmap and quit */
- int i;
- void *va, *pa;
-
- va = (void *)((size_t)v_addr & ~ARCH_PAGE_MASK);
- for (i = 0; i < npages; i++)
- {
- pa = rt_hw_mmu_v2p(mmu_info, va);
- pa = (void *)((char *)pa - mmu_info->pv_off);
- rt_pages_free(pa, 0);
- va = (void *)((char *)va + ARCH_PAGE_SIZE);
- }
-
- __rt_hw_mmu_unmap(mmu_info, v_addr, npages);
- return -1;
+ // do not support other type to be noncache
+ err = RT_ENOSYS;
}
+ return err;
}
-void *_rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr)
+static int _cache(uintptr_t *pte)
{
- size_t vaddr;
- size_t offset;
- int pages;
- int ret;
-
- if (!size)
+ int err = 0;
+ const uintptr_t idx_shift = 2;
+ const uintptr_t idx_mask = 0x7 << idx_shift;
+ uintptr_t entry = *pte;
+ if ((entry & idx_mask) == (NORMAL_NOCACHE_MEM << idx_shift))
{
- return 0;
- }
- offset = (size_t)v_addr & ARCH_PAGE_MASK;
- size += (offset + ARCH_PAGE_SIZE - 1);
- pages = (size >> ARCH_PAGE_SHIFT);
- if (v_addr)
- {
- vaddr = (size_t)v_addr;
- vaddr &= ~ARCH_PAGE_MASK;
- if (check_vaddr(mmu_info, (void *)vaddr, pages) != 0)
- {
- return 0;
- }
+ *pte = (entry & ~idx_mask) | (NORMAL_MEM << idx_shift);
}
else
{
- vaddr = find_vaddr(mmu_info, pages);
+ // do not support other type to be cache
+ err = -RT_ENOSYS;
}
- if (vaddr)
- {
- rt_enter_critical();
- ret = __rt_hw_mmu_map_auto(mmu_info, (void *)vaddr, pages, attr);
- if (ret == 0)
- {
- rt_hw_cpu_tlb_invalidate();
- rt_exit_critical();
- return (void *)((char *)vaddr + offset);
- }
- rt_exit_critical();
- }
- return 0;
-}
-
-void _rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, size_t size)
-{
- size_t va_s, va_e;
- int pages;
-
- va_s = (size_t)v_addr;
- va_e = (size_t)v_addr + size - 1;
- va_s >>= ARCH_PAGE_SHIFT;
- va_e >>= ARCH_PAGE_SHIFT;
- pages = va_e - va_s + 1;
- rt_enter_critical();
- __rt_hw_mmu_unmap(mmu_info, v_addr, pages);
- rt_hw_cpu_tlb_invalidate();
- rt_exit_critical();
+ return err;
}
-void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, size_t size, size_t attr)
-{
- void *ret;
-
- rt_mm_lock();
- ret = _rt_hw_mmu_map(mmu_info, v_addr, p_addr, size, attr);
- rt_mm_unlock();
- return ret;
-}
-
-void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr)
-{
- void *ret;
-
- rt_mm_lock();
- ret = _rt_hw_mmu_map_auto(mmu_info, v_addr, size, attr);
- rt_mm_unlock();
- return ret;
-}
-
-void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, size_t size)
-{
- rt_mm_lock();
- _rt_hw_mmu_unmap(mmu_info, v_addr, size);
- rt_mm_unlock();
-}
-#else
-
-#include <cache.h>
+static int (*control_handler[MMU_CNTL_DUMMY_END])(uintptr_t *pte) = {
+ [MMU_CNTL_CACHE] = _cache,
+ [MMU_CNTL_NONCACHE] = _noncache,
+};
-void rt_hw_init_mmu_table(struct mem_desc *mdesc, rt_size_t desc_nr)
+int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
+ enum rt_mmu_cntl cmd)
{
- rt_memset((void *)MMUTable, 0, sizeof(MMUTable));
- rt_memset((void *)MMUPage, 0, sizeof(MMUPage));
+ int level_shift;
+ int err = -RT_EINVAL;
+ void *vend = vaddr + size;
- /* set page table */
- for (; desc_nr > 0; --desc_nr)
+ int (*handler)(uintptr_t * pte);
+ if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END)
{
- rt_hw_mmu_setmtt(mdesc->vaddr_start, mdesc->vaddr_end, mdesc->paddr_start, mdesc->attr);
- ++mdesc;
- }
-
- rt_hw_cpu_dcache_clean((void *)MMUTable, sizeof(MMUTable));
-}
-
+ handler = control_handler[cmd];
-void rt_hw_mmu_init(void)
-{
- unsigned long reg_val;
-
- reg_val = 0x00447fUL;
- __asm__ volatile("msr mair_el1, %0"::"r"(reg_val));
-
- rt_hw_isb();
-
- reg_val = (16UL << 0) /* t0sz 48bit */
- | (0UL << 6) /* reserved */
- | (0UL << 7) /* epd0 */
- | (3UL << 8) /* t0 wb cacheable */
- | (3UL << 10) /* inner shareable */
- | (2UL << 12) /* t0 outer shareable */
- | (0UL << 14) /* t0 4K */
- | (16UL << 16) /* t1sz 48bit */
- | (0UL << 22) /* define asid use ttbr0.asid */
- | (0UL << 23) /* epd1 */
- | (3UL << 24) /* t1 inner wb cacheable */
- | (3UL << 26) /* t1 outer wb cacheable */
- | (2UL << 28) /* t1 outer shareable */
- | (2UL << 30) /* t1 4k */
- | (1UL << 32) /* 001b 64GB PA */
- | (0UL << 35) /* reserved */
- | (1UL << 36) /* as: 0:8bit 1:16bit */
- | (0UL << 37) /* tbi0 */
- | (0UL << 38); /* tbi1 */
- __asm__ volatile("msr tcr_el1, %0"::"r"(reg_val));
-
- rt_hw_isb();
-
- __asm__ volatile ("mrs %0, sctlr_el1":"=r"(reg_val));
-
- reg_val |= 1 << 2; /* enable dcache */
- reg_val |= 1 << 0; /* enable mmu */
-
- __asm__ volatile (
- "msr ttbr0_el1, %0\n\r"
- "msr sctlr_el1, %1\n\r"
- "dsb sy\n\r"
- "isb sy\n\r"
- ::"r"(MMUTable), "r"(reg_val) :"memory");
-
- rt_hw_cpu_tlb_invalidate();
-}
-#endif
-
-void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr)
-{
- int level;
- unsigned long va = (unsigned long)v_addr;
- unsigned long pa;
- unsigned long *cur_lv_tbl;
- unsigned long page;
- unsigned long off;
- unsigned long off_addr;
- int level_shift = MMU_ADDRESS_BITS;
-
- if (!mmu_info)
- {
- return (void *)0;
- }
- cur_lv_tbl = mmu_info->vtable;
- for (level = 0; level < MMU_TBL_PAGE_4k_LEVEL; level++)
- {
- off = (va >> level_shift);
- off &= MMU_LEVEL_MASK;
- if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
- {
- return (void *)0;
- }
- page = cur_lv_tbl[off];
- if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
+ while (vaddr < vend)
{
- off_addr = va & ((1UL << level_shift) - 1);
- pa = (page & MMU_ADDRESS_MASK);
- pa += off_addr;
- return (void *)pa;
+ uintptr_t *pte = _query(aspace, vaddr, &level_shift);
+ void *range_end = vaddr + (1ul << level_shift);
+ RT_ASSERT(range_end <= vend);
+
+ if (pte)
+ {
+ err = handler(pte);
+ RT_ASSERT(err == RT_EOK);
+ }
+ vaddr = range_end;
}
- cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
- cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
- level_shift -= MMU_LEVEL_SHIFT;
}
- /* now is level MMU_TBL_PAGE_4k_LEVEL */
- off = (va >> ARCH_PAGE_SHIFT);
- off &= MMU_LEVEL_MASK;
- page = cur_lv_tbl[off];
- if (!(page & MMU_TYPE_USED))
+ else
{
- return (void *)0;
+ err = -RT_ENOSYS;
}
- pa = (page & MMU_ADDRESS_MASK);
- pa += (va & ARCH_PAGE_MASK);
- return (void *)pa;
-}
-void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr)
-{
- void *ret;
-
- rt_mm_lock();
- ret = _rt_hw_mmu_v2p(mmu_info, v_addr);
- rt_mm_unlock();
- return ret;
+ return err;
}
-
-void rt_hw_mmu_setup_early(unsigned long *tbl0, unsigned long *tbl1, unsigned long size, unsigned long pv_off)
+void rt_hw_mmu_setup_early(unsigned long *tbl0, unsigned long *tbl1,
+ unsigned long size, unsigned long pv_off)
{
int ret;
unsigned long va = KERNEL_VADDR_START;
@@ -1015,15 +665,15 @@ void rt_hw_mmu_setup_early(unsigned long *tbl0, unsigned long *tbl1, unsigned lo
unsigned long normal_attr = MMU_MAP_CUSTOM(MMU_AP_KAUN, NORMAL_MEM);
/* clean the first two pages */
- mmu_memset((char *)tbl0, 0, ARCH_PAGE_SIZE);
- mmu_memset((char *)tbl1, 0, ARCH_PAGE_SIZE);
+ rt_memset((char *)tbl0, 0, ARCH_PAGE_SIZE);
+ rt_memset((char *)tbl1, 0, ARCH_PAGE_SIZE);
- ret = armv8_init_map_2M(tbl1, va, va + pv_off, count, normal_attr);
+ ret = _init_map_2M(tbl1, va, va + pv_off, count, normal_attr);
if (ret != 0)
{
while (1);
}
- ret = armv8_init_map_2M(tbl0, va + pv_off, va + pv_off, count, normal_attr);
+ ret = _init_map_2M(tbl0, va + pv_off, va + pv_off, count, normal_attr);
if (ret != 0)
{
while (1);
diff --git a/libcpu/aarch64/common/mmu.h b/libcpu/aarch64/common/mmu.h
index 0e61eeac70..6dc8cdca31 100644
--- a/libcpu/aarch64/common/mmu.h
+++ b/libcpu/aarch64/common/mmu.h
@@ -11,6 +11,7 @@
#define __MMU_H_
#include <rtthread.h>
+#include <mm_aspace.h>
/* normal memory wra mapping type */
#define NORMAL_MEM 0
@@ -25,6 +26,7 @@ struct mem_desc
unsigned long vaddr_end;
unsigned long paddr_start;
unsigned long attr;
+ struct rt_varea varea;
};
#define MMU_AF_SHIFT 10
@@ -37,60 +39,17 @@ struct mem_desc
#define MMU_AP_KRUN 2UL /* kernel r, user none */
#define MMU_AP_KRUR 3UL /* kernel r, user r */
-#define MMU_MAP_K_RO (\
- (0x1UL << MMU_AF_SHIFT) |\
- (0x2UL << MMU_SHARED_SHIFT) |\
- (MMU_AP_KRUN << MMU_AP_SHIFT) |\
- (NORMAL_MEM << MMU_MA_SHIFT)\
- )
-#define MMU_MAP_K_RWCB (\
- (0x1UL << MMU_AF_SHIFT) |\
- (0x2UL << MMU_SHARED_SHIFT) |\
- (MMU_AP_KAUN << MMU_AP_SHIFT) |\
- (NORMAL_MEM << MMU_MA_SHIFT)\
- )
-#define MMU_MAP_K_RW (\
- (0x1UL << MMU_AF_SHIFT) |\
- (0x2UL << MMU_SHARED_SHIFT) |\
- (MMU_AP_KAUN << MMU_AP_SHIFT) |\
- (NORMAL_NOCACHE_MEM << MMU_MA_SHIFT)\
- )
-#define MMU_MAP_K_DEVICE (\
- (0x1UL << MMU_AF_SHIFT) |\
- (0x2UL << MMU_SHARED_SHIFT) |\
- (MMU_AP_KAUN << MMU_AP_SHIFT) |\
- (DEVICE_MEM << MMU_MA_SHIFT)\
- )
-#define MMU_MAP_U_RO (\
- (0x1UL << MMU_AF_SHIFT) |\
- (0x2UL << MMU_SHARED_SHIFT) |\
- (MMU_AP_KRUR << MMU_AP_SHIFT) |\
- (NORMAL_NOCACHE_MEM << MMU_MA_SHIFT)\
- )
-#define MMU_MAP_U_RWCB (\
- (0x1UL << MMU_AF_SHIFT) |\
- (0x2UL << MMU_SHARED_SHIFT) |\
- (MMU_AP_KAUA << MMU_AP_SHIFT) |\
- (NORMAL_MEM << MMU_MA_SHIFT)\
- )
-#define MMU_MAP_U_RW (\
- (0x1UL << MMU_AF_SHIFT) |\
- (0x2UL << MMU_SHARED_SHIFT) |\
- (MMU_AP_KAUA << MMU_AP_SHIFT) |\
- (NORMAL_NOCACHE_MEM << MMU_MA_SHIFT)\
- )
-#define MMU_MAP_U_DEVICE (\
- (0x1UL << MMU_AF_SHIFT) |\
- (0x2UL << MMU_SHARED_SHIFT) |\
- (MMU_AP_KAUA << MMU_AP_SHIFT) |\
- (DEVICE_MEM << MMU_MA_SHIFT)\
- )
-#define MMU_MAP_CUSTOM(ap, mtype) (\
- (0x1UL << MMU_AF_SHIFT) |\
- (0x2UL << MMU_SHARED_SHIFT) |\
- ((ap) << MMU_AP_SHIFT) |\
- ((mtype) << MMU_MA_SHIFT)\
- )
+#define MMU_MAP_CUSTOM(ap, mtype) \
+ ((0x1UL << MMU_AF_SHIFT) | (0x2UL << MMU_SHARED_SHIFT) | \
+ ((ap) << MMU_AP_SHIFT) | ((mtype) << MMU_MA_SHIFT))
+#define MMU_MAP_K_RO MMU_MAP_CUSTOM(MMU_AP_KRUN, NORMAL_MEM)
+#define MMU_MAP_K_RWCB MMU_MAP_CUSTOM(MMU_AP_KAUN, NORMAL_MEM)
+#define MMU_MAP_K_RW MMU_MAP_CUSTOM(MMU_AP_KAUN, NORMAL_NOCACHE_MEM)
+#define MMU_MAP_K_DEVICE MMU_MAP_CUSTOM(MMU_AP_KAUN, DEVICE_MEM)
+#define MMU_MAP_U_RO MMU_MAP_CUSTOM(MMU_AP_KRUR, NORMAL_NOCACHE_MEM)
+#define MMU_MAP_U_RWCB MMU_MAP_CUSTOM(MMU_AP_KAUA, NORMAL_MEM)
+#define MMU_MAP_U_RW MMU_MAP_CUSTOM(MMU_AP_KAUA, NORMAL_NOCACHE_MEM)
+#define MMU_MAP_U_DEVICE MMU_MAP_CUSTOM(MMU_AP_KAUA, DEVICE_MEM)
#define ARCH_SECTION_SHIFT 21
#define ARCH_SECTION_SIZE (1 << ARCH_SECTION_SHIFT)
@@ -102,6 +61,7 @@ struct mem_desc
#define ARCH_PAGE_TBL_SIZE (1 << ARCH_PAGE_TBL_SHIFT)
#define ARCH_PAGE_TBL_MASK (ARCH_PAGE_TBL_SIZE - 1)
+#define ARCH_VADDR_WIDTH 48
#define ARCH_ADDRESS_WIDTH_BITS 64
#define MMU_MAP_ERROR_VANOTALIGN -1
@@ -109,37 +69,48 @@ struct mem_desc
#define MMU_MAP_ERROR_NOPAGE -3
#define MMU_MAP_ERROR_CONFLICT -4
-typedef struct
-{
- size_t *vtable;
- size_t vstart;
- size_t vend;
- size_t pv_off;
-} rt_mmu_info;
+#define ARCH_MAP_FAILED ((void *)0x1ffffffffffff)
-void rt_hw_mmu_setup_early(unsigned long *tbl0, unsigned long *tbl1, unsigned long size, unsigned long pv_off);
-void rt_hw_mmu_setup(struct mem_desc *mdesc, int desc_nr);
+struct rt_aspace;
-int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void* v_address, size_t size, size_t *vtable, size_t pv_off);
-int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void* v_address, size_t size);
+void rt_hw_mmu_ktbl_set(unsigned long tbl);
+void rt_hw_mmu_setup_early(unsigned long *tbl0, unsigned long *tbl1,
+ unsigned long size, unsigned long pv_off);
+void rt_hw_mmu_setup(struct rt_aspace *aspace, struct mem_desc *mdesc,
+ int desc_nr);
-#ifdef RT_USING_SMART
-void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void* p_addr, size_t size, size_t attr);
-void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr);
-#else
-void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void* p_addr, size_t size, size_t attr);
-#endif
+int rt_hw_mmu_map_init(struct rt_aspace *aspace, void *v_address,
+ rt_size_t size, rt_size_t *vtable, rt_size_t pv_off);
+void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr,
+ size_t size, size_t attr);
+void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size);
+void rt_hw_aspace_switch(struct rt_aspace *aspace);
+void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr);
+void rt_hw_mmu_kernel_map_init(struct rt_aspace *aspace, rt_size_t vaddr_start,
+ rt_size_t size);
+void rt_hw_mmu_ktbl_set(unsigned long tbl);
-void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void* v_addr, size_t size);
-void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr);
+static inline void *rt_hw_mmu_tbl_get()
+{
+ uintptr_t tbl;
+ __asm__ volatile("MRS %0, TTBR0_EL1" : "=r"(tbl));
+ return (void *)(tbl & ((1ul << 48) - 2));
+}
-void rt_hw_mmu_ktbl_set(unsigned long tbl);
-void *rt_hw_mmu_tbl_get();
-void rt_hw_mmu_switch(void *mmu_table);
+static inline void *_rt_kmem_v2p(void *vaddr)
+{
+ return rt_hw_mmu_v2p(&rt_kernel_space, vaddr);
+}
-void rt_hw_init_mmu_table(struct mem_desc *mdesc, rt_size_t desc_nr);
-void rt_hw_mmu_init(void);
+static inline void *rt_kmem_v2p(void *vaddr)
+{
+ MM_PGTBL_LOCK(&rt_kernel_space);
+ void *paddr = _rt_kmem_v2p(vaddr);
+ MM_PGTBL_UNLOCK(&rt_kernel_space);
+ return paddr;
+}
-extern rt_mmu_info mmu_info;
+int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
+ enum rt_mmu_cntl cmd);
#endif
diff --git a/libcpu/aarch64/common/tlb.h b/libcpu/aarch64/common/tlb.h
new file mode 100644
index 0000000000..088af488fc
--- /dev/null
+++ b/libcpu/aarch64/common/tlb.h
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2006-2022, RT-Thread Development Team
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Change Logs:
+ * Date Author Notes
+ * 2022-11-28 WangXiaoyao the first version
+ */
+#ifndef __TLB_H__
+#define __TLB_H__
+
+#include <rtthread.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "mm_aspace.h"
+#include "mmu.h"
+
+#define TLBI_ARG(addr, asid) \
+ ({ \
+ uintptr_t arg = (uintptr_t)(addr) >> 12; \
+ arg &= (1ull << 44) - 1; \
+ arg |= (uintptr_t)(asid) << 48; \
+ (void *)arg; \
+ })
+
+static inline void rt_hw_tlb_invalidate_all(void)
+{
+ __asm__ volatile(
+ // ensure updates to pte completed
+ "dsb ishst\n"
+ "tlbi vmalle1is\n"
+ "dsb ish\n"
+ // after tlb in new context, refresh inst
+ "isb\n" ::
+ : "memory");
+}
+
+static inline void rt_hw_tlb_invalidate_all_local(void)
+{
+ __asm__ volatile(
+ // ensure updates to pte completed
+ "dsb nshst\n"
+ "tlbi vmalle1is\n"
+ "dsb nsh\n"
+ // after tlb in new context, refresh inst
+ "isb\n" ::
+ : "memory");
+}
+
+static inline void rt_hw_tlb_invalidate_aspace(rt_aspace_t aspace)
+{
+ rt_hw_tlb_invalidate_all_local();
+}
+
+static inline void rt_hw_tlb_invalidate_page(rt_aspace_t aspace, void *start)
+{
+ start = TLBI_ARG(start, 0);
+ __asm__ volatile(
+ "dsb ishst\n"
+ "tlbi vaae1is, %0\n"
+ "dsb ish\n"
+ "isb\n" ::"r"(start)
+ : "memory");
+}
+
+static inline void rt_hw_tlb_invalidate_range(rt_aspace_t aspace, void *start,
+ size_t size, size_t stride)
+{
+ if (size < ARCH_PAGE_SIZE)
+ {
+ rt_hw_tlb_invalidate_page(aspace, start);
+ }
+ else
+ {
+ rt_hw_tlb_invalidate_aspace(aspace);
+ }
+}
+
+#endif /* __TLB_H__ */
diff --git a/libcpu/aarch64/common/trap.c b/libcpu/aarch64/common/trap.c
index 0a449efeb8..0a2dd2e9ed 100644
--- a/libcpu/aarch64/common/trap.c
+++ b/libcpu/aarch64/common/trap.c
@@ -14,6 +14,7 @@
#include <armv8.h>
#include "interrupt.h"
+#include "mm_fault.h"
#include <backtrace.h>
@@ -48,6 +49,29 @@ void check_user_fault(struct rt_hw_exp_stack *regs, uint32_t pc_adj, char *info)
}
}
+int _get_type(unsigned long esr)
+{
+ int ret;
+ int fsc = esr & 0x3f;
+ switch (fsc)
+ {
+ case 0x4:
+ case 0x5:
+ case 0x6:
+ case 0x7:
+ ret = MM_FAULT_TYPE_PAGE_FAULT;
+ break;
+ case 0x9:
+ case 0xa:
+ case 0xb:
+ ret = MM_FAULT_TYPE_ACCESS_FAULT;
+ break;
+ default:
+ ret = MM_FAULT_TYPE_GENERIC;
+ }
+ return ret;
+}
+
int check_user_stack(unsigned long esr, struct rt_hw_exp_stack *regs)
{
unsigned char ec;
@@ -55,19 +79,36 @@ int check_user_stack(unsigned long esr, struct rt_hw_exp_stack *regs)
int ret = 0;
ec = (unsigned char)((esr >> 26) & 0x3fU);
+ enum rt_mm_fault_op fault_op;
+ enum rt_mm_fault_type fault_type;
switch (ec)
{
case 0x20:
+ fault_op = MM_FAULT_OP_EXECUTE;
+ fault_type = _get_type(esr);
+ break;
case 0x21:
case 0x24:
+ fault_op = MM_FAULT_OP_WRITE;
+ fault_type = _get_type(esr);
+ break;
+ default:
+ fault_op = 0;
+ break;
+ }
+
+ if (fault_op)
+ {
asm volatile("mrs %0, far_el1":"=r"(dfar));
- if (arch_expand_user_stack(dfar))
+ struct rt_mm_fault_msg msg = {
+ .fault_op = fault_op,
+ .fault_type = fault_type,
+ .vaddr = dfar,
+ };
+ if (rt_mm_fault_try_fix(&msg))
{
ret = 1;
}
- break;
- default:
- break;
}
return ret;
}
diff --git a/libcpu/aarch64/cortex-a/entry_point.S b/libcpu/aarch64/cortex-a/entry_point.S
index ae3fa02ccd..89c383a1e5 100644
--- a/libcpu/aarch64/cortex-a/entry_point.S
+++ b/libcpu/aarch64/cortex-a/entry_point.S
@@ -91,10 +91,11 @@ __start:
msr cpacr_el1, x1
/* clear bss */
- ldr x1, =__bss_start /* get bss start address */
- ldr x2, =__bss_end
+ adrp x1, __bss_start /* get bss start address */
+ add x1, x1, #:lo12:__bss_start
+ adrp x2, __bss_end
+ add x1, x1, #:lo12:__bss_end
sub x2, x2, x1 /* get bss size */
- add x1, x1, x9
and x3, x2, #7 /* x3 is < 7 */
ldr x4, =~0x7
@@ -140,14 +141,14 @@ __start:
orr x1, x1, #(1 << 0) /* M */
msr sctlr_el1, x1 /* enable MMU */
- dsb sy
- isb sy
+ dsb ish
+ isb
ic ialluis /* Invalidate all instruction caches in Inner Shareable domain to Point of Unification */
- dsb sy
- isb sy
+ dsb ish
+ isb
tlbi vmalle1 /* Invalidate all stage 1 translations used at EL1 with the current VMID */
- dsb sy
- isb sy
+ dsb ish
+ isb
ret
after_mmu_enable:
diff --git a/libcpu/arm/cortex-a/context_gcc.S b/libcpu/arm/cortex-a/context_gcc.S
index c4531dede5..98080de5c1 100644
--- a/libcpu/arm/cortex-a/context_gcc.S
+++ b/libcpu/arm/cortex-a/context_gcc.S
@@ -53,7 +53,7 @@ rt_hw_context_switch_to:
#ifdef RT_USING_SMART
bl rt_thread_self
mov r4, r0
- bl lwp_mmu_switch
+ bl lwp_aspace_switch
mov r0, r4
bl lwp_user_setting_restore
#endif
@@ -116,7 +116,7 @@ rt_hw_context_switch:
#ifdef RT_USING_SMART
bl rt_thread_self
mov r4, r0
- bl lwp_mmu_switch
+ bl lwp_aspace_switch
mov r0, r4
bl lwp_user_setting_restore
#endif
diff --git a/libcpu/arm/cortex-a/cpuport.h b/libcpu/arm/cortex-a/cpuport.h
index ab68d1e311..e881da9147 100644
--- a/libcpu/arm/cortex-a/cpuport.h
+++ b/libcpu/arm/cortex-a/cpuport.h
@@ -10,6 +10,8 @@
#ifndef CPUPORT_H__
#define CPUPORT_H__
+#include <rtthread.h>
+
/* the exception stack without VFP registers */
struct rt_hw_exp_stack
{
diff --git a/libcpu/arm/cortex-a/interrupt.c b/libcpu/arm/cortex-a/interrupt.c
index fb257cc2de..d0cc89d6e2 100644
--- a/libcpu/arm/cortex-a/interrupt.c
+++ b/libcpu/arm/cortex-a/interrupt.c
@@ -70,8 +70,8 @@ void rt_hw_interrupt_init(void)
/* initialize ARM GIC */
#ifdef RT_USING_SMART
- gic_dist_base = (uint32_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_dist_base(), 0x2000, MMU_MAP_K_RW);
- gic_cpu_base = (uint32_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_cpu_base(), 0x1000, MMU_MAP_K_RW);
+ gic_dist_base = (uint32_t)rt_ioremap((void*)platform_get_gic_dist_base(), 0x2000);
+ gic_cpu_base = (uint32_t)rt_ioremap((void*)platform_get_gic_cpu_base(), 0x1000);
#else
gic_dist_base = platform_get_gic_dist_base();
gic_cpu_base = platform_get_gic_cpu_base();
@@ -100,7 +100,8 @@ void rt_hw_interrupt_init(void)
/* initialize ARM GIC */
#ifdef RT_USING_SMART
- gic_dist_base = (uint32_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_dist_base(), 0x2000, MMU_MAP_K_RW);
+ gic_dist_base = (uint32_t)rt_ioremap((void*)platform_get_gic_dist_base(), 0x2000);
+ gic_cpu_base = (uint32_t)rt_ioremap((void*)platform_get_gic_cpu_base(), 0x1000);
#else
gic_dist_base = platform_get_gic_dist_base();
#endif
diff --git a/libcpu/arm/cortex-a/mmu.c b/libcpu/arm/cortex-a/mmu.c
index 19fd0f10d2..150dd98b27 100644
--- a/libcpu/arm/cortex-a/mmu.c
+++ b/libcpu/arm/cortex-a/mmu.c
@@ -12,52 +12,23 @@
#include <rtthread.h>
#include <board.h>
-
#include "cp15.h"
+#include "mm_page.h"
#include "mmu.h"
+#include <mm_aspace.h>
+#include <tlb.h>
#ifdef RT_USING_SMART
#include <lwp_mm.h>
-#include "page.h"
+#include <lwp_arch.h>
+#include "ioremap.h"
+#else
+#define KERNEL_VADDR_START 0
+#define PV_OFFSET 0
#endif
/* level1 page table, each entry for 1MB memory. */
-volatile unsigned long MMUTable[4*1024] __attribute__((aligned(16*1024)));
-
-#ifndef RT_USING_SMART
-static rt_mutex_t mm_lock = RT_NULL;
-
-void rt_mm_lock(void)
-{
- if (rt_thread_self())
- {
- if (!mm_lock)
- {
- mm_lock = rt_mutex_create("mm_lock", RT_IPC_FLAG_FIFO);
- }
- if (mm_lock)
- {
- rt_mutex_take(mm_lock, RT_WAITING_FOREVER);
- }
- }
-}
-
-void rt_mm_unlock(void)
-{
- if (rt_thread_self())
- {
- if (mm_lock)
- {
- rt_mutex_release(mm_lock);
- }
- }
-}
-#endif
-
-void rt_hw_cpu_tlb_invalidate(void)
-{
- asm volatile ("mcr p15, 0, r0, c8, c7, 0\ndsb\nisb" ::: "memory");
-}
+volatile unsigned long MMUTable[4 * 1024] __attribute__((aligned(16 * 1024)));
unsigned long rt_hw_set_domain_register(unsigned long domain_val)
{
@@ -69,117 +40,8 @@ unsigned long rt_hw_set_domain_register(unsigned long domain_val)
return old_domain;
}
-/* dump 2nd level page table */
-void rt_hw_cpu_dump_page_table_2nd(rt_uint32_t *ptb)
-{
- int i;
- int fcnt = 0;
-
- for (i = 0; i < 256; i++)
- {
- rt_uint32_t pte2 = ptb[i];
- if ((pte2 & 0x3) == 0)
- {
- if (fcnt == 0)
- rt_kprintf(" ");
- rt_kprintf("%04x: ", i);
- fcnt++;
- if (fcnt == 16)
- {
- rt_kprintf("fault\n");
- fcnt = 0;
- }
- continue;
- }
- if (fcnt != 0)
- {
- rt_kprintf("fault\n");
- fcnt = 0;
- }
-
- rt_kprintf(" %04x: %x: ", i, pte2);
- if ((pte2 & 0x3) == 0x1)
- {
- rt_kprintf("L,ap:%x,xn:%d,texcb:%02x\n",
- ((pte2 >> 7) | (pte2 >> 4))& 0xf,
- (pte2 >> 15) & 0x1,
- ((pte2 >> 10) | (pte2 >> 2)) & 0x1f);
- }
- else
- {
- rt_kprintf("S,ap:%x,xn:%d,texcb:%02x\n",
- ((pte2 >> 7) | (pte2 >> 4))& 0xf, pte2 & 0x1,
- ((pte2 >> 4) | (pte2 >> 2)) & 0x1f);
- }
- }
-}
-
-void rt_hw_cpu_dump_page_table(rt_uint32_t *ptb)
-{
- int i;
- int fcnt = 0;
-
- rt_kprintf("page table@%p\n", ptb);
- for (i = 0; i < 1024*4; i++)
- {
- rt_uint32_t pte1 = ptb[i];
- if ((pte1 & 0x3) == 0)
- {
- rt_kprintf("%03x: ", i);
- fcnt++;
- if (fcnt == 16)
- {
- rt_kprintf("fault\n");
- fcnt = 0;
- }
- continue;
- }
- if (fcnt != 0)
- {
- rt_kprintf("fault\n");
- fcnt = 0;
- }
-
- rt_kprintf("%03x: %08x: ", i, pte1);
- if ((pte1 & 0x3) == 0x3)
- {
- rt_kprintf("LPAE\n");
- }
- else if ((pte1 & 0x3) == 0x1)
- {
- rt_kprintf("pte,ns:%d,domain:%d\n",
- (pte1 >> 3) & 0x1, (pte1 >> 5) & 0xf);
- /*
- *rt_hw_cpu_dump_page_table_2nd((void*)((pte1 & 0xfffffc000)
- * - 0x80000000 + 0xC0000000));
- */
- }
- else if (pte1 & (1 << 18))
- {
- rt_kprintf("super section,ns:%d,ap:%x,xn:%d,texcb:%02x\n",
- (pte1 >> 19) & 0x1,
- ((pte1 >> 13) | (pte1 >> 10))& 0xf,
- (pte1 >> 4) & 0x1,
- ((pte1 >> 10) | (pte1 >> 2)) & 0x1f);
- }
- else
- {
- rt_kprintf("section,ns:%d,ap:%x,"
- "xn:%d,texcb:%02x,domain:%d\n",
- (pte1 >> 19) & 0x1,
- ((pte1 >> 13) | (pte1 >> 10))& 0xf,
- (pte1 >> 4) & 0x1,
- (((pte1 & (0x7 << 12)) >> 10) |
- ((pte1 & 0x0c) >> 2)) & 0x1f,
- (pte1 >> 5) & 0xf);
- }
- }
-}
-
-void rt_hw_mmu_setmtt(rt_uint32_t vaddrStart,
- rt_uint32_t vaddrEnd,
- rt_uint32_t paddrStart,
- rt_uint32_t attr)
+void rt_hw_mmu_setmtt(rt_uint32_t vaddrStart, rt_uint32_t vaddrEnd,
+ rt_uint32_t paddrStart, rt_uint32_t attr)
{
volatile rt_uint32_t *pTT;
volatile int i, nSec;
@@ -224,11 +86,11 @@ void rt_hw_mmu_init(void)
rt_hw_cpu_dcache_enable();
}
-int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void* v_address, size_t size, size_t *vtable, size_t pv_off)
+int rt_hw_mmu_map_init(struct rt_aspace *aspace, void* v_address, size_t size, size_t *vtable, size_t pv_off)
{
size_t l1_off, va_s, va_e;
- if (!mmu_info || !vtable)
+ if (!aspace || !vtable)
{
return -1;
}
@@ -259,15 +121,20 @@ int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void* v_address, size_t size, size
}
}
- mmu_info->vtable = vtable;
- mmu_info->vstart = va_s;
- mmu_info->vend = va_e;
- mmu_info->pv_off = pv_off;
+#ifdef RT_USING_SMART
+ rt_aspace_init(&rt_kernel_space, (void *)USER_VADDR_TOP, 0 - USER_VADDR_TOP, vtable);
+ rt_ioremap_start = v_address;
+ rt_ioremap_size = size;
+ rt_mpr_start = rt_ioremap_start - rt_mpr_size;
+#else
+ rt_aspace_init(&rt_kernel_space, (void *)0x1000, 0 - 0x1000, vtable);
+ rt_mpr_start = (void *)0 - rt_mpr_size;
+#endif
return 0;
}
-int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void* v_address, size_t size)
+int rt_hw_mmu_ioremap_init(rt_aspace_t aspace, void* v_address, size_t size)
{
#ifdef RT_IOREMAP_LATE
size_t loop_va;
@@ -296,7 +163,7 @@ int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void* v_address, size_t size)
while (sections--)
{
l1_off = (loop_va >> ARCH_SECTION_SHIFT);
- mmu_l1 = (size_t*)mmu_info->vtable + l1_off;
+ mmu_l1 = (size_t*)aspace->page_table + l1_off;
RT_ASSERT((*mmu_l1 & ARCH_MMU_USED_MASK) == 0);
mmu_l2 = (size_t*)rt_pages_alloc(0);
@@ -306,7 +173,7 @@ int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void* v_address, size_t size)
/* cache maintain */
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2, ARCH_PAGE_TBL_SIZE);
- *mmu_l1 = (((size_t)mmu_l2 + mmu_info->pv_off) | 0x1);
+ *mmu_l1 = (((size_t)mmu_l2 + PV_OFFSET) | 0x1);
/* cache maintain */
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l1, 4);
}
@@ -323,427 +190,184 @@ int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void* v_address, size_t size)
return 0;
}
-#ifdef RT_USING_SMART
-static size_t find_vaddr(rt_mmu_info *mmu_info, int pages)
-{
- size_t l1_off, l2_off;
- size_t *mmu_l1, *mmu_l2;
- size_t find_off = 0;
- size_t find_va = 0;
- int n = 0;
-
- if (!pages)
- {
- return 0;
- }
- if (!mmu_info)
- {
- return 0;
- }
- for (l1_off = mmu_info->vstart; l1_off <= mmu_info->vend; l1_off++)
- {
- mmu_l1 = (size_t*)mmu_info->vtable + l1_off;
- if (*mmu_l1 & ARCH_MMU_USED_MASK)
- {
- mmu_l2 = (size_t *)((*mmu_l1 & ~ARCH_PAGE_TBL_MASK) - mmu_info->pv_off);
- for (l2_off = 0; l2_off < (ARCH_SECTION_SIZE/ARCH_PAGE_SIZE); l2_off++)
- {
- if (*(mmu_l2 + l2_off) & ARCH_MMU_USED_MASK)
- {
- /* in use */
- n = 0;
- }
- else
- {
- if (!n)
- {
- find_va = l1_off;
- find_off = l2_off;
- }
- n++;
- if (n >= pages)
- {
- return (find_va << ARCH_SECTION_SHIFT) + (find_off << ARCH_PAGE_SHIFT);
- }
- }
- }
- }
- else
- {
- if (!n)
- {
- find_va = l1_off;
- find_off = 0;
- }
- n += (ARCH_SECTION_SIZE/ARCH_PAGE_SIZE);
- if (n >= pages)
- {
- return (find_va << ARCH_SECTION_SHIFT) + (find_off << ARCH_PAGE_SHIFT);
- }
- }
- }
- return 0;
-}
-static int check_vaddr(rt_mmu_info *mmu_info, void *va, int pages)
+static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr)
{
- size_t loop_va = (size_t)va & ~ARCH_PAGE_MASK;
+ size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK;
size_t l1_off, l2_off;
size_t *mmu_l1, *mmu_l2;
- if (!pages)
- {
- return -1;
- }
+ l1_off = (loop_va >> ARCH_SECTION_SHIFT);
- if (!mmu_info)
- {
- return -1;
- }
+ l2_off = ((loop_va & ARCH_SECTION_MASK) >> ARCH_PAGE_SHIFT);
+ mmu_l1 = (size_t *)lv0_tbl + l1_off;
- l1_off = ((size_t)va >> ARCH_SECTION_SHIFT);
- if (l1_off < mmu_info->vstart || l1_off > mmu_info->vend)
+ if (*mmu_l1 & ARCH_MMU_USED_MASK)
{
- return -1;
+ mmu_l2 = (size_t *)((*mmu_l1 & ~ARCH_PAGE_TBL_MASK) - PV_OFFSET);
}
- l1_off += ((pages << ARCH_PAGE_SHIFT) >> ARCH_SECTION_SHIFT);
- if (l1_off < mmu_info->vstart || l1_off > mmu_info->vend + 1)
+ else
{
- return -1;
+ return;
}
- while (pages--)
+ if (*(mmu_l2 + l2_off) & ARCH_MMU_USED_MASK)
{
- l1_off = (loop_va >> ARCH_SECTION_SHIFT);
- l2_off = ((loop_va & ARCH_SECTION_MASK) >> ARCH_PAGE_SHIFT);
- mmu_l1 = (size_t*)mmu_info->vtable + l1_off;
+ *(mmu_l2 + l2_off) = 0;
+ /* cache maintain */
+ rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2 + l2_off, 4);
- if (*mmu_l1 & ARCH_MMU_USED_MASK)
+ if (rt_pages_free(mmu_l2, 0))
{
- mmu_l2 = (size_t *)((*mmu_l1 & ~ARCH_PAGE_TBL_MASK) - mmu_info->pv_off);
- if (*(mmu_l2 + l2_off) & ARCH_MMU_USED_MASK)
- {
- return -1;
- }
+ *mmu_l1 = 0;
+ rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l1, 4);
}
- loop_va += ARCH_PAGE_SIZE;
}
- return 0;
+ loop_va += ARCH_PAGE_SIZE;
}
-static void __rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void* v_addr, size_t npages)
+static int _kenrel_map_4K(unsigned long *lv0_tbl, void *v_addr, void *p_addr,
+ size_t attr)
{
size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK;
+ size_t loop_pa = (size_t)p_addr & ~ARCH_PAGE_MASK;
size_t l1_off, l2_off;
size_t *mmu_l1, *mmu_l2;
- if (!mmu_info)
+ l1_off = (loop_va >> ARCH_SECTION_SHIFT);
+ l2_off = ((loop_va & ARCH_SECTION_MASK) >> ARCH_PAGE_SHIFT);
+ mmu_l1 = (size_t *)lv0_tbl + l1_off;
+
+ if (*mmu_l1 & ARCH_MMU_USED_MASK)
{
- return;
+ mmu_l2 = (size_t *)((*mmu_l1 & ~ARCH_PAGE_TBL_MASK) - PV_OFFSET);
+ rt_page_ref_inc(mmu_l2, 0);
}
-
- while (npages--)
+ else
{
- l1_off = (loop_va >> ARCH_SECTION_SHIFT);
- if (l1_off < mmu_info->vstart || l1_off > mmu_info->vend)
+ mmu_l2 = (size_t *)rt_pages_alloc(0);
+ if (mmu_l2)
{
- return;
- }
-
- l2_off = ((loop_va & ARCH_SECTION_MASK) >> ARCH_PAGE_SHIFT);
- mmu_l1 = (size_t*)mmu_info->vtable + l1_off;
+ rt_memset(mmu_l2, 0, ARCH_PAGE_TBL_SIZE * 2);
+ /* cache maintain */
+ rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2, ARCH_PAGE_TBL_SIZE);
- if (*mmu_l1 & ARCH_MMU_USED_MASK)
- {
- mmu_l2 = (size_t *)((*mmu_l1 & ~ARCH_PAGE_TBL_MASK) - mmu_info->pv_off);
+ *mmu_l1 = (((size_t)mmu_l2 + PV_OFFSET) | 0x1);
+ /* cache maintain */
+ rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l1, 4);
}
else
{
- return;
+ /* error, quit */
+ return -1;
}
+ }
- if (*(mmu_l2 + l2_off) & ARCH_MMU_USED_MASK)
- {
- *(mmu_l2 + l2_off) = 0;
- /* cache maintain */
- rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2 + l2_off, 4);
+ *(mmu_l2 + l2_off) = (loop_pa | attr);
+ /* cache maintain */
+ rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2 + l2_off, 4);
- if (rt_pages_free(mmu_l2, 0))
- {
- *mmu_l1 = 0;
- rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l1, 4);
- }
- }
- loop_va += ARCH_PAGE_SIZE;
- }
+ loop_va += ARCH_PAGE_SIZE;
+ loop_pa += ARCH_PAGE_SIZE;
+
+ return 0;
}
-static int __rt_hw_mmu_map(rt_mmu_info *mmu_info, void* v_addr, void* p_addr, size_t npages, size_t attr)
+void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
+ size_t attr)
{
- size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK;
- size_t loop_pa = (size_t)p_addr & ~ARCH_PAGE_MASK;
- size_t l1_off, l2_off;
- size_t *mmu_l1, *mmu_l2;
-
- if (!mmu_info)
- {
- return -1;
- }
+ int ret = -1;
+ void *unmap_va = v_addr;
+ size_t npages = size >> ARCH_PAGE_SHIFT;
+ // TODO trying with HUGEPAGE here
while (npages--)
{
- l1_off = (loop_va >> ARCH_SECTION_SHIFT);
- l2_off = ((loop_va & ARCH_SECTION_MASK) >> ARCH_PAGE_SHIFT);
- mmu_l1 = (size_t*)mmu_info->vtable + l1_off;
-
- if (*mmu_l1 & ARCH_MMU_USED_MASK)
- {
- mmu_l2 = (size_t *)((*mmu_l1 & ~ARCH_PAGE_TBL_MASK) - mmu_info->pv_off);
- rt_page_ref_inc(mmu_l2, 0);
- }
- else
+ ret = _kenrel_map_4K(aspace->page_table, v_addr, p_addr, attr);
+ if (ret != 0)
{
- mmu_l2 = (size_t*)rt_pages_alloc(0);
- if (mmu_l2)
+ /* error, undo map */
+ while (unmap_va != v_addr)
{
- rt_memset(mmu_l2, 0, ARCH_PAGE_TBL_SIZE * 2);
- /* cache maintain */
- rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2, ARCH_PAGE_TBL_SIZE);
+ rt_enter_critical();
+ _kenrel_unmap_4K(aspace->page_table, (void *)unmap_va);
+ rt_exit_critical();
- *mmu_l1 = (((size_t)mmu_l2 + mmu_info->pv_off) | 0x1);
- /* cache maintain */
- rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l1, 4);
- }
- else
- {
- /* error, unmap and quit */
- __rt_hw_mmu_unmap(mmu_info, v_addr, npages);
- return -1;
+ unmap_va += ARCH_PAGE_SIZE;
}
+ break;
}
+ v_addr += ARCH_PAGE_SIZE;
+ p_addr += ARCH_PAGE_SIZE;
+ }
- *(mmu_l2 + l2_off) = (loop_pa | attr);
- /* cache maintain */
- rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2 + l2_off, 4);
-
- loop_va += ARCH_PAGE_SIZE;
- loop_pa += ARCH_PAGE_SIZE;
+ if (ret == 0)
+ {
+ return v_addr;
}
- return 0;
+ return NULL;
}
-void *_rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void* p_addr, size_t size, size_t attr)
+void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size)
{
- size_t pa_s, pa_e;
- size_t vaddr;
- int pages;
- int ret;
+ // caller guarantee that v_addr & size are page aligned
+ size_t npages = size >> ARCH_PAGE_SHIFT;
- if (!size)
- {
- return 0;
- }
- pa_s = (size_t)p_addr;
- pa_e = (size_t)p_addr + size - 1;
- pa_s >>= ARCH_PAGE_SHIFT;
- pa_e >>= ARCH_PAGE_SHIFT;
- pages = pa_e - pa_s + 1;
- if (v_addr)
+ if (!aspace->page_table)
{
- vaddr = (size_t)v_addr;
- pa_s = (size_t)p_addr;
- if ((vaddr & ARCH_PAGE_MASK) != (pa_s & ARCH_PAGE_MASK))
- {
- return 0;
- }
- vaddr &= ~ARCH_PAGE_MASK;
- if (check_vaddr(mmu_info, (void*)vaddr, pages) != 0)
- {
- return 0;
- }
+ return;
}
- else
+
+ while (npages--)
{
- vaddr = find_vaddr(mmu_info, pages);
- }
- if (vaddr) {
rt_enter_critical();
- ret = __rt_hw_mmu_map(mmu_info, (void*)vaddr, p_addr, pages, attr);
- if (ret == 0)
- {
- rt_hw_cpu_tlb_invalidate();
- rt_exit_critical();
- return (void*)(vaddr + ((size_t)p_addr & ARCH_PAGE_MASK));
- }
+ _kenrel_unmap_4K(aspace->page_table, v_addr);
rt_exit_critical();
+
+ v_addr += ARCH_PAGE_SIZE;
}
- return 0;
}
-static int __rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void* v_addr, size_t npages, size_t attr)
+void rt_hw_aspace_switch(rt_aspace_t aspace)
{
- size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK;
- size_t loop_pa;
- size_t l1_off, l2_off;
- size_t *mmu_l1, *mmu_l2;
-
- if (!mmu_info)
+ if (aspace != &rt_kernel_space)
{
- return -1;
- }
-
- while (npages--)
- {
- loop_pa = (size_t)rt_pages_alloc(0);
- if (!loop_pa)
- goto err;
-
- l1_off = (loop_va >> ARCH_SECTION_SHIFT);
- l2_off = ((loop_va & ARCH_SECTION_MASK) >> ARCH_PAGE_SHIFT);
- mmu_l1 = (size_t*)mmu_info->vtable + l1_off;
+ void *pgtbl = aspace->page_table;
+ pgtbl = _rt_kmem_v2p(pgtbl);
- if (*mmu_l1 & ARCH_MMU_USED_MASK)
- {
- mmu_l2 = (size_t *)((*mmu_l1 & ~ARCH_PAGE_TBL_MASK) - mmu_info->pv_off);
- rt_page_ref_inc(mmu_l2, 0);
- }
- else
- {
- //mmu_l2 = (size_t*)rt_malloc_align(ARCH_PAGE_TBL_SIZE * 2, ARCH_PAGE_TBL_SIZE);
- mmu_l2 = (size_t*)rt_pages_alloc(0);
- if (mmu_l2)
- {
- rt_memset(mmu_l2, 0, ARCH_PAGE_TBL_SIZE * 2);
- /* cache maintain */
- rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2, ARCH_PAGE_TBL_SIZE);
-
- *mmu_l1 = (((size_t)mmu_l2 + mmu_info->pv_off) | 0x1);
- /* cache maintain */
- rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l1, 4);
- }
- else
- goto err;
- }
-
- loop_pa += mmu_info->pv_off;
- *(mmu_l2 + l2_off) = (loop_pa | attr);
- /* cache maintain */
- rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_l2 + l2_off, 4);
-
- loop_va += ARCH_PAGE_SIZE;
- }
- return 0;
-err:
- {
- /* error, unmap and quit */
- int i;
- void *va, *pa;
-
- va = (void*)((size_t)v_addr & ~ARCH_PAGE_MASK);
- for (i = 0; i < npages; i++)
- {
- pa = rt_hw_mmu_v2p(mmu_info, va);
- pa = (void*)((char*)pa - mmu_info->pv_off);
- rt_pages_free(pa, 0);
- va = (void*)((char*)va + ARCH_PAGE_SIZE);
- }
+ rt_hw_mmu_switch(pgtbl);
- __rt_hw_mmu_unmap(mmu_info, v_addr, npages);
- return -1;
+ rt_hw_tlb_invalidate_all_local();
}
}
-void *_rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr)
+void init_mm_setup(unsigned int *mtbl, unsigned int size, unsigned int pv_off)
{
- size_t vaddr;
- size_t offset;
- int pages;
- int ret;
+ unsigned int va;
- if (!size)
+ for (va = 0; va < 0x1000; va++)
{
- return 0;
- }
+ unsigned int vaddr = (va << 20);
- offset = (size_t)v_addr & ARCH_PAGE_MASK;
- size += (offset + ARCH_PAGE_SIZE - 1);
- pages = (size >> ARCH_PAGE_SHIFT);
- if (v_addr)
- {
- vaddr = (size_t)v_addr;
- vaddr &= ~ARCH_PAGE_MASK;
- if (check_vaddr(mmu_info, (void*)vaddr, pages) != 0)
+ if (vaddr >= KERNEL_VADDR_START && vaddr - KERNEL_VADDR_START < size)
{
- return 0;
+ mtbl[va] = ((va << 20) + pv_off) | NORMAL_MEM;
}
- }
- else
- {
- vaddr = find_vaddr(mmu_info, pages);
- }
-
- if (vaddr)
- {
- rt_enter_critical();
- ret = __rt_hw_mmu_map_auto(mmu_info, (void*)vaddr, pages, attr);
- if (ret == 0)
+ else if (vaddr >= (KERNEL_VADDR_START + pv_off) && vaddr - (KERNEL_VADDR_START + pv_off) < size)
{
- rt_hw_cpu_tlb_invalidate();
- rt_exit_critical();
- return (void*)((char*)vaddr + offset);
+ mtbl[va] = (va << 20) | NORMAL_MEM;
+ }
+ else
+ {
+ mtbl[va] = 0;
}
- rt_exit_critical();
}
- return 0;
}
-void _rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void* v_addr, size_t size)
-{
- size_t va_s, va_e;
- int pages;
-
- va_s = (size_t)v_addr;
- va_e = (size_t)v_addr + size - 1;
- va_s >>= ARCH_PAGE_SHIFT;
- va_e >>= ARCH_PAGE_SHIFT;
- pages = va_e - va_s + 1;
- rt_enter_critical();
- __rt_hw_mmu_unmap(mmu_info, v_addr, pages);
- rt_hw_cpu_tlb_invalidate();
- rt_exit_critical();
-}
-
-void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void* p_addr, size_t size, size_t attr)
-{
- void *ret;
-
- rt_mm_lock();
- ret = _rt_hw_mmu_map(mmu_info, v_addr, p_addr, size, attr);
- rt_mm_unlock();
- return ret;
-}
-
-void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr)
-{
- void *ret;
-
- rt_mm_lock();
- ret = _rt_hw_mmu_map_auto(mmu_info, v_addr, size, attr);
- rt_mm_unlock();
- return ret;
-}
-
-void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void* v_addr, size_t size)
-{
- rt_mm_lock();
- _rt_hw_mmu_unmap(mmu_info, v_addr, size);
- rt_mm_unlock();
-}
-
-void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr)
+void *rt_hw_mmu_v2p(rt_aspace_t aspace, void* v_addr)
{
size_t l1_off, l2_off;
size_t *mmu_l1, *mmu_l2;
@@ -752,12 +376,9 @@ void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr)
l1_off = (size_t)v_addr >> ARCH_SECTION_SHIFT;
- if (!mmu_info)
- {
- return (void*)0;
- }
+ RT_ASSERT(aspace);
- mmu_l1 = (size_t*)mmu_info->vtable + l1_off;
+ mmu_l1 = (size_t*)aspace->page_table + l1_off;
tmp = *mmu_l1;
@@ -766,7 +387,7 @@ void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr)
case 0: /* not used */
break;
case 1: /* page table */
- mmu_l2 = (size_t *)((tmp & ~ARCH_PAGE_TBL_MASK) - mmu_info->pv_off);
+ mmu_l2 = (size_t *)((tmp & ~ARCH_PAGE_TBL_MASK) - PV_OFFSET);
l2_off = (((size_t)v_addr & ARCH_SECTION_MASK) >> ARCH_PAGE_SHIFT);
pa = *(mmu_l2 + l2_off);
if (pa & ARCH_MMU_USED_MASK)
@@ -793,39 +414,11 @@ void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr)
pa += ((size_t)v_addr & ARCH_SECTION_MASK);
return (void*)pa;
}
- return (void*)0;
+ return ARCH_MAP_FAILED;
}
-void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr)
+int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
+ enum rt_mmu_cntl cmd)
{
- void *ret;
-
- rt_mm_lock();
- ret = _rt_hw_mmu_v2p(mmu_info, v_addr);
- rt_mm_unlock();
- return ret;
+ return -RT_ENOSYS;
}
-
-void init_mm_setup(unsigned int *mtbl, unsigned int size, unsigned int pv_off)
-{
- unsigned int va;
-
- for (va = 0; va < 0x1000; va++)
- {
- unsigned int vaddr = (va << 20);
-
- if (vaddr >= KERNEL_VADDR_START && vaddr - KERNEL_VADDR_START < size)
- {
- mtbl[va] = ((va << 20) + pv_off) | NORMAL_MEM;
- }
- else if (vaddr >= (KERNEL_VADDR_START + pv_off) && vaddr - (KERNEL_VADDR_START + pv_off) < size)
- {
- mtbl[va] = (va << 20) | NORMAL_MEM;
- }
- else
- {
- mtbl[va] = 0;
- }
- }
-}
-#endif
diff --git a/libcpu/arm/cortex-a/mmu.h b/libcpu/arm/cortex-a/mmu.h
index 923d10df4b..04b4f485b0 100644
--- a/libcpu/arm/cortex-a/mmu.h
+++ b/libcpu/arm/cortex-a/mmu.h
@@ -11,6 +11,7 @@
#define __MMU_H_
#include <rtthread.h>
+#include <mm_aspace.h>
#define DESC_SEC (0x2)
#define MEMWBWA ((1<<12)|(3<<2)) /* write back, write allocate */
@@ -87,27 +88,43 @@ struct mem_desc
#define ARCH_TYPE_SUPERSECTION (1 << 18)
#define ARCH_ADDRESS_WIDTH_BITS 32
+#define ARCH_VADDR_WIDTH 32
-typedef struct
-{
- size_t *vtable;
- size_t vstart;
- size_t vend;
- size_t pv_off;
-} rt_mmu_info;
+/**
+ * *info it's possible to map (-1ul & ~ARCH_PAGE_MASK) but a not aligned -1 is
+ * never returned on a successful mapping
+ */
+#define ARCH_MAP_FAILED ((void *)-1)
-int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void* v_address, size_t size, size_t *vtable, size_t pv_off);
-int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void* v_address, size_t size);
+int rt_hw_mmu_ioremap_init(struct rt_aspace *aspace, void *v_address, size_t size);
void rt_hw_init_mmu_table(struct mem_desc *mdesc, rt_uint32_t size);
-#ifdef RT_USING_SMART
-void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void* p_addr, size_t size, size_t attr);
-void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr);
-#else
-void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void* p_addr, size_t size, size_t attr);
-#endif
+void rt_hw_mmu_setup(struct rt_aspace *aspace, struct mem_desc *mdesc, int desc_nr);
+
+int rt_hw_mmu_map_init(struct rt_aspace *aspace, void *v_address, size_t size, size_t *vtable, size_t pv_off);
+void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr, size_t size, size_t attr);
+void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size);
+
+void rt_hw_aspace_switch(struct rt_aspace *aspace);
+void rt_hw_mmu_switch(void *tbl);
+
+void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr);
+void rt_hw_mmu_kernel_map_init(struct rt_aspace *aspace, size_t vaddr_start, size_t size);
+void *rt_hw_mmu_tbl_get();
+
+static inline void *_rt_kmem_v2p(void *vaddr)
+{
+ return rt_hw_mmu_v2p(&rt_kernel_space, vaddr);
+}
+
+static inline void *rt_kmem_v2p(void *vaddr)
+{
+ MM_PGTBL_LOCK(&rt_kernel_space);
+ void *paddr = _rt_kmem_v2p(vaddr);
+ MM_PGTBL_UNLOCK(&rt_kernel_space);
+ return paddr;
+}
-void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void* v_addr, size_t size);
-void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr);
+int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, enum rt_mmu_cntl cmd);
#endif
diff --git a/libcpu/arm/cortex-a/start_gcc.S b/libcpu/arm/cortex-a/start_gcc.S
index 865f005dc8..94f96468fa 100644
--- a/libcpu/arm/cortex-a/start_gcc.S
+++ b/libcpu/arm/cortex-a/start_gcc.S
@@ -258,34 +258,27 @@ rt_hw_set_process_id:
MCR p15, 0, r0, c13, c0, 1
mov pc, lr
+#endif
.global rt_hw_mmu_switch
rt_hw_mmu_switch:
- mov r3, #0
- mcr p15, 0, r3, c13, c0, 1 /* set contextid = 0, for synchronization*/
- isb
-
orr r0, #0x18
- mcr p15, 0, r0, c2, c0, 0 /* ttbr0 */
-
- isb
- mov r1, r1, LSL #0x8
- and r2, r2, #0xff
- orr r1, r1, r2 /* contextid.PROCID = pid, contextid.ASID = asid*/
- mcr p15, 0, r1, c13, c0, 1 /* set contextid = r1*/
- isb
+ mcr p15, 0, r0, c2, c0, 0 // ttbr0
- mcr p15, 0, r0, c7, c5, 0 /* iciallu */
- mcr p15, 0, r0, c7, c5, 6 /* bpiall */
+ //invalid tlb
+ mov r0, #0
+ mcr p15, 0, r0, c8, c7, 0
+ mcr p15, 0, r0, c7, c5, 0 //iciallu
+ mcr p15, 0, r0, c7, c5, 6 //bpiall
dsb
isb
mov pc, lr
+
.global rt_hw_mmu_tbl_get
rt_hw_mmu_tbl_get:
mrc p15, 0, r0, c2, c0, 0 /* ttbr0 */
bic r0, #0x18
mov pc, lr
-#endif
_halt:
wfe
@@ -459,7 +452,7 @@ rt_hw_context_switch_interrupt_do:
bl rt_thread_self
#ifdef RT_USING_SMART
mov r4, r0
- bl lwp_mmu_switch
+ bl lwp_aspace_switch
mov r0, r4
bl lwp_user_setting_restore
#endif
diff --git a/libcpu/arm/cortex-a/tlb.h b/libcpu/arm/cortex-a/tlb.h
new file mode 100644
index 0000000000..1dfb741163
--- /dev/null
+++ b/libcpu/arm/cortex-a/tlb.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2006-2022, RT-Thread Development Team
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Change Logs:
+ * Date Author Notes
+ * 2022-12-12 WangXiaoyao the first version
+ */
+#ifndef __TLB_H__
+#define __TLB_H__
+
+#include "mm_aspace.h"
+#include <rtthread.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define dsb(scope) __asm__ volatile("dsb " #scope : : : "memory")
+#define isb() __asm__ volatile("isb" : : : "memory")
+#define STORE_CP32(r, name...) "mcr " RT_STRINGIFY(CP32(%r, name)) ";"
+
+#define WRITE_CP32(v, name...) do { \
+ register uint32_t _r = (v); \
+ asm volatile(STORE_CP32(0, name) : : "r" (_r)); \
+} while (0)
+
+static inline void rt_hw_tlb_invalidate_all(void)
+{
+ asm volatile ("mcr p15, 0, r0, c8, c7, 0\ndsb\nisb" ::: "memory");
+}
+
+static inline void rt_hw_tlb_invalidate_all_local(void)
+{
+ rt_hw_tlb_invalidate_all();
+}
+
+static inline void rt_hw_tlb_invalidate_aspace(rt_aspace_t aspace)
+{
+ rt_hw_tlb_invalidate_all();
+}
+
+static inline void rt_hw_tlb_invalidate_range(rt_aspace_t aspace, void *start,
+ size_t size, size_t stride)
+{
+ rt_hw_tlb_invalidate_all();
+}
+
+#endif /* __TLB_H__ */
diff --git a/libcpu/arm/cortex-a/trap.c b/libcpu/arm/cortex-a/trap.c
index 79897cd39c..425e51b9d0 100644
--- a/libcpu/arm/cortex-a/trap.c
+++ b/libcpu/arm/cortex-a/trap.c
@@ -8,12 +8,13 @@
* 2013-07-20 Bernard first version
*/
-#include <rtthread.h>
-#include <rthw.h>
-#include <board.h>
#include <backtrace.h>
+#include <board.h>
+#include <rthw.h>
+#include <rtthread.h>
#include "interrupt.h"
+#include "mm_fault.h"
#ifdef RT_USING_FINSH
extern long list_thread(void);
@@ -44,14 +45,23 @@ void check_user_fault(struct rt_hw_exp_stack *regs, uint32_t pc_adj, char *info)
int check_user_stack(struct rt_hw_exp_stack *regs)
{
- void* dfar = RT_NULL;
+ void *dfar = RT_NULL;
+ asm volatile("MRC p15, 0, %0, c6, c0, 0" : "=r"(dfar));
- asm volatile ("MRC p15, 0, %0, c6, c0, 0":"=r"(dfar));
- if (arch_expand_user_stack(dfar))
+ if ((dfar >= (void *)USER_STACK_VSTART) && (dfar < (void *)USER_STACK_VEND))
{
- regs->pc -= 8;
- return 1;
+ struct rt_mm_fault_msg msg = {
+ .fault_op = MM_FAULT_OP_WRITE,
+ .fault_type = MM_FAULT_TYPE_PAGE_FAULT,
+ .vaddr = dfar,
+ };
+ if (rt_mm_fault_try_fix(&msg))
+ {
+ regs->pc -= 8;
+ return 1;
+ }
}
+
return 0;
}
#endif
@@ -79,7 +89,7 @@ void rt_hw_show_register(struct rt_hw_exp_stack *regs)
rt_kprintf("ttbr0:0x%08x\n", v);
asm volatile ("MRC p15, 0, %0, c6, c0, 0":"=r"(v));
rt_kprintf("dfar:0x%08x\n", v);
- rt_kprintf("0x%08x -> 0x%08x\n", v, rt_hw_mmu_v2p(&mmu_info, (void *)v));
+ rt_kprintf("0x%08x -> 0x%08x\n", v, rt_kmem_v2p((void *)v));
}
#endif
}
diff --git a/libcpu/risc-v/t-head/c906/backtrace.c b/libcpu/risc-v/t-head/c906/backtrace.c
new file mode 100644
index 0000000000..ad5dd7d2d3
--- /dev/null
+++ b/libcpu/risc-v/t-head/c906/backtrace.c
@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2006-2018, RT-Thread Development Team
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Change Logs:
+ * Date Author Notes
+ */
+
+#include <rtthread.h>
+#ifdef RT_USING_SMART
+#include <lwp_arch.h>
+
+#define TRANCE_LEVEL 20
+
+extern rt_ubase_t __text_start[];
+extern rt_ubase_t __text_end[];
+
+static char *_get_elf_name(size_t sepc);
+
+void rt_hw_backtrace(rt_uint32_t *ffp, rt_ubase_t sepc)
+{
+ rt_ubase_t *ra;
+ rt_ubase_t *fp;
+ rt_ubase_t vas, vae;
+ int i, j;
+
+ rt_kprintf("riscv64-unknown-linux-musl-addr2line -e %s -a -f", _get_elf_name(sepc));
+
+ fp = (rt_ubase_t *)ffp;
+
+ if (!fp)
+ {
+ asm volatile("mv %0, s0"
+ : "=r"(fp));
+ }
+
+ if (sepc)
+ {
+ rt_kprintf(" %p", sepc - 0x4);
+ }
+
+ if (fp > (rt_ubase_t *)USER_VADDR_START && fp < (rt_ubase_t *)USER_VADDR_TOP)
+ {
+ vas = USER_VADDR_START;
+ vae = USER_VADDR_TOP;
+ }
+ else
+ {
+ vas = (rt_ubase_t)&__text_start;
+ vae = (rt_ubase_t)&__text_end;
+ }
+
+ for (i = j = 0; i < TRANCE_LEVEL; i++)
+ {
+ if (RT_ALIGN((rt_ubase_t)fp, sizeof(void *)) != (rt_ubase_t)fp)
+ {
+ break;
+ }
+
+ ra = fp - 1;
+ if (!_rt_kmem_v2p(ra) || *ra < vas || *ra > vae)
+ break;
+
+ rt_kprintf(" %p", *ra - 0x04);
+
+ fp = fp - 2;
+ if (!_rt_kmem_v2p(fp))
+ break;
+ fp = (rt_ubase_t *)(*fp);
+ if (!fp)
+ break;
+ }
+
+ rt_kputs("\r\n");
+}
+
+static void _assert_backtrace_cb(const char *ex, const char *func, rt_size_t line)
+{
+ rt_hw_interrupt_disable();
+ rt_kprintf("(%s) assertion failed at function:%s, line number:%d \n", ex, func, line);
+
+ rt_hw_backtrace(0, 0);
+ rt_hw_cpu_shutdown();
+}
+
+static int rt_hw_backtrace_init(void)
+{
+ rt_assert_set_hook(_assert_backtrace_cb);
+
+ return 0;
+}
+INIT_BOARD_EXPORT(rt_hw_backtrace_init);
+
+static void backtrace_test(int args, char *argv[])
+{
+ int *p = (void *)-1;
+ init_fn_t ft = 0;
+
+ if (args < 2)
+ {
+ rt_kprintf("backtrace_test usage:backtrace_test a(assert)/m(invalid memory)/i(illegal instruction)\r\n");
+ return;
+ }
+
+ if (!rt_strcmp(argv[1], "a"))
+ {
+ rt_kprintf("Assert test:\r\n", argv[1]);
+ RT_ASSERT(0);
+ }
+ else if (!rt_strcmp(argv[1], "m"))
+ {
+ rt_kprintf("Access invalid memory:\r\n", argv[1]);
+ *p = 0;
+ }
+ else if (!rt_strcmp(argv[1], "i"))
+ {
+ rt_kprintf("Illegal instruction:\r\n", argv[1]);
+ ft();
+ }
+ else
+ {
+ rt_kprintf("Unknown cmd :%s.\r\n", argv[1]);
+ }
+}
+MSH_CMD_EXPORT(backtrace_test, backtrace test case);
+
+extern struct rt_thread *rt_current_thread;
+
+#define IN_USERSPACE (sepc > USER_VADDR_START && sepc < USER_VADDR_TOP)
+
+static char *_get_elf_name(size_t sepc)
+{
+ return IN_USERSPACE ? rt_current_thread->name : "rtthread.elf";
+}
+
+#endif /* RT_USING_SMART */
diff --git a/libcpu/risc-v/t-head/c906/context_gcc.S b/libcpu/risc-v/t-head/c906/context_gcc.S
index 1442e2bf0a..595cd8f3d7 100644
--- a/libcpu/risc-v/t-head/c906/context_gcc.S
+++ b/libcpu/risc-v/t-head/c906/context_gcc.S
@@ -23,7 +23,7 @@ rt_hw_context_switch_to:
#ifdef RT_USING_SMART
mv a0, s1
- jal lwp_mmu_switch
+ jal lwp_aspace_switch
#endif
RESTORE_ALL
@@ -56,7 +56,7 @@ rt_hw_context_switch:
#ifdef RT_USING_SMART
mv a0, s1
- jal lwp_mmu_switch
+ jal lwp_aspace_switch
#endif
RESTORE_ALL
diff --git a/libcpu/risc-v/t-head/c906/cpuport.c b/libcpu/risc-v/t-head/c906/cpuport.c
index 592f66bfb2..847d0b7671 100644
--- a/libcpu/risc-v/t-head/c906/cpuport.c
+++ b/libcpu/risc-v/t-head/c906/cpuport.c
@@ -14,6 +14,7 @@
#include <rtthread.h>
#include "cpuport.h"
+#include "sbi.h"
#include "stack.h"
#include <lwp_arch.h>
@@ -110,10 +111,8 @@ void rt_hw_cpu_shutdown()
rt_kprintf("shutdown...\n");
level = rt_hw_interrupt_disable();
- while (level)
- {
- RT_ASSERT(0);
- }
+ sbi_shutdown();
+ while (1);
}
int rt_hw_cpu_id(void)
diff --git a/libcpu/risc-v/t-head/c906/cpuport.h b/libcpu/risc-v/t-head/c906/cpuport.h
index ceea4ef5af..640c5281cf 100644
--- a/libcpu/risc-v/t-head/c906/cpuport.h
+++ b/libcpu/risc-v/t-head/c906/cpuport.h
@@ -37,19 +37,21 @@
#define CTX_REG_NR (CTX_GENERAL_REG_NR + CTX_FPU_REG_NR)
#ifndef __ASSEMBLY__
+#include <rtthread.h>
+
rt_inline void rt_hw_dsb()
{
- asm volatile("fence":::"memory");
+ __asm__ volatile("fence":::"memory");
}
rt_inline void rt_hw_dmb()
{
- asm volatile("fence":::"memory");
+ __asm__ volatile("fence":::"memory");
}
rt_inline void rt_hw_isb()
{
- asm volatile(OPC_FENCE_I:::"memory");
+ __asm__ volatile(OPC_FENCE_I:::"memory");
}
int rt_hw_cpu_id(void);
diff --git a/libcpu/risc-v/t-head/c906/encoding.h b/libcpu/risc-v/t-head/c906/encoding.h
index 7197e6b84b..25f1b771db 100644
--- a/libcpu/risc-v/t-head/c906/encoding.h
+++ b/libcpu/risc-v/t-head/c906/encoding.h
@@ -33,7 +33,7 @@
#define SSTATUS_FS_CLEAN 0x00004000
#define SSTATUS_FS_DIRTY 0x00006000
#define SSTATUS_XS 0x00018000
-#define SSTATUS_PUM 0x00040000
+#define SSTATUS_SUM 0x00040000
#define SSTATUS32_SD 0x80000000
#define SSTATUS64_SD 0x8000000000000000
diff --git a/libcpu/risc-v/t-head/c906/interrupt_gcc.S b/libcpu/risc-v/t-head/c906/interrupt_gcc.S
index 084dfa25f4..c9659eb512 100644
--- a/libcpu/risc-v/t-head/c906/interrupt_gcc.S
+++ b/libcpu/risc-v/t-head/c906/interrupt_gcc.S
@@ -96,7 +96,7 @@ copy_context_loop_interrupt:
#ifdef RT_USING_SMART
mv a0, s1
jal rt_thread_sp_to_thread
- jal lwp_mmu_switch
+ jal lwp_aspace_switch
#endif
spurious_interrupt:
diff --git a/libcpu/risc-v/t-head/c906/mmu.c b/libcpu/risc-v/t-head/c906/mmu.c
index 0d461bc861..0093f5e836 100644
--- a/libcpu/risc-v/t-head/c906/mmu.c
+++ b/libcpu/risc-v/t-head/c906/mmu.c
@@ -8,628 +8,498 @@
* 2021-01-30 lizhirui first version
*/
+#include "rtconfig.h"
#include <rtthread.h>
-#include <rthw.h>
-#include <board.h>
-#include "page.h"
-#include <stdlib.h>
-#include <string.h>
+#include <stddef.h>
+#include <stdint.h>
+
#include <cache.h>
+#include <mm_aspace.h>
+#include <mm_page.h>
+#include <mmu.h>
+#include <riscv_mmu.h>
+#include <tlb.h>
+
+#ifdef RT_USING_SMART
+#include <ioremap.h>
+#include <lwp_user_mm.h>
+#include <tlb.h>
+#endif
-#include "riscv.h"
-#include "riscv_mmu.h"
-#include "mmu.h"
+#define DBG_TAG "MMU"
+#define DBG_LVL DBG_LOG
+#include <rtdbg.h>
-void *current_mmu_table = RT_NULL;
+#ifndef RT_USING_SMART
+#define PV_OFFSET 0
+#define USER_VADDR_START 0
+#endif
-static void rt_hw_cpu_tlb_invalidate()
+static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr, size_t size);
+
+void rt_hw_aspace_switch(rt_aspace_t aspace)
{
- rt_size_t satpv = read_csr(satp);
- write_csr(satp,satpv);
- mmu_flush_tlb();
+ uintptr_t page_table = (uintptr_t)_rt_kmem_v2p(aspace->page_table);
+
+ write_csr(satp, (((size_t)SATP_MODE) << SATP_MODE_OFFSET) |
+ ((rt_ubase_t)page_table >> PAGE_OFFSET_BIT));
+ rt_hw_tlb_invalidate_all_local();
}
+static void *current_mmu_table = RT_NULL;
+
+volatile __attribute__((aligned(4 * 1024)))
+rt_ubase_t MMUTable[__SIZE(VPN2_BIT)];
+
void *rt_hw_mmu_tbl_get()
{
return current_mmu_table;
}
-void rt_hw_mmu_switch(void *mmu_table)
+static int _map_one_page(struct rt_aspace *aspace, void *va, void *pa,
+ size_t attr)
{
- current_mmu_table = mmu_table;
- RT_ASSERT(__CHECKALIGN(mmu_table,PAGE_OFFSET_BIT));
- mmu_set_pagetable((rt_ubase_t)mmu_table);
- rt_hw_cpu_dcache_clean_all();
- rt_hw_cpu_icache_invalidate_all();
-}
+ rt_size_t l1_off, l2_off, l3_off;
+ rt_size_t *mmu_l1, *mmu_l2, *mmu_l3;
-int rt_hw_mmu_map_init(rt_mmu_info *mmu_info,void *v_address,rt_size_t size,rt_size_t *vtable,rt_size_t pv_off)
-{
- size_t l1_off,va_s,va_e;
- rt_base_t level;
+ l1_off = GET_L1((size_t)va);
+ l2_off = GET_L2((size_t)va);
+ l3_off = GET_L3((size_t)va);
- if((!mmu_info) || (!vtable))
- {
- return -1;
- }
+ mmu_l1 = ((rt_size_t *)aspace->page_table) + l1_off;
- va_s = (rt_size_t)v_address;
- va_e = ((rt_size_t)v_address) + size - 1;
-
- if(va_e < va_s)
+ if (PTE_USED(*mmu_l1))
{
- return -1;
+ mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET);
}
-
- //convert address to level 1 page frame id
- va_s = GET_L1(va_s);
- va_e = GET_L1(va_e);
-
- if(va_s == 0)
- {
- return -1;
- }
-
- level = rt_hw_interrupt_disable();
-
- //vtable initialization check
- for(l1_off = va_s;l1_off <= va_e;l1_off++)
+ else
{
- size_t v = vtable[l1_off];
+ mmu_l2 = (rt_size_t *)rt_pages_alloc(0);
- if(v)
+ if (mmu_l2)
{
- rt_hw_interrupt_enable(level);
- return 0;
+ rt_memset(mmu_l2, 0, PAGE_SIZE);
+ rt_hw_cpu_dcache_clean(mmu_l2, PAGE_SIZE);
+ *mmu_l1 = COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l2, PV_OFFSET),
+ PAGE_DEFAULT_ATTR_NEXT);
+ rt_hw_cpu_dcache_clean(mmu_l1, sizeof(*mmu_l1));
+ }
+ else
+ {
+ return -1;
}
}
- mmu_info -> vtable = vtable;
- mmu_info -> vstart = va_s;
- mmu_info -> vend = va_e;
- mmu_info -> pv_off = pv_off;
-
- rt_hw_interrupt_enable(level);
- return 0;
-}
-
-void rt_hw_mmu_kernel_map_init(rt_mmu_info *mmu_info,rt_size_t vaddr_start,rt_size_t size)
-{
- rt_size_t paddr_start = __UMASKVALUE(VPN_TO_PPN(vaddr_start,mmu_info -> pv_off),PAGE_OFFSET_MASK);
- rt_size_t va_s = GET_L1(vaddr_start);
- rt_size_t va_e = GET_L1(vaddr_start + size - 1);
- rt_size_t i;
-
- for(i = va_s;i <= va_e;i++)
- {
- mmu_info -> vtable[i] = COMBINEPTE(paddr_start,PAGE_ATTR_RWX | PTE_G | PTE_V | PTE_CACHE | PTE_SHARE | PTE_BUF | PTE_A | PTE_D);
- paddr_start += L1_PAGE_SIZE;
- }
-
- rt_hw_cpu_tlb_invalidate();
-}
-
-//find a range of free virtual address specified by pages
-static rt_size_t find_vaddr(rt_mmu_info *mmu_info,rt_size_t pages)
-{
- rt_size_t l1_off,l2_off,l3_off;
- rt_size_t *mmu_l1,*mmu_l2,*mmu_l3;
- rt_size_t find_l1 = 0,find_l2 = 0,find_l3 = 0;
- rt_size_t n = 0;
-
- if(!pages)
- {
- return 0;
- }
-
- if(!mmu_info)
+ if (PTE_USED(*(mmu_l2 + l2_off)))
{
- return 0;
+ RT_ASSERT(!PAGE_IS_LEAF(*(mmu_l2 + l2_off)));
+ mmu_l3 =
+ (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), PV_OFFSET);
}
-
- for(l1_off = mmu_info -> vstart;l1_off <= mmu_info -> vend;l1_off++)
+ else
{
- mmu_l1 = ((rt_size_t *)mmu_info -> vtable) + l1_off;
+ mmu_l3 = (rt_size_t *)rt_pages_alloc(0);
- if(PTE_USED(*mmu_l1))
+ if (mmu_l3)
{
- RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1));
- mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1),mmu_info -> pv_off);
-
- for(l2_off = 0;l2_off < __SIZE(VPN1_BIT);l2_off++)
- {
- if(PTE_USED(*(mmu_l2 + l2_off)))
- {
- RT_ASSERT(!PAGE_IS_LEAF(*(mmu_l2 + l2_off)));
- mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)),mmu_info -> pv_off);
-
- for(l3_off = 0;l3_off < __SIZE(VPN0_BIT);l3_off++)
- {
- if(PTE_USED(*(mmu_l3 + l3_off)))
- {
- RT_ASSERT(PAGE_IS_LEAF(*(mmu_l3 + l3_off)));
- n = 0;//in use
- }
- else
- {
- if(!n)
- {
- find_l1 = l1_off;
- find_l2 = l2_off;
- find_l3 = l3_off;
- }
-
- n++;
-
- if(n >= pages)
- {
- return COMBINEVADDR(find_l1,find_l2,find_l3);
- }
- }
- }
- }
- else
- {
- if(!n)
- {
- find_l1 = l1_off;
- find_l2 = l2_off;
- find_l3 = 0;
- }
-
- n += __SIZE(VPN0_BIT);
-
- if(n >= pages)
- {
- return COMBINEVADDR(find_l1,find_l2,find_l3);
- }
- }
- }
+ rt_memset(mmu_l3, 0, PAGE_SIZE);
+ rt_hw_cpu_dcache_clean(mmu_l3, PAGE_SIZE);
+ *(mmu_l2 + l2_off) =
+ COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l3, PV_OFFSET),
+ PAGE_DEFAULT_ATTR_NEXT);
+ rt_hw_cpu_dcache_clean(mmu_l2, sizeof(*mmu_l2));
+ // declares a reference to parent page table
+ rt_page_ref_inc((void *)mmu_l2, 0);
}
else
{
- if(!n)
- {
- find_l1 = l1_off;
- find_l2 = 0;
- find_l3 = 0;
- }
-
- n += __SIZE(VPN1_BIT);
-
- if(n >= pages)
- {
- return COMBINEVADDR(find_l1,find_l2,find_l3);
- }
+ return -1;
}
}
+ RT_ASSERT(!PTE_USED(*(mmu_l3 + l3_off)));
+ // declares a reference to parent page table
+ rt_page_ref_inc((void *)mmu_l3, 0);
+ *(mmu_l3 + l3_off) = COMBINEPTE((rt_size_t)pa, attr);
+ rt_hw_cpu_dcache_clean(mmu_l3 + l3_off, sizeof(*(mmu_l3 + l3_off)));
return 0;
}
-//check whether the range of virtual address are free
-static int check_vaddr(rt_mmu_info *mmu_info,void *va,rt_size_t pages)
+/** rt_hw_mmu_map will never override existed page table entry */
+void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr,
+ size_t size, size_t attr)
{
- rt_size_t loop_va = __UMASKVALUE((rt_size_t)va,PAGE_OFFSET_MASK);
- rt_size_t l1_off,l2_off,l3_off;
- rt_size_t *mmu_l1,*mmu_l2,*mmu_l3;
+ int ret = -1;
+ void *unmap_va = v_addr;
+ size_t npages = size >> ARCH_PAGE_SHIFT;
- if(!pages)
+ // TODO trying with HUGEPAGE here
+ while (npages--)
{
- return -1;
- }
-
- if(!mmu_info)
- {
- return -1;
- }
-
- while(pages--)
- {
- l1_off = GET_L1(loop_va);
- l2_off = GET_L2(loop_va);
- l3_off = GET_L3(loop_va);
- mmu_l1 = ((rt_size_t *)mmu_info -> vtable) + l1_off;
-
- if(PTE_USED(*mmu_l1))
+ ret = _map_one_page(aspace, v_addr, p_addr, attr);
+ if (ret != 0)
{
- RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1));
- mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1),mmu_info -> pv_off) + l2_off;
-
- if(PTE_USED(*mmu_l2))
+ /* error, undo map */
+ while (unmap_va != v_addr)
{
- RT_ASSERT(!PAGE_IS_LEAF(*mmu_l2));
- mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l2),mmu_info -> pv_off) + l3_off;
-
- if(PTE_USED(*mmu_l3))
- {
- RT_ASSERT(PAGE_IS_LEAF(*mmu_l3));
- return -1;
- }
+ MM_PGTBL_LOCK(aspace);
+ _unmap_area(aspace, unmap_va, ARCH_PAGE_SIZE);
+ MM_PGTBL_UNLOCK(aspace);
+ unmap_va += ARCH_PAGE_SIZE;
}
+ break;
}
+ v_addr += ARCH_PAGE_SIZE;
+ p_addr += ARCH_PAGE_SIZE;
+ }
- loop_va += PAGE_SIZE;
+ if (ret == 0)
+ {
+ return unmap_va;
}
- return 0;
+ return NULL;
}
-static void __rt_hw_mmu_unmap(rt_mmu_info *mmu_info,void *v_addr,rt_size_t npages)
+static void _unmap_pte(rt_size_t *pentry, rt_size_t *lvl_entry[], int level)
{
- rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr,PAGE_OFFSET_MASK);
- rt_size_t l1_off,l2_off,l3_off;
- rt_size_t *mmu_l1,*mmu_l2,*mmu_l3;
- rt_size_t *ref_cnt;
-
- RT_ASSERT(mmu_info);
-
- while(npages--)
+ int loop_flag = 1;
+ while (loop_flag)
{
- l1_off = (rt_size_t)GET_L1(loop_va);
- RT_ASSERT((l1_off >= mmu_info -> vstart) && (l1_off <= mmu_info -> vend));
- l2_off = (rt_size_t)GET_L2(loop_va);
- l3_off = (rt_size_t)GET_L3(loop_va);
- mmu_l1 = ((rt_size_t *)mmu_info -> vtable) + l1_off;
- RT_ASSERT(PTE_USED(*mmu_l1))
- RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1));
- mmu_l2 = ((rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1),mmu_info -> pv_off)) + l2_off;
- RT_ASSERT(PTE_USED(*mmu_l2));
- RT_ASSERT(!PAGE_IS_LEAF(*mmu_l2));
- mmu_l3 = ((rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l2),mmu_info -> pv_off)) + l3_off;
- RT_ASSERT(PTE_USED(*mmu_l3));
- RT_ASSERT(PAGE_IS_LEAF(*(mmu_l3)));
- *mmu_l3 = 0;
- rt_hw_cpu_dcache_clean(mmu_l3,sizeof(*mmu_l3));
- mmu_l3 -= l3_off;
- ref_cnt = mmu_l3 + __SIZE(VPN0_BIT);
- (*ref_cnt)--;
+ loop_flag = 0;
+ *pentry = 0;
+ rt_hw_cpu_dcache_clean(pentry, sizeof(*pentry));
- if(!*ref_cnt)
+ // we don't handle level 0, which is maintained by caller
+ if (level > 0)
{
- //release level 3 page
- rt_pages_free(mmu_l3,1);//entry page and ref_cnt page
- *mmu_l2 = 0;
- rt_hw_cpu_dcache_clean(mmu_l2,sizeof(*mmu_l2));
- mmu_l2 -= l2_off;
+ void *page = (void *)((rt_ubase_t)pentry & ~ARCH_PAGE_MASK);
- ref_cnt = mmu_l2 + __SIZE(VPN1_BIT);
- (*ref_cnt)--;
+ // decrease reference from child page to parent
+ rt_pages_free(page, 0);
- if(!*ref_cnt)
+ int free = rt_page_ref_get(page, 0);
+ if (free == 1)
{
- //release level 2 page
- rt_pages_free(mmu_l2,1);//entry page and ref_cnt page
- *mmu_l1 = 0;
- rt_hw_cpu_dcache_clean(mmu_l1,sizeof(*mmu_l1));
+ rt_pages_free(page, 0);
+ pentry = lvl_entry[--level];
+ loop_flag = 1;
}
}
-
- loop_va += PAGE_SIZE;
}
}
-static int __rt_hw_mmu_map(rt_mmu_info *mmu_info,void *v_addr,void *p_addr,rt_size_t npages,rt_size_t attr)
+static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr, size_t size)
{
- rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr,PAGE_OFFSET_MASK);
- rt_size_t loop_pa = __UMASKVALUE((rt_size_t)p_addr,PAGE_OFFSET_MASK);
- rt_size_t l1_off,l2_off,l3_off;
- rt_size_t *mmu_l1,*mmu_l2,*mmu_l3;
- rt_size_t *ref_cnt;
- //rt_kprintf("v_addr = 0x%p,p_addr = 0x%p,npages = %lu\n",v_addr,p_addr,npages);
+ rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr, PAGE_OFFSET_MASK);
+ size_t unmapped = 0;
+
+ int i = 0;
+ rt_size_t lvl_off[3];
+ rt_size_t *lvl_entry[3];
+ lvl_off[0] = (rt_size_t)GET_L1(loop_va);
+ lvl_off[1] = (rt_size_t)GET_L2(loop_va);
+ lvl_off[2] = (rt_size_t)GET_L3(loop_va);
+ unmapped = 1 << (ARCH_PAGE_SHIFT + ARCH_INDEX_WIDTH * 2ul);
+
+ rt_size_t *pentry;
+ lvl_entry[i] = ((rt_size_t *)aspace->page_table + lvl_off[i]);
+ pentry = lvl_entry[i];
- if(!mmu_info)
+ // find leaf page table entry
+ while (PTE_USED(*pentry) && !PAGE_IS_LEAF(*pentry))
{
- return -1;
+ i += 1;
+ lvl_entry[i] = ((rt_size_t *)PPN_TO_VPN(GET_PADDR(*pentry), PV_OFFSET) +
+ lvl_off[i]);
+ pentry = lvl_entry[i];
+ unmapped >>= ARCH_INDEX_WIDTH;
}
- while(npages--)
+ // clear PTE & setup its
+ if (PTE_USED(*pentry))
{
- l1_off = GET_L1(loop_va);
- l2_off = GET_L2(loop_va);
- l3_off = GET_L3(loop_va);
- mmu_l1 = ((rt_size_t *)mmu_info -> vtable) + l1_off;
-
- if(PTE_USED(*mmu_l1))
- {
- RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1));
- mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1),mmu_info -> pv_off);
- }
- else
- {
- mmu_l2 = (rt_size_t *)rt_pages_alloc(1);
+ _unmap_pte(pentry, lvl_entry, i);
+ }
- if(mmu_l2)
- {
- rt_memset(mmu_l2,0,PAGE_SIZE * 2);
- rt_hw_cpu_dcache_clean(mmu_l2,PAGE_SIZE * 2);
- *mmu_l1 = COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l2,mmu_info -> pv_off),PAGE_DEFAULT_ATTR_NEXT);
- rt_hw_cpu_dcache_clean(mmu_l1,sizeof(*mmu_l1));
- }
- else
- {
- __rt_hw_mmu_unmap(mmu_info,v_addr,npages);
- return -1;
- }
- }
+ return unmapped;
+}
- if(PTE_USED(*(mmu_l2 + l2_off)))
- {
- RT_ASSERT(!PAGE_IS_LEAF(*(mmu_l2 + l2_off)));
- mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)),mmu_info -> pv_off);
- }
- else
- {
- mmu_l3 = (rt_size_t *)rt_pages_alloc(1);
+/** unmap is different from map that it can handle multiple pages */
+void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size)
+{
+ // caller guarantee that v_addr & size are page aligned
+ if (!aspace->page_table)
+ {
+ return;
+ }
+ size_t unmapped = 0;
- if(mmu_l3)
- {
- rt_memset(mmu_l3,0,PAGE_SIZE * 2);
- rt_hw_cpu_dcache_clean(mmu_l3,PAGE_SIZE * 2);
- *(mmu_l2 + l2_off) = COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l3,mmu_info -> pv_off),PAGE_DEFAULT_ATTR_NEXT);
- rt_hw_cpu_dcache_clean(mmu_l2,sizeof(*mmu_l2));
- ref_cnt = mmu_l2 + __SIZE(VPN1_BIT);
- (*ref_cnt)++;
- }
- else
- {
- __rt_hw_mmu_unmap(mmu_info,v_addr,npages);
- return -1;
- }
- }
+ while (size > 0)
+ {
+ MM_PGTBL_LOCK(aspace);
+ unmapped = _unmap_area(aspace, v_addr, size);
+ MM_PGTBL_UNLOCK(aspace);
- RT_ASSERT(!PTE_USED(*(mmu_l3 + l3_off)));
- ref_cnt = mmu_l3 + __SIZE(VPN0_BIT);
- (*ref_cnt)++;
- *(mmu_l3 + l3_off) = COMBINEPTE((rt_size_t)loop_pa,PAGE_DEFAULT_ATTR_LEAF);
- rt_hw_cpu_dcache_clean(mmu_l3 + l3_off,sizeof(*(mmu_l3 + l3_off)));
+ // when unmapped == 0, region not exist in pgtbl
+ if (!unmapped || unmapped > size)
+ break;
- loop_va += PAGE_SIZE;
- loop_pa += PAGE_SIZE;
+ size -= unmapped;
+ v_addr += unmapped;
}
+}
- return 0;
+#ifdef RT_USING_SMART
+static inline void _init_region(void *vaddr, size_t size)
+{
+ rt_ioremap_start = vaddr;
+ rt_ioremap_size = size;
+ rt_mpr_start = rt_ioremap_start - rt_mpr_size;
+ rt_kprintf("rt_ioremap_start: %p, rt_mpr_start: %p\n", rt_ioremap_start, rt_mpr_start);
+}
+#else
+static inline void _init_region(void *vaddr, size_t size)
+{
+ rt_mpr_start = vaddr - rt_mpr_size;
}
+#endif
-void *_rt_hw_mmu_map(rt_mmu_info *mmu_info,void *v_addr,void *p_addr,rt_size_t size,rt_size_t attr)
+int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_size_t size,
+ rt_size_t *vtable, rt_size_t pv_off)
{
- rt_size_t pa_s,pa_e;
- rt_size_t vaddr;
- rt_size_t pages;
- int ret;
+ size_t l1_off, va_s, va_e;
+ rt_base_t level;
- if(!size)
+ if ((!aspace) || (!vtable))
{
- return 0;
+ return -1;
}
- pa_s = (rt_size_t)p_addr;
- pa_e = ((rt_size_t)p_addr) + size - 1;
- pa_s = GET_PF_ID(pa_s);
- pa_e = GET_PF_ID(pa_e);
- pages = pa_e - pa_s + 1;
+ va_s = (rt_size_t)v_address;
+ va_e = ((rt_size_t)v_address) + size - 1;
- if(v_addr)
+ if (va_e < va_s)
{
- vaddr = (rt_size_t)v_addr;
- pa_s = (rt_size_t)p_addr;
-
- if(GET_PF_OFFSET(vaddr) != GET_PF_OFFSET(pa_s))
- {
- return 0;
- }
+ return -1;
+ }
- vaddr = __UMASKVALUE(vaddr,PAGE_OFFSET_MASK);
+ // convert address to PPN2 index
+ va_s = GET_L1(va_s);
+ va_e = GET_L1(va_e);
- if(check_vaddr(mmu_info,(void *)vaddr,pages) != 0)
- {
- return 0;
- }
- }
- else
+ if (va_s == 0)
{
- vaddr = find_vaddr(mmu_info,pages);
+ return -1;
}
- if(vaddr)
+ // vtable initialization check
+ for (l1_off = va_s; l1_off <= va_e; l1_off++)
{
- ret = __rt_hw_mmu_map(mmu_info,(void *)vaddr,p_addr,pages,attr);
+ size_t v = vtable[l1_off];
- if(ret == 0)
+ if (v)
{
- rt_hw_cpu_tlb_invalidate();
- return (void *)(vaddr | GET_PF_OFFSET((rt_size_t)p_addr));
+ return -1;
}
}
+ rt_aspace_init(&rt_kernel_space, (void *)0x1000, USER_VADDR_START - 0x1000,
+ vtable);
+
+ _init_region(v_address, size);
return 0;
}
-static int __rt_hw_mmu_map_auto(rt_mmu_info *mmu_info,void *v_addr,rt_size_t npages,rt_size_t attr)
+const static int max_level =
+ (ARCH_VADDR_WIDTH - ARCH_PAGE_SHIFT) / ARCH_INDEX_WIDTH;
+
+static inline uintptr_t _get_level_size(int level)
+{
+ return 1ul << (ARCH_PAGE_SHIFT + (max_level - level) * ARCH_INDEX_WIDTH);
+}
+
+static rt_size_t *_query(struct rt_aspace *aspace, void *vaddr, int *level)
{
- rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr,PAGE_OFFSET_MASK);
- rt_size_t loop_pa;
- rt_size_t l1_off,l2_off,l3_off;
- rt_size_t *mmu_l1,*mmu_l2,*mmu_l3;
- rt_size_t *ref_cnt;
- rt_size_t i;
- void *va,*pa;
+ rt_size_t l1_off, l2_off, l3_off;
+ rt_size_t *mmu_l1, *mmu_l2, *mmu_l3;
+ rt_size_t pa;
- if(!mmu_info)
- {
- return -1;
- }
+ l1_off = GET_L1((rt_size_t)vaddr);
+ l2_off = GET_L2((rt_size_t)vaddr);
+ l3_off = GET_L3((rt_size_t)vaddr);
- while(npages--)
+ if (!aspace)
{
- loop_pa = (rt_size_t)rt_pages_alloc(0);
+ LOG_W("%s: no aspace", __func__);
+ return RT_NULL;
+ }
- if(!loop_pa)
- {
- goto err;
- }
+ mmu_l1 = ((rt_size_t *)aspace->page_table) + l1_off;
- if(__rt_hw_mmu_map(mmu_info,(void *)loop_va,(void *)loop_pa,1,attr) < 0)
+ if (PTE_USED(*mmu_l1))
+ {
+ if (*mmu_l1 & PTE_XWR_MASK)
{
- goto err;
+ *level = 1;
+ return mmu_l1;
}
- loop_va += PAGE_SIZE;
- }
+ mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET);
- return 0;
-
- err:
- va = (void *)__UMASKVALUE((rt_size_t)v_addr,PAGE_OFFSET_MASK);
-
- for(i = 0;i < npages;i++)
+ if (PTE_USED(*(mmu_l2 + l2_off)))
{
- pa = rt_hw_mmu_v2p(mmu_info,va);
-
- if(pa)
+ if (*(mmu_l2 + l2_off) & PTE_XWR_MASK)
{
- rt_pages_free((void *)PPN_TO_VPN(pa,mmu_info -> pv_off),0);
+ *level = 2;
+ return mmu_l2 + l2_off;
}
- va = (void *)((rt_uint8_t *)va + PAGE_SIZE);
+ mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)),
+ PV_OFFSET);
+
+ if (PTE_USED(*(mmu_l3 + l3_off)))
+ {
+ *level = 3;
+ return mmu_l3 + l3_off;
+ }
}
+ }
- __rt_hw_mmu_unmap(mmu_info,v_addr,npages);
- return -1;
+ return RT_NULL;
}
-void *_rt_hw_mmu_map_auto(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size,rt_size_t attr)
+void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr)
{
- rt_size_t vaddr;
- rt_size_t offset;
- rt_size_t pages;
- int ret;
-
- if(!size)
- {
- return 0;
- }
+ int level;
+ uintptr_t *pte = _query(aspace, vaddr, &level);
+ uintptr_t paddr;
- offset = GET_PF_OFFSET((rt_size_t)v_addr);
- size += (offset + PAGE_SIZE - 1);
- pages = size >> PAGE_OFFSET_BIT;
-
- if(v_addr)
+ if (pte)
{
- vaddr = __UMASKVALUE((rt_size_t)v_addr,PAGE_OFFSET_MASK);
-
- if(check_vaddr(mmu_info,(void *)vaddr,pages) != 0)
- {
- return 0;
- }
+ paddr = GET_PADDR(*pte);
+ paddr |= ((intptr_t)vaddr & (_get_level_size(level) - 1));
}
else
{
- vaddr = find_vaddr(mmu_info,pages);
- }
-
- if(vaddr)
- {
- ret = __rt_hw_mmu_map_auto(mmu_info,(void *)vaddr,pages,attr);
-
- if(ret == 0)
- {
- rt_hw_cpu_tlb_invalidate();
- return (void *)(vaddr | offset);
- }
+ paddr = 0;
}
-
- return 0;
+ return (void *)paddr;
}
-void _rt_hw_mmu_unmap(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size)
+static int _noncache(uintptr_t *pte)
{
- rt_size_t va_s,va_e;
- rt_size_t pages;
-
- va_s = ((rt_size_t)v_addr) >> PAGE_OFFSET_BIT;
- va_e = (((rt_size_t)v_addr) + size - 1) >> PAGE_OFFSET_BIT;
- pages = va_e - va_s + 1;
- __rt_hw_mmu_unmap(mmu_info,v_addr,pages);
- rt_hw_cpu_tlb_invalidate();
+ return 0;
}
-void *rt_hw_mmu_map(rt_mmu_info *mmu_info,void *v_addr,void *p_addr,rt_size_t size,rt_size_t attr)
+static int _cache(uintptr_t *pte)
{
- void *ret;
- rt_base_t level;
-
- level = rt_hw_interrupt_disable();
- ret = _rt_hw_mmu_map(mmu_info,v_addr,p_addr,size,attr);
- rt_hw_interrupt_enable(level);
- return ret;
+ return 0;
}
-void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size,rt_size_t attr)
-{
- void *ret;
- rt_base_t level;
-
- level = rt_hw_interrupt_disable();
- ret = _rt_hw_mmu_map_auto(mmu_info,v_addr,size,attr);
- rt_hw_interrupt_enable(level);
- return ret;
-}
+static int (*control_handler[MMU_CNTL_DUMMY_END])(uintptr_t *pte) = {
+ [MMU_CNTL_CACHE] = _cache,
+ [MMU_CNTL_NONCACHE] = _noncache,
+};
-void rt_hw_mmu_unmap(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size)
+int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
+ enum rt_mmu_cntl cmd)
{
- rt_base_t level;
-
- level = rt_hw_interrupt_disable();
- _rt_hw_mmu_unmap(mmu_info,v_addr,size);
- rt_hw_interrupt_enable(level);
-}
+ int level;
+ int err = -RT_EINVAL;
+ void *vend = vaddr + size;
-void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info,void *v_addr)
-{
- rt_size_t l1_off,l2_off,l3_off;
- rt_size_t *mmu_l1,*mmu_l2,*mmu_l3;
- rt_size_t pa;
+ int (*handler)(uintptr_t * pte);
+ if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END)
+ {
+ handler = control_handler[cmd];
- l1_off = GET_L1((rt_size_t)v_addr);
- l2_off = GET_L2((rt_size_t)v_addr);
- l3_off = GET_L3((rt_size_t)v_addr);
+ while (vaddr < vend)
+ {
+ uintptr_t *pte = _query(aspace, vaddr, &level);
+ void *range_end = vaddr + _get_level_size(level);
+ RT_ASSERT(range_end < vend);
- if(!mmu_info)
+ if (pte)
+ {
+ err = handler(pte);
+ RT_ASSERT(err == RT_EOK);
+ }
+ vaddr = range_end;
+ }
+ }
+ else
{
- return RT_NULL;
+ err = -RT_ENOSYS;
}
- mmu_l1 = ((rt_size_t *)mmu_info -> vtable) + l1_off;
+ return err;
+}
- if(PTE_USED(*mmu_l1))
+/**
+ * @brief setup Page Table for kernel space. It's a fixed map
+ * and all mappings cannot be changed after initialization.
+ *
+ * Memory region in struct mem_desc must be page aligned,
+ * otherwise is a failure and no report will be
+ * returned.
+ *
+ * @param mmu_info
+ * @param mdesc
+ * @param desc_nr
+ */
+void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr)
+{
+ void *err;
+ for (size_t i = 0; i < desc_nr; i++)
{
- RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1));
- mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1),mmu_info -> pv_off);
-
- if(PTE_USED(*(mmu_l2 + l2_off)))
+ size_t attr;
+ switch (mdesc->attr)
{
- RT_ASSERT(!PAGE_IS_LEAF(*(mmu_l2 + l2_off)));
- mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)),mmu_info -> pv_off);
-
- if(PTE_USED(*(mmu_l3 + l3_off)))
- {
- RT_ASSERT(PAGE_IS_LEAF(*(mmu_l3 + l3_off)));
- return (void *)(GET_PADDR(*(mmu_l3 + l3_off)) | GET_PF_OFFSET((rt_size_t)v_addr));
- }
+ case NORMAL_MEM:
+ attr = MMU_MAP_K_RWCB;
+ break;
+ case NORMAL_NOCACHE_MEM:
+ attr = MMU_MAP_K_RWCB;
+ break;
+ case DEVICE_MEM:
+ attr = MMU_MAP_K_DEVICE;
+ break;
+ default:
+ attr = MMU_MAP_K_DEVICE;
}
+
+ struct rt_mm_va_hint hint = {.flags = MMF_MAP_FIXED,
+ .limit_start = aspace->start,
+ .limit_range_size = aspace->size,
+ .map_size = mdesc->vaddr_end -
+ mdesc->vaddr_start + 1,
+ .prefer = (void *)mdesc->vaddr_start};
+
+ rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr,
+ mdesc->paddr_start >> MM_PAGE_SHIFT, &err);
+ mdesc++;
}
- return RT_NULL;
+ rt_hw_aspace_switch(&rt_kernel_space);
+ rt_page_cleanup();
}
-void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info,void *v_addr)
+void rt_hw_mmu_kernel_map_init(rt_aspace_t aspace, rt_size_t vaddr_start, rt_size_t size)
{
- void *ret;
- rt_base_t level;
+ rt_size_t paddr_start =
+ __UMASKVALUE(VPN_TO_PPN(vaddr_start, PV_OFFSET), PAGE_OFFSET_MASK);
+ rt_size_t va_s = GET_L1(vaddr_start);
+ rt_size_t va_e = GET_L1(vaddr_start + size - 1);
+ rt_size_t i;
+
+ for (i = va_s; i <= va_e; i++)
+ {
+ MMUTable[i] =
+ COMBINEPTE(paddr_start, PAGE_ATTR_RWX | PTE_G | PTE_V | PTE_CACHE |
+ PTE_SHARE | PTE_BUF | PTE_A | PTE_D);
+ paddr_start += L1_PAGE_SIZE;
+ }
- level = rt_hw_interrupt_disable();
- ret = _rt_hw_mmu_v2p(mmu_info,v_addr);
- rt_hw_interrupt_enable(level);
- return ret;
+ rt_hw_tlb_invalidate_all_local();
}
diff --git a/libcpu/risc-v/t-head/c906/mmu.h b/libcpu/risc-v/t-head/c906/mmu.h
index 09fdc494e7..bb3eb69cd0 100644
--- a/libcpu/risc-v/t-head/c906/mmu.h
+++ b/libcpu/risc-v/t-head/c906/mmu.h
@@ -13,45 +13,83 @@
#include "riscv.h"
#include "riscv_mmu.h"
+#include <mm_aspace.h>
+#include <stddef.h>
+
+/* RAM, Flash, or ROM */
+#define NORMAL_MEM 0
+/* normal nocache memory mapping type */
+#define NORMAL_NOCACHE_MEM 1
+/* MMIO region */
+#define DEVICE_MEM 2
+
+typedef size_t rt_pte_t;
+
struct mem_desc
{
rt_size_t vaddr_start;
rt_size_t vaddr_end;
rt_size_t paddr_start;
rt_size_t attr;
+ struct rt_varea varea;
};
-#define GET_PF_ID(addr) ((addr) >> PAGE_OFFSET_BIT)
-#define GET_PF_OFFSET(addr) __MASKVALUE(addr,PAGE_OFFSET_MASK)
-#define GET_L1(addr) __PARTBIT(addr,VPN2_SHIFT,VPN2_BIT)
-#define GET_L2(addr) __PARTBIT(addr,VPN1_SHIFT,VPN1_BIT)
-#define GET_L3(addr) __PARTBIT(addr,VPN0_SHIFT,VPN0_BIT)
-#define GET_PPN(pte) (__PARTBIT(pte,PTE_PPN_SHIFT,PHYSICAL_ADDRESS_WIDTH_BITS - PTE_PPN_SHIFT))
-#define GET_PADDR(pte) (GET_PPN(pte) << PAGE_OFFSET_BIT)
-#define VPN_TO_PPN(vaddr,pv_off) (((rt_size_t)(vaddr)) + (pv_off))
-#define PPN_TO_VPN(paddr,pv_off) (((rt_size_t)(paddr)) - (pv_off))
-#define COMBINEVADDR(l1_off,l2_off,l3_off) (((l1_off) << VPN2_SHIFT) | ((l2_off) << VPN1_SHIFT) | ((l3_off) << VPN0_SHIFT))
-#define COMBINEPTE(paddr,attr) ((((paddr) >> PAGE_OFFSET_BIT) << PTE_PPN_SHIFT) | (attr))
+#define GET_PF_ID(addr) ((addr) >> PAGE_OFFSET_BIT)
+#define GET_PF_OFFSET(addr) __MASKVALUE(addr, PAGE_OFFSET_MASK)
+#define GET_L1(addr) __PARTBIT(addr, VPN2_SHIFT, VPN2_BIT)
+#define GET_L2(addr) __PARTBIT(addr, VPN1_SHIFT, VPN1_BIT)
+#define GET_L3(addr) __PARTBIT(addr, VPN0_SHIFT, VPN0_BIT)
+#define GET_PPN(pte) \
+ (__PARTBIT(pte, PTE_PPN_SHIFT, PHYSICAL_ADDRESS_WIDTH_BITS - PTE_PPN_SHIFT))
+#define GET_PADDR(pte) (GET_PPN(pte) << PAGE_OFFSET_BIT)
+#define VPN_TO_PPN(vaddr, pv_off) (((rt_size_t)(vaddr)) + (pv_off))
+#define PPN_TO_VPN(paddr, pv_off) (((rt_size_t)(paddr)) - (pv_off))
+#define COMBINEVADDR(l1_off, l2_off, l3_off) \
+ (((l1_off) << VPN2_SHIFT) | ((l2_off) << VPN1_SHIFT) | \
+ ((l3_off) << VPN0_SHIFT))
+#define COMBINEPTE(paddr, attr) \
+ ((((paddr) >> PAGE_OFFSET_BIT) << PTE_PPN_SHIFT) | (attr))
-typedef struct
-{
- size_t *vtable;
- size_t vstart;
- size_t vend;
- size_t pv_off;
-}rt_mmu_info;
+#define ARCH_ADDRESS_WIDTH_BITS 64
+
+#define MMU_MAP_ERROR_VANOTALIGN -1
+#define MMU_MAP_ERROR_PANOTALIGN -2
+#define MMU_MAP_ERROR_NOPAGE -3
+#define MMU_MAP_ERROR_CONFLICT -4
void *rt_hw_mmu_tbl_get();
-void rt_hw_mmu_switch(void *mmu_table);
-int rt_hw_mmu_map_init(rt_mmu_info *mmu_info,void *v_address,rt_size_t size,rt_size_t *vtable,rt_size_t pv_off);
-void rt_hw_mmu_kernel_map_init(rt_mmu_info *mmu_info,rt_size_t vaddr_start,rt_size_t size);
-void *_rt_hw_mmu_map(rt_mmu_info *mmu_info,void *v_addr,void *p_addr,rt_size_t size,rt_size_t attr);
-void *_rt_hw_mmu_map_auto(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size,rt_size_t attr);
-void _rt_hw_mmu_unmap(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size);
-void *rt_hw_mmu_map(rt_mmu_info *mmu_info,void *v_addr,void *p_addr,rt_size_t size,rt_size_t attr);
-void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size,rt_size_t attr);
-void rt_hw_mmu_unmap(rt_mmu_info *mmu_info,void *v_addr,rt_size_t size);
-void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info,void *v_addr);
-void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info,void *v_addr);
+int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_size_t size,
+ rt_size_t *vtable, rt_size_t pv_off);
+void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr);
+void rt_hw_mmu_kernel_map_init(rt_aspace_t aspace, rt_size_t vaddr_start,
+ rt_size_t size);
+void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
+ size_t attr);
+void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size);
+void rt_hw_aspace_switch(rt_aspace_t aspace);
+void *rt_hw_mmu_v2p(rt_aspace_t aspace, void *vaddr);
+
+static inline void *_rt_kmem_v2p(void *vaddr)
+{
+ return rt_hw_mmu_v2p(&rt_kernel_space, vaddr);
+}
+
+static inline void *rt_kmem_v2p(void *vaddr)
+{
+ MM_PGTBL_LOCK(&rt_kernel_space);
+ void *paddr = _rt_kmem_v2p(vaddr);
+ MM_PGTBL_UNLOCK(&rt_kernel_space);
+ return paddr;
+}
+
+enum rt_mmu_cntl
+{
+ MMU_CNTL_NONCACHE,
+ MMU_CNTL_CACHE,
+ MMU_CNTL_DUMMY_END,
+};
+
+int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
+ enum rt_mmu_cntl cmd);
#endif
diff --git a/libcpu/risc-v/t-head/c906/riscv_mmu.c b/libcpu/risc-v/t-head/c906/riscv_mmu.c
index 9ba48e34c4..e648c52269 100644
--- a/libcpu/risc-v/t-head/c906/riscv_mmu.c
+++ b/libcpu/risc-v/t-head/c906/riscv_mmu.c
@@ -18,20 +18,12 @@
#include "riscv_mmu.h"
-void mmu_set_pagetable(rt_ubase_t addr)
-{
- RT_ASSERT(__CHECKALIGN(addr,PAGE_OFFSET_BIT));
- RT_ASSERT(__CHECKUPBOUND(addr,PHYSICAL_ADDRESS_WIDTH_BITS));
- write_csr(satp,(((size_t)8) << 60) | (addr >> PAGE_OFFSET_BIT));
- mmu_flush_tlb();
-}
-
void mmu_enable_user_page_access()
{
- set_csr(sstatus,SSTATUS_PUM);
+ set_csr(sstatus,SSTATUS_SUM);
}
void mmu_disable_user_page_access()
{
- clear_csr(sstatus,SSTATUS_PUM);
+ clear_csr(sstatus,SSTATUS_SUM);
}
diff --git a/libcpu/risc-v/t-head/c906/riscv_mmu.h b/libcpu/risc-v/t-head/c906/riscv_mmu.h
index 138bd596e9..386bda9884 100644
--- a/libcpu/risc-v/t-head/c906/riscv_mmu.h
+++ b/libcpu/risc-v/t-head/c906/riscv_mmu.h
@@ -12,8 +12,8 @@
#ifndef __RISCV_MMU_H__
#define __RISCV_MMU_H__
-#include <rthw.h>
#include <rtthread.h>
+#include <rthw.h>
#include "riscv.h"
#undef PAGE_SIZE
@@ -59,21 +59,42 @@
#define PAGE_ATTR_USER (PTE_U)
#define PAGE_ATTR_SYSTEM (0)
-#define PAGE_DEFAULT_ATTR_LEAF (PAGE_ATTR_RWX | PAGE_ATTR_USER | PTE_V | PTE_G | PTE_SHARE | PTE_BUF | PTE_CACHE | PTE_A | PTE_D)
-#define PAGE_DEFAULT_ATTR_NEXT (PAGE_ATTR_NEXT_LEVEL | PTE_V | PTE_G | PTE_SHARE | PTE_BUF | PTE_CACHE | PTE_A | PTE_D)
+#define PAGE_DEFAULT_ATTR_LEAF (PAGE_ATTR_RWX | PAGE_ATTR_USER | PTE_V | PTE_G)
+#define PAGE_DEFAULT_ATTR_NEXT (PAGE_ATTR_NEXT_LEVEL | PTE_V | PTE_G)
+
+#define PAGE_IS_LEAF(pte) __MASKVALUE(pte, PAGE_ATTR_RWX)
+
+#define PTE_USED(pte) __MASKVALUE(pte, PTE_V)
+
+/**
+ * encoding of SATP (Supervisor Address Translation and Protection register)
+ */
+#define SATP_MODE_OFFSET 60
+#define SATP_MODE_BARE 0
+#define SATP_MODE_SV39 8
+#define SATP_MODE_SV48 9
+#define SATP_MODE_SV57 10
+#define SATP_MODE_SV64 11
+
+#define ARCH_VADDR_WIDTH 39
+#define SATP_MODE SATP_MODE_SV39
-#define PAGE_IS_LEAF(pte) __MASKVALUE(pte,PAGE_ATTR_RWX)
+#define MMU_MAP_K_DEVICE (PTE_G | PTE_W | PTE_R | PTE_V)
+#define MMU_MAP_K_RWCB (PTE_G | PTE_X | PTE_W | PTE_R | PTE_V)
+#define MMU_MAP_U_RWCB (PTE_U | PTE_X | PTE_W | PTE_R | PTE_V)
+#define MMU_MAP_U_RWCB_XN (PTE_U | PTE_W | PTE_R | PTE_V)
+#define MMU_MAP_U_RW (PTE_U | PTE_X | PTE_W | PTE_R | PTE_V)
-#define PTE_USED(pte) __MASKVALUE(pte,PTE_V)
+#define PTE_XWR_MASK 0xe
-#define mmu_flush_tlb() do{asm volatile("sfence.vma x0,x0");}while(0)
+#define ARCH_PAGE_SIZE PAGE_SIZE
+#define ARCH_PAGE_MASK (ARCH_PAGE_SIZE - 1)
+#define ARCH_PAGE_SHIFT PAGE_OFFSET_BIT
+#define ARCH_INDEX_WIDTH 9
+#define ARCH_INDEX_SIZE (1ul << ARCH_INDEX_WIDTH)
+#define ARCH_INDEX_MASK (ARCH_INDEX_SIZE - 1)
-//compatible to rt-smart new version
-#define MMU_MAP_K_DEVICE (PAGE_ATTR_RWX | PTE_V | PTE_G | PTE_SO | PTE_BUF | PTE_A | PTE_D)
-#define MMU_MAP_K_RWCB (PAGE_ATTR_RWX | PTE_V | PTE_G | PTE_SHARE | PTE_BUF | PTE_CACHE | PTE_A | PTE_D)
-#define ARCH_PAGE_SIZE PAGE_SIZE
-#define ARCH_PAGE_MASK (ARCH_PAGE_SIZE - 1)
-#define ARCH_PAGE_SHIFT PAGE_OFFSET_BIT
+#define ARCH_MAP_FAILED ((void *)0x8000000000000000)
void mmu_set_pagetable(rt_ubase_t addr);
void mmu_enable_user_page_access();
diff --git a/libcpu/risc-v/t-head/c906/sbi.c b/libcpu/risc-v/t-head/c906/sbi.c
index f848ec7810..e4577e8f05 100644
--- a/libcpu/risc-v/t-head/c906/sbi.c
+++ b/libcpu/risc-v/t-head/c906/sbi.c
@@ -36,12 +36,12 @@
*/
#include "sbi.h"
-#include <stdbool.h>
#include <rtthread.h>
+#include <stdbool.h>
/* SBI Implementation-Specific Definitions */
-#define OPENSBI_VERSION_MAJOR_OFFSET 16
-#define OPENSBI_VERSION_MINOR_MASK 0xFFFF
+#define OPENSBI_VERSION_MAJOR_OFFSET 16
+#define OPENSBI_VERSION_MINOR_MASK 0xFFFF
unsigned long sbi_spec_version;
unsigned long sbi_impl_id;
@@ -51,26 +51,22 @@ static bool has_time_extension = false;
static bool has_ipi_extension = false;
static bool has_rfnc_extension = false;
-static struct sbi_ret
-sbi_get_spec_version(void)
+static struct sbi_ret sbi_get_spec_version(void)
{
return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_SPEC_VERSION));
}
-static struct sbi_ret
-sbi_get_impl_id(void)
+static struct sbi_ret sbi_get_impl_id(void)
{
return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_IMPL_ID));
}
-static struct sbi_ret
-sbi_get_impl_version(void)
+static struct sbi_ret sbi_get_impl_version(void)
{
return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_IMPL_VERSION));
}
-void
-sbi_print_version(void)
+void sbi_print_version(void)
{
u_int major;
u_int minor;
@@ -89,7 +85,8 @@ sbi_print_version(void)
rt_kprintf("SBI: Berkely Boot Loader %lu\n", sbi_impl_version);
break;
case (SBI_IMPL_ID_XVISOR):
- rt_kprintf("SBI: eXtensible Versatile hypervISOR %lu\n", sbi_impl_version);
+ rt_kprintf("SBI: eXtensible Versatile hypervISOR %lu\n",
+ sbi_impl_version);
break;
case (SBI_IMPL_ID_KVM):
rt_kprintf("SBI: Kernel-based Virtual Machine %lu\n", sbi_impl_version);
@@ -116,8 +113,7 @@ sbi_print_version(void)
rt_kprintf("SBI Specification Version: %u.%u\n", major, minor);
}
-void
-sbi_set_timer(uint64_t val)
+void sbi_set_timer(uint64_t val)
{
struct sbi_ret ret;
@@ -133,16 +129,14 @@ sbi_set_timer(uint64_t val)
}
}
-void
-sbi_send_ipi(const unsigned long *hart_mask)
+void sbi_send_ipi(const unsigned long *hart_mask)
{
struct sbi_ret ret;
/* Use the IPI legacy replacement extension, if available. */
if (has_ipi_extension)
{
- ret = SBI_CALL2(SBI_EXT_ID_IPI, SBI_IPI_SEND_IPI,
- *hart_mask, 0);
+ ret = SBI_CALL2(SBI_EXT_ID_IPI, SBI_IPI_SEND_IPI, *hart_mask, 0);
RT_ASSERT(ret.error == SBI_SUCCESS);
}
else
@@ -151,16 +145,15 @@ sbi_send_ipi(const unsigned long *hart_mask)
}
}
-void
-sbi_remote_fence_i(const unsigned long *hart_mask)
+void sbi_remote_fence_i(const unsigned long *hart_mask)
{
struct sbi_ret ret;
/* Use the RFENCE legacy replacement extension, if available. */
if (has_rfnc_extension)
{
- ret = SBI_CALL2(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_FENCE_I,
- *hart_mask, 0);
+ ret =
+ SBI_CALL2(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_FENCE_I, *hart_mask, 0);
RT_ASSERT(ret.error == SBI_SUCCESS);
}
else
@@ -169,28 +162,29 @@ sbi_remote_fence_i(const unsigned long *hart_mask)
}
}
-void
-sbi_remote_sfence_vma(const unsigned long *hart_mask, unsigned long start, unsigned long size)
+int sbi_remote_sfence_vma(const unsigned long *hart_mask,
+ const unsigned long hart_mask_base,
+ unsigned long start, unsigned long size)
{
- struct sbi_ret ret;
+ struct sbi_ret ret = {.error = SBI_SUCCESS};
/* Use the RFENCE legacy replacement extension, if available. */
if (has_rfnc_extension)
{
- ret = SBI_CALL4(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_SFENCE_VMA,
- *hart_mask, 0, start, size);
- RT_ASSERT(ret.error == SBI_SUCCESS);
+ ret = SBI_CALL4(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_SFENCE_VMA, *hart_mask,
+ hart_mask_base, start, size);
}
else
{
- (void)SBI_CALL3(SBI_REMOTE_SFENCE_VMA, 0, (uint64_t)hart_mask,
- start, size);
+ (void)SBI_CALL3(SBI_REMOTE_SFENCE_VMA, 0, (uint64_t)hart_mask, start,
+ size);
}
+ return ret.error;
}
-void
-sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start, unsigned long size,
- unsigned long asid)
+void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
+ unsigned long start, unsigned long size,
+ unsigned long asid)
{
struct sbi_ret ret;
@@ -203,13 +197,13 @@ sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start,
}
else
{
- (void)SBI_CALL4(SBI_REMOTE_SFENCE_VMA_ASID, 0,
- (uint64_t)hart_mask, start, size, asid);
+ (void)SBI_CALL4(SBI_REMOTE_SFENCE_VMA_ASID, 0, (uint64_t)hart_mask,
+ start, size, asid);
}
}
-int
-sbi_hsm_hart_start(unsigned long hart, unsigned long start_addr, unsigned long priv)
+int sbi_hsm_hart_start(unsigned long hart, unsigned long start_addr,
+ unsigned long priv)
{
struct sbi_ret ret;
@@ -217,14 +211,12 @@ sbi_hsm_hart_start(unsigned long hart, unsigned long start_addr, unsigned long p
return (ret.error != 0 ? (int)ret.error : 0);
}
-void
-sbi_hsm_hart_stop(void)
+void sbi_hsm_hart_stop(void)
{
(void)SBI_CALL0(SBI_EXT_ID_HSM, SBI_HSM_HART_STOP);
}
-int
-sbi_hsm_hart_status(unsigned long hart)
+int sbi_hsm_hart_status(unsigned long hart)
{
struct sbi_ret ret;
@@ -233,8 +225,7 @@ sbi_hsm_hart_status(unsigned long hart)
return (ret.error != 0 ? (int)ret.error : (int)ret.value);
}
-void
-sbi_init(void)
+void sbi_init(void)
{
struct sbi_ret sret;
@@ -263,3 +254,11 @@ sbi_init(void)
if (sbi_probe_extension(SBI_EXT_ID_RFNC) != 0)
has_rfnc_extension = true;
}
+
+void rt_hw_console_output(const char *str)
+{
+ while (*str)
+ {
+ sbi_console_putchar(*str++);
+ }
+}
diff --git a/libcpu/risc-v/t-head/c906/sbi.h b/libcpu/risc-v/t-head/c906/sbi.h
index 0e95ede334..ed29ecfd20 100644
--- a/libcpu/risc-v/t-head/c906/sbi.h
+++ b/libcpu/risc-v/t-head/c906/sbi.h
@@ -140,7 +140,7 @@ struct sbi_ret
long value;
};
-static __inline struct sbi_ret
+static inline struct sbi_ret
sbi_call(uint64_t arg7, uint64_t arg6, uint64_t arg0, uint64_t arg1,
uint64_t arg2, uint64_t arg3, uint64_t arg4)
{
@@ -184,7 +184,9 @@ void sbi_send_ipi(const unsigned long *hart_mask);
/* RFENCE extension functions. */
void sbi_remote_fence_i(const unsigned long *hart_mask);
-void sbi_remote_sfence_vma(const unsigned long *hart_mask, unsigned long start, unsigned long size);
+int sbi_remote_sfence_vma(const unsigned long *hart_mask,
+ const unsigned long hart_mask_base,
+ unsigned long start, unsigned long size);
void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start,
unsigned long size, unsigned long asid);
diff --git a/libcpu/risc-v/t-head/c906/tlb.h b/libcpu/risc-v/t-head/c906/tlb.h
new file mode 100644
index 0000000000..5bdf18c189
--- /dev/null
+++ b/libcpu/risc-v/t-head/c906/tlb.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2006-2022, RT-Thread Development Team
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Change Logs:
+ * Date Author Notes
+ * 2022-11-28 WangXiaoyao the first version
+ */
+#ifndef __TLB_H__
+#define __TLB_H__
+
+#include "mm_aspace.h"
+#include "riscv_mmu.h"
+#include "rtdbg.h"
+#include "rtthread.h"
+#include <sbi.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define HANDLE_FAULT(ret) \
+ if (__builtin_expect((ret) != SBI_SUCCESS, 0)) \
+ LOG_W("%s failed", __FUNCTION__);
+
+static inline void rt_hw_tlb_invalidate_all(void)
+{
+ uintptr_t mask = -1ul;
+ HANDLE_FAULT(sbi_remote_sfence_vma(&mask, -1ul, 0, mask));
+}
+
+static inline void rt_hw_tlb_invalidate_all_local(void)
+{
+ __asm__ volatile("sfence.vma" ::: "memory");
+}
+
+static inline void rt_hw_tlb_invalidate_aspace(rt_aspace_t aspace)
+{
+ // TODO ASID
+ rt_hw_tlb_invalidate_all_local();
+}
+
+static inline void rt_hw_tlb_invalidate_page(rt_aspace_t aspace, void *start)
+{
+ __asm__ volatile("sfence.vma %0, zero" ::"r"(start) : "memory");
+}
+
+static inline void rt_hw_tlb_invalidate_range(rt_aspace_t aspace, void *start,
+ size_t size, size_t stride)
+{
+ // huge page is taking as normal page
+ if (size <= ARCH_PAGE_SIZE)
+ {
+ rt_hw_tlb_invalidate_page(aspace, start);
+ }
+ else
+ {
+ rt_hw_tlb_invalidate_aspace(aspace);
+ }
+}
+
+#endif /* __TLB_H__ */
diff --git a/libcpu/risc-v/t-head/c906/trap.c b/libcpu/risc-v/t-head/c906/trap.c
index beca180831..541d24377b 100644
--- a/libcpu/risc-v/t-head/c906/trap.c
+++ b/libcpu/risc-v/t-head/c906/trap.c
@@ -107,7 +107,7 @@ void dump_regs(struct rt_hw_stack_frame *regs)
rt_kprintf("\t%s\n",(regs->sstatus & SSTATUS_SIE) ? "Supervisor Interrupt Enabled" : "Supervisor Interrupt Disabled");
rt_kprintf("\t%s\n",(regs->sstatus & SSTATUS_SPIE) ? "Last Time Supervisor Interrupt Enabled" : "Last Time Supervisor Interrupt Disabled");
rt_kprintf("\t%s\n",(regs->sstatus & SSTATUS_SPP) ? "Last Privilege is Supervisor Mode" : "Last Privilege is User Mode");
- rt_kprintf("\t%s\n",(regs->sstatus & SSTATUS_PUM) ? "Permit to Access User Page" : "Not Permit to Access User Page");
+ rt_kprintf("\t%s\n",(regs->sstatus & SSTATUS_SUM) ? "Permit to Access User Page" : "Not Permit to Access User Page");
rt_kprintf("\t%s\n",(regs->sstatus & (1 << 19)) ? "Permit to Read Executable-only Page" : "Not Permit to Read Executable-only Page");
rt_size_t satp_v = read_csr(satp);
rt_kprintf("satp = 0x%p\n",satp_v);
@@ -202,6 +202,45 @@ void generic_handle_irq(int irq)
plic_complete(irq);
}
+static const char *get_exception_msg(int id)
+{
+ const char *msg;
+ if (id < sizeof(Exception_Name) / sizeof(const char *))
+ {
+ msg = Exception_Name[id];
+ }
+ else
+ {
+ msg = "Unknown Exception";
+ }
+ return msg;
+}
+
+void handle_user(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw_stack_frame *sp)
+{
+ rt_size_t id = __MASKVALUE(scause, __MASK(63UL));
+
+#ifdef RT_USING_USERSPACE
+ /* user page fault */
+ if (id == EP_LOAD_PAGE_FAULT ||
+ id == EP_STORE_PAGE_FAULT)
+ {
+ if (arch_expand_user_stack((void *)stval))
+ {
+ return;
+ }
+ }
+#endif
+ LOG_E("[FATAL ERROR] Exception %ld:%s", id, get_exception_msg(id));
+ LOG_E("scause:0x%p,stval:0x%p,sepc:0x%p", scause, stval, sepc);
+ dump_regs(sp);
+
+ rt_hw_backtrace((uint32_t *)sp->s0_fp, sepc);
+
+ LOG_E("User Fault, killing thread: %s", rt_thread_self()->name);
+ sys_exit(-1);
+}
+
/* Trap entry */
void handle_trap(rt_size_t scause,rt_size_t stval,rt_size_t sepc,struct rt_hw_stack_frame *sp)
{
@@ -238,28 +277,28 @@ void handle_trap(rt_size_t scause,rt_size_t stval,rt_size_t sepc,struct rt_hw_st
}
else
{
-#ifdef RT_USING_SMART
- /* page fault */
- if (id == EP_LOAD_PAGE_FAULT ||
- id == EP_STORE_PAGE_FAULT)
- {
- arch_expand_user_stack((void *)stval);
- return;
- }
-#endif
- if(id < sizeof(Exception_Name) / sizeof(const char *))
- {
- msg = Exception_Name[id];
- }
- else
+ if (!(sp->sstatus & 0x100))
{
- msg = "Unknown Exception";
+ handle_user(scause, stval, sepc, sp);
+ // after handle_user(), return to user space.
+ // otherwise it never returns
+ return ;
}
- rt_kprintf("Unhandled Exception %ld:%s\n",id,msg);
+ // handle kernel exception:
+ rt_kprintf("Unhandled Exception %ld:%s\n", id, get_exception_msg(id));
}
- rt_kprintf("scause:0x%p,stval:0x%p,sepc:0x%p\n",scause,stval,sepc);
+ rt_kprintf("scause:0x%p,stval:0x%p,sepc:0x%p\n", scause, stval, sepc);
dump_regs(sp);
- while(1);
+ rt_kprintf("--------------Thread list--------------\n");
+ rt_kprintf("current thread: %s\n", rt_thread_self()->name);
+ list_process();
+
+ extern struct rt_thread *rt_current_thread;
+ rt_kprintf("--------------Backtrace--------------\n");
+ rt_hw_backtrace((uint32_t *)sp->s0_fp, sepc);
+
+ while (1)
+ ;
}
diff --git a/libcpu/risc-v/virt64/backtrace.c b/libcpu/risc-v/virt64/backtrace.c
index 11651daa16..ad5dd7d2d3 100644
--- a/libcpu/risc-v/virt64/backtrace.c
+++ b/libcpu/risc-v/virt64/backtrace.c
@@ -16,11 +16,10 @@
extern rt_ubase_t __text_start[];
extern rt_ubase_t __text_end[];
-static char *_get_elf_name();
+static char *_get_elf_name(size_t sepc);
void rt_hw_backtrace(rt_uint32_t *ffp, rt_ubase_t sepc)
{
- extern rt_mmu_info mmu_info;
rt_ubase_t *ra;
rt_ubase_t *fp;
rt_ubase_t vas, vae;
@@ -60,13 +59,13 @@ void rt_hw_backtrace(rt_uint32_t *ffp, rt_ubase_t sepc)
}
ra = fp - 1;
- if (!_rt_hw_mmu_v2p(&mmu_info, ra) || *ra < vas || *ra > vae)
+ if (!_rt_kmem_v2p(ra) || *ra < vas || *ra > vae)
break;
rt_kprintf(" %p", *ra - 0x04);
fp = fp - 2;
- if (!_rt_hw_mmu_v2p(&mmu_info, fp))
+ if (!_rt_kmem_v2p(fp))
break;
fp = (rt_ubase_t *)(*fp);
if (!fp)
diff --git a/libcpu/risc-v/virt64/context_gcc.S b/libcpu/risc-v/virt64/context_gcc.S
index c1fe828822..0f3f2e08a5 100644
--- a/libcpu/risc-v/virt64/context_gcc.S
+++ b/libcpu/risc-v/virt64/context_gcc.S
@@ -80,7 +80,7 @@ rt_hw_context_switch_to:
#ifdef RT_USING_SMART
mv a0, s1
- jal lwp_mmu_switch
+ jal lwp_aspace_switch
#endif
RESTORE_CONTEXT
@@ -108,7 +108,7 @@ rt_hw_context_switch:
#ifdef RT_USING_SMART
mv a0, s1
- jal lwp_mmu_switch
+ jal lwp_aspace_switch
#endif
RESTORE_CONTEXT
diff --git a/libcpu/risc-v/virt64/mmu.c b/libcpu/risc-v/virt64/mmu.c
index 3a9daee3dc..17760c8779 100644
--- a/libcpu/risc-v/virt64/mmu.c
+++ b/libcpu/risc-v/virt64/mmu.c
@@ -6,282 +6,72 @@
* Change Logs:
* Date Author Notes
* 2021-01-30 lizhirui first version
+ * 2022-12-13 WangXiaoyao Port to new mm
*/
#include <rtthread.h>
-#include <rthw.h>
+#include <stddef.h>
+#include <stdint.h>
-#ifdef RT_USING_SMART
-
-#include <board.h>
-#include <page.h>
-#include <stdlib.h>
-#include <string.h>
-#include <lwp_mm.h>
#include <cache.h>
+#include <mm_aspace.h>
+#include <mm_page.h>
+#include <mmu.h>
+#include <riscv_mmu.h>
+#include <tlb.h>
-#define DBG_TAG "mmu"
-#define DBG_LVL DBG_INFO
-#include <rtdbg.h>
-
-#include "riscv.h"
-#include "riscv_mmu.h"
-#include "mmu.h"
-
-void *current_mmu_table = RT_NULL;
-
-volatile rt_ubase_t MMUTable[__SIZE(VPN2_BIT)] __attribute__((aligned(4 * 1024)));
-
-static void rt_hw_cpu_tlb_invalidate()
-{
- rt_size_t satpv = read_csr(satp);
- write_csr(satp, satpv);
- mmu_flush_tlb();
-}
-
-void *rt_hw_mmu_tbl_get()
-{
- return current_mmu_table;
-}
-
-void rt_hw_mmu_switch(void *mmu_table)
-{
- current_mmu_table = mmu_table;
- RT_ASSERT(__CHECKALIGN(mmu_table, PAGE_OFFSET_BIT));
- mmu_set_pagetable((rt_ubase_t)mmu_table);
- rt_hw_cpu_dcache_clean_all();
- rt_hw_cpu_icache_invalidate_all();
-}
-
-int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void *v_address, rt_size_t size, rt_size_t *vtable, rt_size_t pv_off)
-{
- size_t l1_off, va_s, va_e;
- rt_base_t level;
-
- if ((!mmu_info) || (!vtable))
- {
- return -1;
- }
-
- va_s = (rt_size_t)v_address;
- va_e = ((rt_size_t)v_address) + size - 1;
-
- if (va_e < va_s)
- {
- return -1;
- }
-
- // convert address to PPN2 index
- va_s = GET_L1(va_s);
- va_e = GET_L1(va_e);
-
- if (va_s == 0)
- {
- return -1;
- }
-
- rt_mm_lock();
-
- // vtable initialization check
- for (l1_off = va_s; l1_off <= va_e; l1_off++)
- {
- size_t v = vtable[l1_off];
-
- if (v)
- {
- rt_mm_unlock();
- return -1;
- }
- }
-
- rt_mm_unlock();
- mmu_info->vtable = vtable;
- mmu_info->vstart = va_s;
- mmu_info->vend = va_e;
- mmu_info->pv_off = pv_off;
-
- return 0;
-}
-
-void rt_hw_mmu_kernel_map_init(rt_mmu_info *mmu_info, rt_size_t vaddr_start, rt_size_t size)
-{
- rt_size_t paddr_start = __UMASKVALUE(VPN_TO_PPN(vaddr_start, mmu_info->pv_off), PAGE_OFFSET_MASK);
- rt_size_t va_s = GET_L1(vaddr_start);
- rt_size_t va_e = GET_L1(vaddr_start + size - 1);
- rt_size_t i;
-
- for (i = va_s; i <= va_e; i++)
- {
- mmu_info->vtable[i] = COMBINEPTE(paddr_start, PAGE_ATTR_RWX | PTE_G | PTE_V);
- paddr_start += L1_PAGE_SIZE;
- }
+#ifdef RT_USING_SMART
+#include <ioremap.h>
+#include <lwp_user_mm.h>
+#include <tlb.h>
+#endif
- rt_hw_cpu_tlb_invalidate();
-}
+#define DBG_TAG "hw.mmu"
+#define DBG_LVL DBG_LOG
+#include <rtdbg.h>
-// find a range of free virtual address specified by pages
-static size_t find_vaddr(rt_mmu_info *mmu_info, int pages)
-{
- size_t loop_pages;
- size_t va;
- size_t find_va = 0;
- int n = 0;
- size_t i;
+#ifndef RT_USING_SMART
+#define PV_OFFSET 0
+#define USER_VADDR_START 0
+#endif
- if (!pages || !mmu_info)
- {
- return 0;
- }
+static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr, size_t size);
- loop_pages = (mmu_info->vend - mmu_info->vstart) ? (mmu_info->vend - mmu_info->vstart) : 1;
- loop_pages <<= (ARCH_INDEX_WIDTH * 2);
- va = mmu_info->vstart;
- va <<= (ARCH_PAGE_SHIFT + ARCH_INDEX_WIDTH * 2);
+static void *current_mmu_table = RT_NULL;
- for (i = 0; i < loop_pages; i++, va += ARCH_PAGE_SIZE)
- {
- if (_rt_hw_mmu_v2p(mmu_info, (void *)va))
- {
- n = 0;
- find_va = 0;
- continue;
- }
- if (!find_va)
- {
- find_va = va;
- }
- n++;
- if (n >= pages)
- {
- return find_va;
- }
- }
- return 0;
-}
+volatile __attribute__((aligned(4 * 1024)))
+rt_ubase_t MMUTable[__SIZE(VPN2_BIT)];
-// check whether the range of virtual address are free
-static int check_vaddr(rt_mmu_info *mmu_info, void *va, rt_size_t pages)
+void rt_hw_aspace_switch(rt_aspace_t aspace)
{
- rt_size_t loop_va = __UMASKVALUE((rt_size_t)va, PAGE_OFFSET_MASK);
- rt_size_t l1_off, l2_off, l3_off;
- rt_size_t *mmu_l1, *mmu_l2, *mmu_l3;
-
- if (!pages)
- {
- return -1;
- }
-
- if (!mmu_info)
- {
- return -1;
- }
-
- while (pages--)
- {
- l1_off = GET_L1(loop_va);
- l2_off = GET_L2(loop_va);
- l3_off = GET_L3(loop_va);
- mmu_l1 = ((rt_size_t *)mmu_info->vtable) + l1_off;
-
- if (PTE_USED(*mmu_l1))
- {
- RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1));
- mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), mmu_info->pv_off) + l2_off;
-
- if (PTE_USED(*mmu_l2))
- {
- RT_ASSERT(!PAGE_IS_LEAF(*mmu_l2));
- mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l2), mmu_info->pv_off) + l3_off;
-
- if (PTE_USED(*mmu_l3))
- {
- RT_ASSERT(PAGE_IS_LEAF(*mmu_l3));
- return -1;
- }
- }
- }
+ uintptr_t page_table = (uintptr_t)_rt_kmem_v2p(aspace->page_table);
+ current_mmu_table = aspace->page_table;
- loop_va += PAGE_SIZE;
- }
-
- return 0;
+ write_csr(satp, (((size_t)SATP_MODE) << SATP_MODE_OFFSET) |
+ ((rt_ubase_t)page_table >> PAGE_OFFSET_BIT));
+ rt_hw_tlb_invalidate_all_local();
}
-static void __rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, rt_size_t npages)
+void *rt_hw_mmu_tbl_get()
{
- rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr, PAGE_OFFSET_MASK);
- rt_size_t l1_off, l2_off, l3_off;
- rt_size_t *mmu_l1, *mmu_l2, *mmu_l3;
-
- RT_ASSERT(mmu_info);
-
- while (npages--)
- {
- l1_off = (rt_size_t)GET_L1(loop_va);
- RT_ASSERT((l1_off >= mmu_info->vstart) && (l1_off <= mmu_info->vend));
- l2_off = (rt_size_t)GET_L2(loop_va);
- l3_off = (rt_size_t)GET_L3(loop_va);
-
- mmu_l1 = ((rt_size_t *)mmu_info->vtable) + l1_off;
- RT_ASSERT(PTE_USED(*mmu_l1))
- RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1));
- mmu_l2 = ((rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), mmu_info->pv_off)) + l2_off;
- RT_ASSERT(PTE_USED(*mmu_l2));
- RT_ASSERT(!PAGE_IS_LEAF(*mmu_l2));
- mmu_l3 = ((rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l2), mmu_info->pv_off)) + l3_off;
- RT_ASSERT(PTE_USED(*mmu_l3));
- RT_ASSERT(PAGE_IS_LEAF(*(mmu_l3)));
-
- *mmu_l3 = 0;
- rt_hw_cpu_dcache_clean(mmu_l3, sizeof(*mmu_l3));
-
- // decrease reference from leaf page to l3 page
- mmu_l3 -= l3_off;
- rt_pages_free(mmu_l3, 0);
- int free = rt_page_ref_get(mmu_l3, 0);
-
- if (free == 1)
- {
- // free l3 page
- rt_pages_free(mmu_l3, 0);
-
- *mmu_l2 = 0;
- rt_hw_cpu_dcache_clean(mmu_l2, sizeof(*mmu_l2));
-
- // decrease reference from l3 page to l2 page
- mmu_l2 -= l2_off;
- rt_pages_free(mmu_l2, 0);
-
- free = rt_page_ref_get(mmu_l2, 0);
- if (free == 1)
- {
- // free l3 page
- rt_pages_free(mmu_l2, 0);
- // reset PTE in l1
- *mmu_l1 = 0;
- rt_hw_cpu_dcache_clean(mmu_l1, sizeof(*mmu_l1));
- }
- }
-
- loop_va += PAGE_SIZE;
- }
+ return current_mmu_table;
}
-static int _mmu_map_one_page(rt_mmu_info *mmu_info, size_t va, size_t pa, size_t attr)
+static int _map_one_page(struct rt_aspace *aspace, void *va, void *pa,
+ size_t attr)
{
rt_size_t l1_off, l2_off, l3_off;
rt_size_t *mmu_l1, *mmu_l2, *mmu_l3;
- l1_off = GET_L1(va);
- l2_off = GET_L2(va);
- l3_off = GET_L3(va);
+ l1_off = GET_L1((size_t)va);
+ l2_off = GET_L2((size_t)va);
+ l3_off = GET_L3((size_t)va);
- mmu_l1 = ((rt_size_t *)mmu_info->vtable) + l1_off;
+ mmu_l1 = ((rt_size_t *)aspace->page_table) + l1_off;
if (PTE_USED(*mmu_l1))
{
- RT_ASSERT(!PAGE_IS_LEAF(*mmu_l1));
- mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), mmu_info->pv_off);
+ mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET);
}
else
{
@@ -291,7 +81,8 @@ static int _mmu_map_one_page(rt_mmu_info *mmu_info, size_t va, size_t pa, size_t
{
rt_memset(mmu_l2, 0, PAGE_SIZE);
rt_hw_cpu_dcache_clean(mmu_l2, PAGE_SIZE);
- *mmu_l1 = COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l2, mmu_info->pv_off), PAGE_DEFAULT_ATTR_NEXT);
+ *mmu_l1 = COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l2, PV_OFFSET),
+ PAGE_DEFAULT_ATTR_NEXT);
rt_hw_cpu_dcache_clean(mmu_l1, sizeof(*mmu_l1));
}
else
@@ -303,7 +94,8 @@ static int _mmu_map_one_page(rt_mmu_info *mmu_info, size_t va, size_t pa, size_t
if (PTE_USED(*(mmu_l2 + l2_off)))
{
RT_ASSERT(!PAGE_IS_LEAF(*(mmu_l2 + l2_off)));
- mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), mmu_info->pv_off);
+ mmu_l3 =
+ (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), PV_OFFSET);
}
else
{
@@ -313,7 +105,9 @@ static int _mmu_map_one_page(rt_mmu_info *mmu_info, size_t va, size_t pa, size_t
{
rt_memset(mmu_l3, 0, PAGE_SIZE);
rt_hw_cpu_dcache_clean(mmu_l3, PAGE_SIZE);
- *(mmu_l2 + l2_off) = COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l3, mmu_info->pv_off), PAGE_DEFAULT_ATTR_NEXT);
+ *(mmu_l2 + l2_off) =
+ COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l3, PV_OFFSET),
+ PAGE_DEFAULT_ATTR_NEXT);
rt_hw_cpu_dcache_clean(mmu_l2, sizeof(*mmu_l2));
// declares a reference to parent page table
rt_page_ref_inc((void *)mmu_l2, 0);
@@ -332,259 +126,242 @@ static int _mmu_map_one_page(rt_mmu_info *mmu_info, size_t va, size_t pa, size_t
return 0;
}
-static int __rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, rt_size_t npages, rt_size_t attr)
+/** rt_hw_mmu_map will never override existed page table entry */
+void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr,
+ size_t size, size_t attr)
{
- rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr, PAGE_OFFSET_MASK);
- rt_size_t loop_pa = __UMASKVALUE((rt_size_t)p_addr, PAGE_OFFSET_MASK);
-
- if (!mmu_info)
- {
- return -1;
- }
+ int ret = -1;
+ void *unmap_va = v_addr;
+ size_t npages = size >> ARCH_PAGE_SHIFT;
+ // TODO trying with HUGEPAGE here
while (npages--)
{
- if (_mmu_map_one_page(mmu_info, loop_va, loop_pa, attr) != 0)
+ ret = _map_one_page(aspace, v_addr, p_addr, attr);
+ if (ret != 0)
{
- __rt_hw_mmu_unmap(mmu_info, v_addr, npages);
- return -1;
+ /* error, undo map */
+ while (unmap_va != v_addr)
+ {
+ MM_PGTBL_LOCK(aspace);
+ _unmap_area(aspace, unmap_va, ARCH_PAGE_SIZE);
+ MM_PGTBL_UNLOCK(aspace);
+ unmap_va += ARCH_PAGE_SIZE;
+ }
+ break;
}
-
- loop_va += PAGE_SIZE;
- loop_pa += PAGE_SIZE;
+ v_addr += ARCH_PAGE_SIZE;
+ p_addr += ARCH_PAGE_SIZE;
}
- return 0;
-}
-
-void *_rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, rt_size_t size, rt_size_t attr)
-{
- rt_size_t pa_s, pa_e;
- rt_size_t vaddr;
- rt_size_t pages;
- int ret;
-
- if (!size)
+ if (ret == 0)
{
- return 0;
+ return unmap_va;
}
- pa_s = (rt_size_t)p_addr;
- pa_e = ((rt_size_t)p_addr) + size - 1;
- pa_s = GET_PF_ID(pa_s);
- pa_e = GET_PF_ID(pa_e);
- pages = pa_e - pa_s + 1;
+ return NULL;
+}
- if (v_addr)
+static void _unmap_pte(rt_size_t *pentry, rt_size_t *lvl_entry[], int level)
+{
+ int loop_flag = 1;
+ while (loop_flag)
{
- vaddr = (rt_size_t)v_addr;
- pa_s = (rt_size_t)p_addr;
+ loop_flag = 0;
+ *pentry = 0;
+ rt_hw_cpu_dcache_clean(pentry, sizeof(*pentry));
- if (GET_PF_OFFSET(vaddr) != GET_PF_OFFSET(pa_s))
+ // we don't handle level 0, which is maintained by caller
+ if (level > 0)
{
- return 0;
- }
+ void *page = (void *)((rt_ubase_t)pentry & ~ARCH_PAGE_MASK);
- vaddr = __UMASKVALUE(vaddr, PAGE_OFFSET_MASK);
+ // decrease reference from child page to parent
+ rt_pages_free(page, 0);
- if (check_vaddr(mmu_info, (void *)vaddr, pages) != 0)
- {
- return 0;
- }
- }
- else
- {
- vaddr = find_vaddr(mmu_info, pages);
- }
-
- if (vaddr)
- {
- ret = __rt_hw_mmu_map(mmu_info, (void *)vaddr, p_addr, pages, attr);
-
- if (ret == 0)
- {
- rt_hw_cpu_tlb_invalidate();
- return (void *)(vaddr | GET_PF_OFFSET((rt_size_t)p_addr));
+ int free = rt_page_ref_get(page, 0);
+ if (free == 1)
+ {
+ rt_pages_free(page, 0);
+ pentry = lvl_entry[--level];
+ loop_flag = 1;
+ }
}
}
-
- return 0;
}
-static int __rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, rt_size_t npages, rt_size_t attr)
+static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr, size_t size)
{
rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr, PAGE_OFFSET_MASK);
- rt_size_t loop_pa;
- rt_size_t l1_off, l2_off, l3_off;
- rt_size_t *mmu_l1, *mmu_l2, *mmu_l3;
- rt_size_t *ref_cnt;
- rt_size_t i;
- void *va, *pa;
+ size_t unmapped = 0;
- if (!mmu_info)
+ int i = 0;
+ rt_size_t lvl_off[3];
+ rt_size_t *lvl_entry[3];
+ lvl_off[0] = (rt_size_t)GET_L1(loop_va);
+ lvl_off[1] = (rt_size_t)GET_L2(loop_va);
+ lvl_off[2] = (rt_size_t)GET_L3(loop_va);
+ unmapped = 1 << (ARCH_PAGE_SHIFT + ARCH_INDEX_WIDTH * 2ul);
+
+ rt_size_t *pentry;
+ lvl_entry[i] = ((rt_size_t *)aspace->page_table + lvl_off[i]);
+ pentry = lvl_entry[i];
+
+ // find leaf page table entry
+ while (PTE_USED(*pentry) && !PAGE_IS_LEAF(*pentry))
{
- return -1;
+ i += 1;
+ lvl_entry[i] = ((rt_size_t *)PPN_TO_VPN(GET_PADDR(*pentry), PV_OFFSET) +
+ lvl_off[i]);
+ pentry = lvl_entry[i];
+ unmapped >>= ARCH_INDEX_WIDTH;
}
- while (npages--)
+ // clear PTE & setup its
+ if (PTE_USED(*pentry))
{
- loop_pa = (rt_size_t)rt_pages_alloc(0);
-
- if (!loop_pa)
- {
- goto err;
- }
-
- if (__rt_hw_mmu_map(mmu_info, (void *)loop_va, (void *)loop_pa, 1, attr) < 0)
- {
- goto err;
- }
-
- loop_va += PAGE_SIZE;
+ _unmap_pte(pentry, lvl_entry, i);
}
- return 0;
+ return unmapped;
+}
-err:
- va = (void *)__UMASKVALUE((rt_size_t)v_addr, PAGE_OFFSET_MASK);
+/** unmap is different from map that it can handle multiple pages */
+void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size)
+{
+ // caller guarantee that v_addr & size are page aligned
+ if (!aspace->page_table)
+ {
+ return;
+ }
+ size_t unmapped = 0;
- for (i = 0; i < npages; i++)
+ while (size > 0)
{
- pa = rt_hw_mmu_v2p(mmu_info, va);
+ MM_PGTBL_LOCK(aspace);
+ unmapped = _unmap_area(aspace, v_addr, size);
+ MM_PGTBL_UNLOCK(aspace);
- if (pa)
- {
- rt_pages_free((void *)PPN_TO_VPN(pa, mmu_info->pv_off), 0);
- }
+ // when unmapped == 0, region not exist in pgtbl
+ if (!unmapped || unmapped > size)
+ break;
- va = (void *)((rt_uint8_t *)va + PAGE_SIZE);
+ size -= unmapped;
+ v_addr += unmapped;
}
+}
- __rt_hw_mmu_unmap(mmu_info, v_addr, npages);
- return -1;
+#ifdef RT_USING_SMART
+static inline void _init_region(void *vaddr, size_t size)
+{
+ rt_ioremap_start = vaddr;
+ rt_ioremap_size = size;
+ rt_mpr_start = rt_ioremap_start - rt_mpr_size;
+}
+#else
+static inline void _init_region(void *vaddr, size_t size)
+{
+ rt_mpr_start = vaddr - rt_mpr_size;
}
+#endif
-void *_rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size, rt_size_t attr)
+int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_size_t size,
+ rt_size_t *vtable, rt_size_t pv_off)
{
- rt_size_t vaddr;
- rt_size_t offset;
- rt_size_t pages;
- int ret;
+ size_t l1_off, va_s, va_e;
+ rt_base_t level;
- if (!size)
+ if ((!aspace) || (!vtable))
{
- return 0;
+ return -1;
}
- offset = GET_PF_OFFSET((rt_size_t)v_addr);
- size += (offset + PAGE_SIZE - 1);
- pages = size >> PAGE_OFFSET_BIT;
+ va_s = (rt_size_t)v_address;
+ va_e = ((rt_size_t)v_address) + size - 1;
- if (v_addr)
+ if (va_e < va_s)
{
- vaddr = __UMASKVALUE((rt_size_t)v_addr, PAGE_OFFSET_MASK);
-
- if (check_vaddr(mmu_info, (void *)vaddr, pages) != 0)
- {
- return 0;
- }
+ return -1;
}
- else
+
+ // convert address to PPN2 index
+ va_s = GET_L1(va_s);
+ va_e = GET_L1(va_e);
+
+ if (va_s == 0)
{
- vaddr = find_vaddr(mmu_info, pages);
+ return -1;
}
- if (vaddr)
+ // vtable initialization check
+ for (l1_off = va_s; l1_off <= va_e; l1_off++)
{
- ret = __rt_hw_mmu_map_auto(mmu_info, (void *)vaddr, pages, attr);
+ size_t v = vtable[l1_off];
- if (ret == 0)
+ if (v)
{
- rt_hw_cpu_tlb_invalidate();
- return (void *)(vaddr | offset);
+ return -1;
}
}
- return 0;
-}
-
-void _rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size)
-{
- rt_size_t va_s, va_e;
- rt_size_t pages;
-
- va_s = ((rt_size_t)v_addr) >> PAGE_OFFSET_BIT;
- va_e = (((rt_size_t)v_addr) + size - 1) >> PAGE_OFFSET_BIT;
- pages = va_e - va_s + 1;
- __rt_hw_mmu_unmap(mmu_info, v_addr, pages);
- rt_hw_cpu_tlb_invalidate();
-}
-
-void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, rt_size_t size, rt_size_t attr)
-{
- void *ret;
- rt_base_t level;
+ rt_aspace_init(&rt_kernel_space, (void *)0x1000, USER_VADDR_START - 0x1000,
+ vtable);
- rt_mm_lock();
- ret = _rt_hw_mmu_map(mmu_info, v_addr, p_addr, size, attr);
- rt_mm_unlock();
- return ret;
+ _init_region(v_address, size);
+ return 0;
}
-void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size, rt_size_t attr)
-{
- void *ret;
- rt_base_t level;
-
- rt_mm_lock();
- ret = _rt_hw_mmu_map_auto(mmu_info, v_addr, size, attr);
- rt_mm_unlock();
- return ret;
-}
+const static int max_level =
+ (ARCH_VADDR_WIDTH - ARCH_PAGE_SHIFT) / ARCH_INDEX_WIDTH;
-void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size)
+static inline uintptr_t _get_level_size(int level)
{
- rt_base_t level;
-
- rt_mm_lock();
- _rt_hw_mmu_unmap(mmu_info, v_addr, size);
- rt_mm_unlock();
+ return 1ul << (ARCH_PAGE_SHIFT + (max_level - level) * ARCH_INDEX_WIDTH);
}
-void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr)
+static rt_size_t *_query(struct rt_aspace *aspace, void *vaddr, int *level)
{
rt_size_t l1_off, l2_off, l3_off;
rt_size_t *mmu_l1, *mmu_l2, *mmu_l3;
rt_size_t pa;
- l1_off = GET_L1((rt_size_t)v_addr);
- l2_off = GET_L2((rt_size_t)v_addr);
- l3_off = GET_L3((rt_size_t)v_addr);
+ l1_off = GET_L1((rt_size_t)vaddr);
+ l2_off = GET_L2((rt_size_t)vaddr);
+ l3_off = GET_L3((rt_size_t)vaddr);
- if (!mmu_info)
+ if (!aspace)
{
+ LOG_W("%s: no aspace", __func__);
return RT_NULL;
}
- mmu_l1 = ((rt_size_t *)mmu_info->vtable) + l1_off;
+ mmu_l1 = ((rt_size_t *)aspace->page_table) + l1_off;
if (PTE_USED(*mmu_l1))
{
if (*mmu_l1 & PTE_XWR_MASK)
- return (void *)(GET_PADDR(*mmu_l1) | ((rt_size_t)v_addr & ((1 << 30) - 1)));
+ {
+ *level = 1;
+ return mmu_l1;
+ }
- mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), mmu_info->pv_off);
+ mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET);
if (PTE_USED(*(mmu_l2 + l2_off)))
{
if (*(mmu_l2 + l2_off) & PTE_XWR_MASK)
- return (void *)(GET_PADDR(*(mmu_l2 + l2_off)) | ((rt_size_t)v_addr & ((1 << 21) - 1)));
+ {
+ *level = 2;
+ return mmu_l2 + l2_off;
+ }
- mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), mmu_info->pv_off);
+ mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)),
+ PV_OFFSET);
if (PTE_USED(*(mmu_l3 + l3_off)))
{
- return (void *)(GET_PADDR(*(mmu_l3 + l3_off)) | GET_PF_OFFSET((rt_size_t)v_addr));
+ *level = 3;
+ return mmu_l3 + l3_off;
}
}
}
@@ -592,15 +369,71 @@ void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr)
return RT_NULL;
}
-void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr)
+void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr)
{
- void *ret;
- rt_base_t level;
+ int level;
+ uintptr_t *pte = _query(aspace, vaddr, &level);
+ uintptr_t paddr;
+
+ if (pte)
+ {
+ paddr = GET_PADDR(*pte);
+ paddr |= ((intptr_t)vaddr & (_get_level_size(level) - 1));
+ }
+ else
+ {
+ paddr = (uintptr_t)ARCH_MAP_FAILED;
+ }
+ return (void *)paddr;
+}
+
+static int _noncache(uintptr_t *pte)
+{
+ return 0;
+}
+
+static int _cache(uintptr_t *pte)
+{
+ return 0;
+}
+
+static int (*control_handler[MMU_CNTL_DUMMY_END])(uintptr_t *pte) = {
+ [MMU_CNTL_CACHE] = _cache,
+ [MMU_CNTL_NONCACHE] = _noncache,
+};
+
+int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
+ enum rt_mmu_cntl cmd)
+{
+ int level;
+ int err = -RT_EINVAL;
+ void *vend = vaddr + size;
+
+ int (*handler)(uintptr_t * pte);
+ if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END)
+ {
+ handler = control_handler[cmd];
+
+ while (vaddr < vend)
+ {
+ uintptr_t *pte = _query(aspace, vaddr, &level);
+ void *range_end = vaddr + _get_level_size(level);
+ RT_ASSERT(range_end <= vend);
+
+ if (pte)
+ {
+ err = handler(pte);
+ RT_ASSERT(err == RT_EOK);
+ }
+ vaddr = range_end;
+ }
+ }
+ else
+ {
+ err = -RT_ENOSYS;
+ }
- rt_mm_lock();
- ret = _rt_hw_mmu_v2p(mmu_info, v_addr);
- rt_mm_unlock();
- return ret;
+ return err;
}
/**
@@ -615,7 +448,7 @@ void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr)
* @param mdesc
* @param desc_nr
*/
-void rt_hw_mmu_setup(rt_mmu_info *mmu_info, struct mem_desc *mdesc, int desc_nr)
+void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr)
{
void *err;
for (size_t i = 0; i < desc_nr; i++)
@@ -623,24 +456,31 @@ void rt_hw_mmu_setup(rt_mmu_info *mmu_info, struct mem_desc *mdesc, int desc_nr)
size_t attr;
switch (mdesc->attr)
{
- case NORMAL_MEM:
- attr = MMU_MAP_K_RWCB;
- break;
- case NORMAL_NOCACHE_MEM:
- attr = MMU_MAP_K_RWCB;
- break;
- case DEVICE_MEM:
- attr = MMU_MAP_K_DEVICE;
- break;
- default:
- attr = MMU_MAP_K_DEVICE;
+ case NORMAL_MEM:
+ attr = MMU_MAP_K_RWCB;
+ break;
+ case NORMAL_NOCACHE_MEM:
+ attr = MMU_MAP_K_RWCB;
+ break;
+ case DEVICE_MEM:
+ attr = MMU_MAP_K_DEVICE;
+ break;
+ default:
+ attr = MMU_MAP_K_DEVICE;
}
- err = _rt_hw_mmu_map(mmu_info, (void *)mdesc->vaddr_start, (void *)mdesc->paddr_start,
- mdesc->vaddr_end - mdesc->vaddr_start + 1, attr);
+
+ struct rt_mm_va_hint hint = {.flags = MMF_MAP_FIXED,
+ .limit_start = aspace->start,
+ .limit_range_size = aspace->size,
+ .map_size = mdesc->vaddr_end -
+ mdesc->vaddr_start + 1,
+ .prefer = (void *)mdesc->vaddr_start};
+
+ rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr,
+ mdesc->paddr_start >> MM_PAGE_SHIFT, &err);
mdesc++;
}
- rt_hw_mmu_switch((void *)MMUTable);
+ rt_hw_aspace_switch(&rt_kernel_space);
+ rt_page_cleanup();
}
-
-#endif /* RT_USING_SMART */
diff --git a/libcpu/risc-v/virt64/mmu.h b/libcpu/risc-v/virt64/mmu.h
index 086ad1edb9..ece5c1ccac 100644
--- a/libcpu/risc-v/virt64/mmu.h
+++ b/libcpu/risc-v/virt64/mmu.h
@@ -13,13 +13,17 @@
#include "riscv.h"
#include "riscv_mmu.h"
+#include <mm_aspace.h>
+#include <stddef.h>
/* RAM, Flash, or ROM */
-#define NORMAL_MEM 0
+#define NORMAL_MEM 0
/* normal nocache memory mapping type */
-#define NORMAL_NOCACHE_MEM 1
+#define NORMAL_NOCACHE_MEM 1
/* MMIO region */
-#define DEVICE_MEM 2
+#define DEVICE_MEM 2
+
+typedef size_t rt_pte_t;
struct mem_desc
{
@@ -27,48 +31,58 @@ struct mem_desc
rt_size_t vaddr_end;
rt_size_t paddr_start;
rt_size_t attr;
+ struct rt_varea varea;
};
-#define GET_PF_ID(addr) ((addr) >> PAGE_OFFSET_BIT)
+#define GET_PF_ID(addr) ((addr) >> PAGE_OFFSET_BIT)
#define GET_PF_OFFSET(addr) __MASKVALUE(addr, PAGE_OFFSET_MASK)
-#define GET_L1(addr) __PARTBIT(addr, VPN2_SHIFT, VPN2_BIT)
-#define GET_L2(addr) __PARTBIT(addr, VPN1_SHIFT, VPN1_BIT)
-#define GET_L3(addr) __PARTBIT(addr, VPN0_SHIFT, VPN0_BIT)
-#define GET_PPN(pte) (__PARTBIT(pte, PTE_PPN_SHIFT, PHYSICAL_ADDRESS_WIDTH_BITS - PTE_PPN_SHIFT))
-#define GET_PADDR(pte) (GET_PPN(pte) << PAGE_OFFSET_BIT)
+#define GET_L1(addr) __PARTBIT(addr, VPN2_SHIFT, VPN2_BIT)
+#define GET_L2(addr) __PARTBIT(addr, VPN1_SHIFT, VPN1_BIT)
+#define GET_L3(addr) __PARTBIT(addr, VPN0_SHIFT, VPN0_BIT)
+#define GET_PPN(pte) \
+ (__PARTBIT(pte, PTE_PPN_SHIFT, PHYSICAL_ADDRESS_WIDTH_BITS - PTE_PPN_SHIFT))
+#define GET_PADDR(pte) (GET_PPN(pte) << PAGE_OFFSET_BIT)
#define VPN_TO_PPN(vaddr, pv_off) (((rt_size_t)(vaddr)) + (pv_off))
#define PPN_TO_VPN(paddr, pv_off) (((rt_size_t)(paddr)) - (pv_off))
-#define COMBINEVADDR(l1_off, l2_off, l3_off) (((l1_off) << VPN2_SHIFT) | ((l2_off) << VPN1_SHIFT) | ((l3_off) << VPN0_SHIFT))
-#define COMBINEPTE(paddr, attr) ((((paddr) >> PAGE_OFFSET_BIT) << PTE_PPN_SHIFT) | (attr))
+#define COMBINEVADDR(l1_off, l2_off, l3_off) \
+ (((l1_off) << VPN2_SHIFT) | ((l2_off) << VPN1_SHIFT) | \
+ ((l3_off) << VPN0_SHIFT))
+#define COMBINEPTE(paddr, attr) \
+ ((((paddr) >> PAGE_OFFSET_BIT) << PTE_PPN_SHIFT) | (attr))
-typedef struct
-{
- size_t *vtable;
- size_t vstart;
- size_t vend;
- size_t pv_off;
-} rt_mmu_info;
+#define ARCH_ADDRESS_WIDTH_BITS 64
+
+#define MMU_MAP_ERROR_VANOTALIGN -1
+#define MMU_MAP_ERROR_PANOTALIGN -2
+#define MMU_MAP_ERROR_NOPAGE -3
+#define MMU_MAP_ERROR_CONFLICT -4
void *rt_hw_mmu_tbl_get();
-void rt_hw_mmu_switch(void *mmu_table);
-int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void *v_address, rt_size_t size, rt_size_t *vtable, rt_size_t pv_off);
-void rt_hw_mmu_kernel_map_init(rt_mmu_info *mmu_info, rt_size_t vaddr_start, rt_size_t size);
-void *_rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, rt_size_t size, rt_size_t attr);
-void *_rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size, rt_size_t attr);
-void _rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size);
-void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, rt_size_t size, rt_size_t attr);
-void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size, rt_size_t attr);
-void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, rt_size_t size);
-void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr);
-void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr);
+int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_size_t size,
+ rt_size_t *vtable, rt_size_t pv_off);
+void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr);
+void rt_hw_mmu_kernel_map_init(rt_aspace_t aspace, rt_size_t vaddr_start,
+ rt_size_t size);
+void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
+ size_t attr);
+void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size);
+void rt_hw_aspace_switch(rt_aspace_t aspace);
+void *rt_hw_mmu_v2p(rt_aspace_t aspace, void *vaddr);
-void rt_hw_mmu_setup(rt_mmu_info *mmu_info, struct mem_desc *mdesc, int desc_nr);
+static inline void *_rt_kmem_v2p(void *vaddr)
+{
+ return rt_hw_mmu_v2p(&rt_kernel_space, vaddr);
+}
-#define ARCH_ADDRESS_WIDTH_BITS 64
+static inline void *rt_kmem_v2p(void *vaddr)
+{
+ MM_PGTBL_LOCK(&rt_kernel_space);
+ void *paddr = _rt_kmem_v2p(vaddr);
+ MM_PGTBL_UNLOCK(&rt_kernel_space);
+ return paddr;
+}
-#define MMU_MAP_ERROR_VANOTALIGN -1
-#define MMU_MAP_ERROR_PANOTALIGN -2
-#define MMU_MAP_ERROR_NOPAGE -3
-#define MMU_MAP_ERROR_CONFLICT -4
+int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
+ enum rt_mmu_cntl cmd);
#endif
diff --git a/libcpu/risc-v/virt64/riscv_mmu.c b/libcpu/risc-v/virt64/riscv_mmu.c
index e79ca8b37b..7c4f886345 100644
--- a/libcpu/risc-v/virt64/riscv_mmu.c
+++ b/libcpu/risc-v/virt64/riscv_mmu.c
@@ -11,23 +11,13 @@
#include <rthw.h>
#include <rtthread.h>
-#include <stdint.h>
#include <riscv.h>
-#include <string.h>
+#include <stdint.h>
#include <stdlib.h>
+#include <string.h>
#include "riscv_mmu.h"
-void mmu_set_pagetable(rt_ubase_t addr)
-{
- RT_ASSERT(__CHECKALIGN(addr, PAGE_OFFSET_BIT));
- RT_ASSERT(__CHECKUPBOUND(addr, PHYSICAL_ADDRESS_WIDTH_BITS));
-
- mmu_flush_tlb();
- write_csr(satp, (((size_t)SATP_MODE) << SATP_MODE_OFFSET) | (addr >> PAGE_OFFSET_BIT));
- mmu_flush_tlb();
-}
-
void mmu_enable_user_page_access()
{
set_csr(sstatus, SSTATUS_SUM);
diff --git a/libcpu/risc-v/virt64/riscv_mmu.h b/libcpu/risc-v/virt64/riscv_mmu.h
index 04c4201aab..fc5a143b65 100644
--- a/libcpu/risc-v/virt64/riscv_mmu.h
+++ b/libcpu/risc-v/virt64/riscv_mmu.h
@@ -58,7 +58,7 @@
#define PTE_USED(pte) __MASKVALUE(pte, PTE_V)
-/**
+/**
* encoding of SATP (Supervisor Address Translation and Protection register)
*/
#define SATP_MODE_OFFSET 60
@@ -68,16 +68,9 @@
#define SATP_MODE_SV57 10
#define SATP_MODE_SV64 11
-#define ARCH_VA_WIDTH 39
+#define ARCH_VADDR_WIDTH 39
#define SATP_MODE SATP_MODE_SV39
-#define mmu_flush_tlb() \
- do \
- { \
- asm volatile("sfence.vma x0,x0"); \
- } while (0)
-
-
#define MMU_MAP_K_DEVICE (PTE_G | PTE_W | PTE_R | PTE_V)
#define MMU_MAP_K_RWCB (PTE_G | PTE_X | PTE_W | PTE_R | PTE_V)
#define MMU_MAP_U_RWCB (PTE_U | PTE_X | PTE_W | PTE_R | PTE_V)
@@ -93,6 +86,8 @@
#define ARCH_INDEX_SIZE (1ul << ARCH_INDEX_WIDTH)
#define ARCH_INDEX_MASK (ARCH_INDEX_SIZE - 1)
+#define ARCH_MAP_FAILED ((void *)0x8000000000000000)
+
void mmu_set_pagetable(rt_ubase_t addr);
void mmu_enable_user_page_access();
void mmu_disable_user_page_access();
diff --git a/libcpu/risc-v/virt64/sbi.c b/libcpu/risc-v/virt64/sbi.c
index 93084bd441..b2b1e8e788 100644
--- a/libcpu/risc-v/virt64/sbi.c
+++ b/libcpu/risc-v/virt64/sbi.c
@@ -36,8 +36,8 @@
*/
#include "sbi.h"
-#include <stdbool.h>
#include <rtthread.h>
+#include <stdbool.h>
/* SBI Implementation-Specific Definitions */
#define OPENSBI_VERSION_MAJOR_OFFSET 16
@@ -51,20 +51,17 @@ static bool has_time_extension = false;
static bool has_ipi_extension = false;
static bool has_rfnc_extension = false;
-static struct sbi_ret
-sbi_get_spec_version(void)
+static struct sbi_ret sbi_get_spec_version(void)
{
return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_SPEC_VERSION));
}
-static struct sbi_ret
-sbi_get_impl_id(void)
+static struct sbi_ret sbi_get_impl_id(void)
{
return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_IMPL_ID));
}
-static struct sbi_ret
-sbi_get_impl_version(void)
+static struct sbi_ret sbi_get_impl_version(void)
{
return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_IMPL_VERSION));
}
@@ -88,7 +85,8 @@ void sbi_print_version(void)
rt_kprintf("SBI: Berkely Boot Loader %lu\n", sbi_impl_version);
break;
case (SBI_IMPL_ID_XVISOR):
- rt_kprintf("SBI: eXtensible Versatile hypervISOR %lu\n", sbi_impl_version);
+ rt_kprintf("SBI: eXtensible Versatile hypervISOR %lu\n",
+ sbi_impl_version);
break;
case (SBI_IMPL_ID_KVM):
rt_kprintf("SBI: Kernel-based Virtual Machine %lu\n", sbi_impl_version);
@@ -138,8 +136,7 @@ void sbi_send_ipi(const unsigned long *hart_mask)
/* Use the IPI legacy replacement extension, if available. */
if (has_ipi_extension)
{
- ret = SBI_CALL2(SBI_EXT_ID_IPI, SBI_IPI_SEND_IPI,
- *hart_mask, 0);
+ ret = SBI_CALL2(SBI_EXT_ID_IPI, SBI_IPI_SEND_IPI, *hart_mask, 0);
RT_ASSERT(ret.error == SBI_SUCCESS);
}
else
@@ -155,8 +152,8 @@ void sbi_remote_fence_i(const unsigned long *hart_mask)
/* Use the RFENCE legacy replacement extension, if available. */
if (has_rfnc_extension)
{
- ret = SBI_CALL2(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_FENCE_I,
- *hart_mask, 0);
+ ret =
+ SBI_CALL2(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_FENCE_I, *hart_mask, 0);
RT_ASSERT(ret.error == SBI_SUCCESS);
}
else
@@ -165,25 +162,28 @@ void sbi_remote_fence_i(const unsigned long *hart_mask)
}
}
-void sbi_remote_sfence_vma(const unsigned long *hart_mask, unsigned long start, unsigned long size)
+int sbi_remote_sfence_vma(const unsigned long *hart_mask,
+ const unsigned long hart_mask_base,
+ unsigned long start, unsigned long size)
{
- struct sbi_ret ret;
+ struct sbi_ret ret = {.error = SBI_SUCCESS};
/* Use the RFENCE legacy replacement extension, if available. */
if (has_rfnc_extension)
{
- ret = SBI_CALL4(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_SFENCE_VMA,
- *hart_mask, 0, start, size);
- RT_ASSERT(ret.error == SBI_SUCCESS);
+ ret = SBI_CALL4(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_SFENCE_VMA, *hart_mask,
+ hart_mask_base, start, size);
}
else
{
- (void)SBI_CALL3(SBI_REMOTE_SFENCE_VMA, 0, (uint64_t)hart_mask,
- start, size);
+ (void)SBI_CALL3(SBI_REMOTE_SFENCE_VMA, 0, (uint64_t)hart_mask, start,
+ size);
}
+ return ret.error;
}
-void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start, unsigned long size,
+void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
+ unsigned long start, unsigned long size,
unsigned long asid)
{
struct sbi_ret ret;
@@ -197,12 +197,13 @@ void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long st
}
else
{
- (void)SBI_CALL4(SBI_REMOTE_SFENCE_VMA_ASID, 0,
- (uint64_t)hart_mask, start, size, asid);
+ (void)SBI_CALL4(SBI_REMOTE_SFENCE_VMA_ASID, 0, (uint64_t)hart_mask,
+ start, size, asid);
}
}
-int sbi_hsm_hart_start(unsigned long hart, unsigned long start_addr, unsigned long priv)
+int sbi_hsm_hart_start(unsigned long hart, unsigned long start_addr,
+ unsigned long priv)
{
struct sbi_ret ret;
diff --git a/libcpu/risc-v/virt64/sbi.h b/libcpu/risc-v/virt64/sbi.h
index a5ab83d27d..be54dede72 100644
--- a/libcpu/risc-v/virt64/sbi.h
+++ b/libcpu/risc-v/virt64/sbi.h
@@ -185,7 +185,9 @@ void sbi_send_ipi(const unsigned long *hart_mask);
/* RFENCE extension functions. */
void sbi_remote_fence_i(const unsigned long *hart_mask);
-void sbi_remote_sfence_vma(const unsigned long *hart_mask, unsigned long start, unsigned long size);
+int sbi_remote_sfence_vma(const unsigned long *hart_mask,
+ const unsigned long hart_mask_base,
+ unsigned long start, unsigned long size);
void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start,
unsigned long size, unsigned long asid);
diff --git a/libcpu/risc-v/virt64/start.c b/libcpu/risc-v/virt64/start.c
new file mode 100644
index 0000000000..a8c374f698
--- /dev/null
+++ b/libcpu/risc-v/virt64/start.c
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2006-2022, RT-Thread Development Team
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Change Logs:
+ * Date Author Notes
+ * 2022-12-07 RT-Thread the first version
+ */
+
+#include <rthw.h>
+#include <rtthread.h>
+#include "board.h"
+
+void init_bss(void)
+{
+ unsigned int *dst;
+
+ dst = &__bss_start;
+ while (dst < &__bss_end)
+ {
+ *dst++ = 0;
+ }
+}
diff --git a/libcpu/risc-v/virt64/startup_gcc.S b/libcpu/risc-v/virt64/startup_gcc.S
index 700f31181d..c9ce4bcf1e 100644
--- a/libcpu/risc-v/virt64/startup_gcc.S
+++ b/libcpu/risc-v/virt64/startup_gcc.S
@@ -10,7 +10,7 @@
* 2020/6/12 Xim Port to QEMU and remove SMP support
*/
-#define SSTATUS_FS 0x00006000U /* initial state of FPU, clear to disable */
+#include <encoding.h>
#include <cpuport.h>
boot_hartid: .int
@@ -72,7 +72,7 @@ _start:
li x31,0
/* set to disable FPU */
- li t0, SSTATUS_FS
+ li t0, SSTATUS_FS + SSTATUS_VS
csrc sstatus, t0
li t0, 0x40000 // SUM in sstatus
csrs sstatus, t0
@@ -91,4 +91,6 @@ _start:
* sscratch is always zero on kernel mode
*/
csrw sscratch, zero
+ call init_bss
+ call sbi_init
j primary_cpu_entry
diff --git a/libcpu/risc-v/virt64/syscall_c.c b/libcpu/risc-v/virt64/syscall_c.c
index 9a946d98b9..8bda3e6278 100644
--- a/libcpu/risc-v/virt64/syscall_c.c
+++ b/libcpu/risc-v/virt64/syscall_c.c
@@ -21,7 +21,6 @@
#include <stdint.h>
#include <mmu.h>
#include <page.h>
-#include <lwp_mm_area.h>
#include <lwp_user_mm.h>
#include "riscv_mmu.h"
diff --git a/libcpu/risc-v/virt64/tlb.h b/libcpu/risc-v/virt64/tlb.h
new file mode 100644
index 0000000000..5bdf18c189
--- /dev/null
+++ b/libcpu/risc-v/virt64/tlb.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2006-2022, RT-Thread Development Team
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Change Logs:
+ * Date Author Notes
+ * 2022-11-28 WangXiaoyao the first version
+ */
+#ifndef __TLB_H__
+#define __TLB_H__
+
+#include "mm_aspace.h"
+#include "riscv_mmu.h"
+#include "rtdbg.h"
+#include "rtthread.h"
+#include <sbi.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define HANDLE_FAULT(ret) \
+ if (__builtin_expect((ret) != SBI_SUCCESS, 0)) \
+ LOG_W("%s failed", __FUNCTION__);
+
+static inline void rt_hw_tlb_invalidate_all(void)
+{
+ uintptr_t mask = -1ul;
+ HANDLE_FAULT(sbi_remote_sfence_vma(&mask, -1ul, 0, mask));
+}
+
+static inline void rt_hw_tlb_invalidate_all_local(void)
+{
+ __asm__ volatile("sfence.vma" ::: "memory");
+}
+
+static inline void rt_hw_tlb_invalidate_aspace(rt_aspace_t aspace)
+{
+ // TODO ASID
+ rt_hw_tlb_invalidate_all_local();
+}
+
+static inline void rt_hw_tlb_invalidate_page(rt_aspace_t aspace, void *start)
+{
+ __asm__ volatile("sfence.vma %0, zero" ::"r"(start) : "memory");
+}
+
+static inline void rt_hw_tlb_invalidate_range(rt_aspace_t aspace, void *start,
+ size_t size, size_t stride)
+{
+ // huge page is taking as normal page
+ if (size <= ARCH_PAGE_SIZE)
+ {
+ rt_hw_tlb_invalidate_page(aspace, start);
+ }
+ else
+ {
+ rt_hw_tlb_invalidate_aspace(aspace);
+ }
+}
+
+#endif /* __TLB_H__ */
diff --git a/libcpu/risc-v/virt64/trap.c b/libcpu/risc-v/virt64/trap.c
index 0a5c082f30..07d9894f4a 100644
--- a/libcpu/risc-v/virt64/trap.c
+++ b/libcpu/risc-v/virt64/trap.c
@@ -12,6 +12,7 @@
#include <stdint.h>
#include "encoding.h"
+#include "mm_fault.h"
#include "stack.h"
#include "sbi.h"
#include "riscv.h"
@@ -139,7 +140,17 @@ extern struct rt_irq_desc irq_desc[];
#include "rtdbg.h"
#include "encoding.h"
-void sys_exit(int value);
+
+#ifndef RT_USING_SMP
+static volatile int nested = 0;
+#define ENTER_TRAP \
+ nested += 1
+#define EXIT_TRAP \
+ nested -= 1
+#define CHECK_NESTED_PANIC(cause, tval, epc, eframe) \
+ if (nested != 1) \
+ handle_nested_trap_panic(cause, tval, epc, eframe)
+#endif /* RT_USING_SMP */
static const char *get_exception_msg(int id)
{
@@ -161,10 +172,59 @@ void handle_user(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw
rt_size_t id = __MASKVALUE(scause, __MASK(63UL));
/* user page fault */
- if (id == EP_LOAD_PAGE_FAULT ||
- id == EP_STORE_PAGE_FAULT)
+ enum rt_mm_fault_op fault_op;
+ enum rt_mm_fault_type fault_type;
+ switch (id)
+ {
+ case EP_LOAD_PAGE_FAULT:
+ fault_op = MM_FAULT_OP_READ;
+ fault_type = MM_FAULT_TYPE_PAGE_FAULT;
+ break;
+ case EP_LOAD_ACCESS_FAULT:
+ fault_op = MM_FAULT_OP_READ;
+ fault_type = MM_FAULT_TYPE_ACCESS_FAULT;
+ break;
+ case EP_LOAD_ADDRESS_MISALIGNED:
+ fault_op = MM_FAULT_OP_READ;
+ fault_type = MM_FAULT_TYPE_BUS_ERROR;
+ break;
+ case EP_STORE_PAGE_FAULT:
+ fault_op = MM_FAULT_OP_WRITE;
+ fault_type = MM_FAULT_TYPE_PAGE_FAULT;
+ break;
+ case EP_STORE_ACCESS_FAULT:
+ fault_op = MM_FAULT_OP_WRITE;
+ fault_type = MM_FAULT_TYPE_ACCESS_FAULT;
+ break;
+ case EP_STORE_ADDRESS_MISALIGNED:
+ fault_op = MM_FAULT_OP_WRITE;
+ fault_type = MM_FAULT_TYPE_BUS_ERROR;
+ break;
+ case EP_INSTRUCTION_PAGE_FAULT:
+ fault_op = MM_FAULT_OP_EXECUTE;
+ fault_type = MM_FAULT_TYPE_PAGE_FAULT;
+ break;
+ case EP_INSTRUCTION_ACCESS_FAULT:
+ fault_op = MM_FAULT_OP_EXECUTE;
+ fault_type = MM_FAULT_TYPE_ACCESS_FAULT;
+ break;
+ case EP_INSTRUCTION_ADDRESS_MISALIGNED:
+ fault_op = MM_FAULT_OP_EXECUTE;
+ fault_type = MM_FAULT_TYPE_BUS_ERROR;
+ break;
+ default:
+ fault_op = 0;
+ }
+
+ if (fault_op)
{
- if (arch_expand_user_stack((void *)stval))
+ struct rt_mm_fault_msg msg = {
+ .fault_op = fault_op,
+ .fault_type = fault_type,
+ .vaddr = (void *)stval,
+ };
+
+ if (rt_mm_fault_try_fix(&msg))
{
return;
}
@@ -176,6 +236,8 @@ void handle_user(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw
rt_hw_backtrace((uint32_t *)sp->s0_fp, sepc);
LOG_E("User Fault, killing thread: %s", rt_thread_self()->name);
+
+ EXIT_TRAP;
sys_exit(-1);
}
#endif
@@ -228,17 +290,6 @@ static void handle_nested_trap_panic(
rt_hw_cpu_shutdown();
}
-#ifndef RT_USING_SMP
-static volatile int nested = 0;
-#define ENTER_TRAP \
- nested += 1
-#define EXIT_TRAP \
- nested -= 1
-#define CHECK_NESTED_PANIC(cause, tval, epc, eframe) \
- if (nested != 1) \
- handle_nested_trap_panic(cause, tval, epc, eframe)
-#endif /* RT_USING_SMP */
-
/* Trap entry */
void handle_trap(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw_stack_frame *sp)
{