diff options
Diffstat (limited to 'libcpu')
112 files changed, 13930 insertions, 488 deletions
diff --git a/libcpu/Kconfig b/libcpu/Kconfig index 1185c49521..7f3dfca11f 100644 --- a/libcpu/Kconfig +++ b/libcpu/Kconfig @@ -288,6 +288,10 @@ config ARCH_RISCV64 select ARCH_CPU_64BIT bool +if ARCH_RISCV + orsource "./risc-v/Kconfig" +endif + if ARCH_RISCV64 config ARCH_USING_NEW_CTX_SWITCH bool diff --git a/libcpu/risc-v/.gitignore b/libcpu/risc-v/.gitignore new file mode 100644 index 0000000000..feb550d190 --- /dev/null +++ b/libcpu/risc-v/.gitignore @@ -0,0 +1 @@ +link.lds diff --git a/libcpu/risc-v/Kconfig b/libcpu/risc-v/Kconfig new file mode 100644 index 0000000000..d148210c0d --- /dev/null +++ b/libcpu/risc-v/Kconfig @@ -0,0 +1,83 @@ +menu "RISC-V Architecture Configuration" + +config RT_USING_RISCV_NEW_COMMON + bool + select RT_USING_CPU_FFS + select RT_USING_HW_ATOMIC if RT_USING_SMP + select RT_PAGE_MPR_SIZE_DYNAMIC if ARCH_MM_MMU + help + Select the unified RISC-V architecture implementation. BSPs that do + not select this option continue to use the legacy implementation while + the architecture migration is in progress. + +if RT_USING_RISCV_NEW_COMMON + +config ARCH_HAVE_EFFICIENT_UNALIGNED_ACCESS + bool + default n + +config ARCH_RISCV_DISABLE_FENCE_I + bool + default n + +config ARCH_RISCV_DISABLE_SFENCE_VMA_ASID + bool + default n + +config ARCH_RISCV_M_MODE + bool + depends on !ARCH_MM_MMU + default n + +config ARCH_RISCV_32E + bool + depends on ARCH_RISCV32 + depends on !ARCH_MM_MMU + depends on !RT_USING_DM + +config ARCH_RISCV_USING_IRQ_STACK + bool "Use a dedicated interrupt stack" + depends on !RT_USING_SMART + default y if ARCH_RISCV_M_MODE || !RT_USING_DM + help + Keep the saved exception frame on the interrupted thread stack, then + run C-level trap and interrupt handlers on a per-CPU interrupt stack. + This is recommended for MCU ports. + +config ARCH_RISCV_IRQ_STACK_SIZE + int "Interrupt stack size per CPU" + depends on ARCH_RISCV_USING_IRQ_STACK + range 256 65536 + default 1024 + +config RT_USING_USERSPACE_32BIT_LIMIT + bool "Enable userspace 32bit limit" + depends on ARCH_RISCV64 + default n + + config ARCH_TEXT_OFFSET + hex "Kernel image offset from RAM base" + default 0x200000 if ARCH_RISCV64 + default 0x400000 + +config ARCH_RAM_OFFSET + hex "RAM base address" + default 0 + +config ARCH_SECONDARY_CPU_STACK_SIZE + int "Secondary CPU stack size" + depends on RT_USING_SMP + default 4096 + +config ARCH_HEAP_SIZE + hex "Size of system heap" + depends on RT_USING_DM + +config ARCH_INIT_PAGE_SIZE + hex "Size of init page region" + depends on RT_USING_DM && ARCH_MM_MMU + default 0x200000 + +endif + +endmenu diff --git a/libcpu/risc-v/SConscript b/libcpu/risc-v/SConscript index fafb8524ec..eb259f7125 100644 --- a/libcpu/risc-v/SConscript +++ b/libcpu/risc-v/SConscript @@ -1,27 +1,68 @@ -# RT-Thread building script for bridge +# RT-Thread building script for RISC-V import os from building import * Import('rtconfig') -common64_arch = ['virt64', 'c906', 'c908','ur-cp100'] -cwd = GetCurrentDir() +cwd = GetCurrentDir() group = [] -list = os.listdir(cwd) +cpu = getattr(rtconfig, 'CPU', '') +vendor = getattr(rtconfig, 'VENDOR', '') + +if GetDepend('RT_USING_RISCV_NEW_COMMON'): + bsp_path = Dir('#').abspath + + if not os.path.exists(os.path.join(bsp_path, 'link.lds')): + Env['LINKFLAGS'] = Env['LINKFLAGS'].replace( + 'link.lds', os.path.join(cwd, 'link.lds')) + Preprocessing('link.lds.S', '.lds', + CPPPATH=[bsp_path, os.path.join(cwd, 'common', 'include')]) + + # The new common owns the complete C runtime entry. + Env['LINKFLAGS'] += ' -nostartfiles' -# add common code files -if rtconfig.CPU in common64_arch : - group += SConscript(os.path.join('common64', 'SConscript')) -else : group += SConscript(os.path.join('common', 'SConscript')) -group += SConscript(os.path.join('vector', 'SConscript')) + # New vendor ports are architecture-wide overrides, not per-CPU copies. + if vendor: + vendor_path = os.path.join(cwd, vendor) + vendor_script = os.path.join(vendor, 'SConscript') + + if os.path.isfile(os.path.join(cwd, vendor_script)): + Env.Append(CPPPATH=[vendor_path]) + group += SConscript(vendor_script) + + if os.path.exists(os.path.join(vendor_path, 'mmu_vendor.h')): + Env.Append(ASFLAGS=' -D__HAS_MMU_VENDOR__') + Env.Append(CFLAGS=' -D__HAS_MMU_VENDOR__') + Env.Append(CXXFLAGS=' -D__HAS_MMU_VENDOR__') + + if os.path.exists(os.path.join(vendor_path, 'opcode_vendor.h')): + Env.Append(ASFLAGS=' -D__HAS_OPCODE_VENDOR__') + Env.Append(CFLAGS=' -D__HAS_OPCODE_VENDOR__') + Env.Append(CXXFLAGS=' -D__HAS_OPCODE_VENDOR__') +else: + common64_cpus = ['virt64', 'c906', 'c908', 'ur-cp100'] + + if cpu in common64_cpus: + group += SConscript(os.path.join('common64', 'SConscript')) + else: + group += SConscript(os.path.join('legacy_common', 'SConscript')) + + group += SConscript(os.path.join('vector', 'SConscript')) + + if vendor: + if vendor == 't-head': + port_script = os.path.join('legacy-t-head', cpu, 'SConscript') + else: + port_script = os.path.join(vendor, cpu, 'SConscript') -# cpu porting code files -if 'VENDOR' in vars(rtconfig) and rtconfig.VENDOR != '': - group = group + SConscript(os.path.join(rtconfig.VENDOR, rtconfig.CPU, 'SConscript')) -elif rtconfig.CPU in list: - group = group + SConscript(os.path.join(rtconfig.CPU, 'SConscript')) + if os.path.isfile(os.path.join(cwd, port_script)): + group += SConscript(port_script) + elif cpu: + port_script = os.path.join(cpu, 'SConscript') + if os.path.isfile(os.path.join(cwd, port_script)): + group += SConscript(port_script) Return('group') diff --git a/libcpu/risc-v/common/README.md b/libcpu/risc-v/common/README.md new file mode 100644 index 0000000000..89be8d0135 --- /dev/null +++ b/libcpu/risc-v/common/README.md @@ -0,0 +1,214 @@ +# RISC-V common porting guide + +The unified common port separates architectural code from platform policy. A +BSP selects the complete common startup, context and trap ABI with +`RT_USING_RISCV_NEW_COMMON`, then configures platform properties and interrupt +ownership independently. + +## Configuration model + +The following options describe independent properties. Do not infer one from a +CPU name or from another option. + +| Option | Responsibility | +| --- | --- | +| `ARCH_RISCV32` / `ARCH_RISCV64` | RISC-V sub-architecture and XLEN | +| `ARCH_RISCV_M_MODE` | M-mode CSR, trap return, timer and interrupt numbers | +| `ARCH_MM_MMU` | Page tables and virtual memory support | +| `RT_USING_CACHE` | Cache maintenance API and Zicbom block operations; optional on cacheless MCUs | +| `RT_USING_DM` | Device model and OFW driver framework | +| `ARCH_RISCV_USING_IRQ_STACK` | Per-CPU interrupt stack | + +The architecture generates `libcpu/risc-v/link.lds` only when the BSP root does +not provide `link.lds`. Device-model builds include `setup.c`; non-DM builds do +not. The DM setup supports both MMU and no-MMU systems. In a no-MMU build it +derives the heap range from `/memory`, limits it to `ARCH_HEAP_SIZE`, excludes +the `rt-thread,dma-pool` range in `/reserved-memory`, and installs that range +as the global DMA pool before device probes begin. A BSP without DMA uses the +remaining `/memory` range for the heap. The FDT itself is accessed directly and +firmware-provided FDT memory must remain accessible. OFW device MMIO uses the +identity mapping provided by `ioremap-nommu.c`. + +## Startup + +The generic startup follows the SBI boot ABI in S-mode: + +```text +a0 = boot hart ID +a1 = physical address of the FDT +``` + +In M-mode it reads `mhartid` directly and does not depend on the reset value of +`a0`. By default, selecting the new common makes this architecture code the sole +owner of `_start`; the BSP must stop compiling its previous reset entry. The +common startup clears BSS and enters `rtthread_startup()`, but it does not +relocate MCU data from flash or initialize clocks. + +A BSP that needs a platform-specific reset sequence can provide +`startup_gcc.S` in its BSP root. The build uses that file instead of the common +startup while continuing to use the common context, trap, CPU and timer +implementation. The BSP startup must provide `_start`, initialize the C runtime +state required by the platform, call `riscv_boot_hart_init()` for the boot hart, +and enter `rtthread_startup()`. An SMP BSP must also provide +`_secondary_cpu_entry`, use `riscv_hartid_to_cpu_id()` and +`riscv_cpu_id_set()` to establish the logical CPU ID before accessing per-CPU +state, and enter `rt_hw_secondary_cpu_bsp_start()`. An OFW BSP must preserve the +boot FDT and call `rt_hw_fdt_install_early()` before enabling address +translation or overwriting the firmware-provided FDT memory. + +`riscv_boot_hart_init()` registers the boot hart as logical CPU0 and initializes +its current CPU ID. The common SMP startup maps each secondary hart to a logical +CPU ID before initializing its per-CPU architecture state. + +## Hart IDs and SMP + +For a non-DM SMP system, override the weak hart enumeration function when hart +IDs are not `0..RT_CPUS_NR-1`: + +```c +#include <cpu.h> + +rt_size_t riscv_hartid_list_get(rt_ubase_t *hartids, rt_size_t max_harts) +{ + static const rt_ubase_t platform_hartids[] = { 1, 3, 5, 7 }; + rt_size_t count = RT_ARRAY_SIZE(platform_hartids); + + if (count > max_harts) + { + count = max_harts; + } + + rt_memcpy(hartids, platform_hartids, count * sizeof(*hartids)); + return count; +} +``` + +CPU boot, IPI and interrupt-controller code must translate IDs through +`riscv_cpu_id_to_hartid()` and `riscv_hartid_to_cpu_id()`. + +The `cpu_ops_t` SBI and spin-wait backends belong to the DM CPU-node startup +flow. A non-DM SMP BSP provides its own `rt_hw_secondary_cpu_up()` and +`rt_hw_secondary_cpu_bsp_start()` implementations according to its reset, +power and per-CPU initialization sequence. + +## Interrupt stack + +`ARCH_RISCV_USING_IRQ_STACK` gives each CPU a dedicated interrupt stack. It is +enabled by default for M-mode and non-DM ports, and disabled for RT-Smart, +where `xscratch` is used to switch between user and kernel stacks. The common +startup initializes it after the current logical CPU ID is available. + +The trap entry saves the exception frame on the interrupted thread stack, so +the scheduler and signal code keep the same frame ABI. Only the C interrupt or +exception handler runs on the per-CPU interrupt stack. Nested traps continue +on that interrupt stack. + +## Non-DM interrupts + +Non-DM builds provide the classic `rt_hw_interrupt_*` interface automatically. +The interrupt-controller topology is a BSP property, not an architecture +Kconfig choice. Define the controller type and IRQ table size in `board.h`: + +```c +#include <interrupt.h> + +#define MAX_HANDLERS 128 +#define INTC_TYPE INTC_TYPE_RISCV /* RISC-V INTC plus SiFive PLIC */ +/* #define INTC_TYPE INTC_TYPE_ANDESTECH */ +/* #define INTC_TYPE INTC_TYPE_CLIC */ +``` + +`INTC_TYPE_RISCV` is the fallback when a BSP does not define `INTC_TYPE`. +`MAX_HANDLERS` defaults to 128. Set both values explicitly for a new BSP. + +The BSP provides MMIO information by overriding only the hooks it uses: + +```c +#include <interrupt.h> + +rt_ubase_t platform_get_clint_base(void) { return CLINT_BASE; } +rt_size_t platform_get_clint_size(void) { return CLINT_SIZE; } + +rt_ubase_t platform_get_plic_base(void) { return PLIC_BASE; } +rt_size_t platform_get_plic_size(void) { return PLIC_SIZE; } +rt_ubase_t platform_get_plic_quirks(void) +{ + return RISCV_PLIC_QUIRK_EDGE_INTERRUPT; +} +``` + +All MMIO hooks return zero by default. CLINT is optional when a platform +overrides the timer and IPI implementations; PLIC or CLIC MMIO is required when +the corresponding non-DM interrupt backend is selected. In a no-MMU build, +these addresses are used through an explicit identity mapping. + +For CLIC, implement `platform_get_clic_base()`, +`platform_get_clic_size()` and optionally `platform_get_clic_quirks()`. The +common driver uses direct mode by default. Return `RISCV_CLIC_QUIRK_SHV` only +when the BSP supplies a valid hardware vector table through `fast_irq_vectors`. + +Use the mapping macros instead of assuming local and external vectors share a +number space: + +```c +rt_hw_interrupt_install(IRQ_PLIC(device_hwirq), isr, data, "device"); +rt_hw_interrupt_umask(IRQ_PLIC(device_hwirq)); +``` + +A platform with a different interrupt controller should add its backend to the +common PIC implementation as part of its migration. + +## Timer + +The common timer uses `rdtime` with SBI/Sstc in S-mode and CLINT `mtime` and +`mtimecmp` in M-mode. Initialize the frequency before starting the timer: + +```c +riscv_timer_set_frequency(TIMER_FREQUENCY); +riscv_timer_init(); +``` + +The frequency must be at least `RT_TICK_PER_SECOND`. On SMP systems the boot CPU +publishes the timer parameters before secondary CPUs initialize their local +timer interrupt. + +The common non-DM M-mode interrupt setup calls `riscv_timer_probe()` with the +CLINT registers. Platforms with another timer implementation can override the +weak `riscv_timer_rdtime()`, `riscv_timer_set_compare()` and +`rt_hw_tick_init()` functions. + +## ISA and atomics + +Non-DM BSPs populate the ISA bitmap during board initialization. For example: + +```c +riscv_isa_ext_set(I); +riscv_isa_ext_set(M); +riscv_isa_ext_set(A); +riscv_isa_ext_set(C); +``` + +`atomic_riscv.c` is compiled only with `RT_USING_HW_ATOMIC`. SMP selects this +option because the common spinlock implementation requires the A extension. +Uniprocessor MCU ports without A use the kernel software atomic path. + +## Board initialization + +A typical non-DM board performs initialization in this order: + +```c +void rt_hw_board_init(void) +{ + riscv_isa_ext_set(I); + rt_hw_cpu_icache_enable(); + rt_hw_cpu_dcache_enable(); + rt_hw_interrupt_init(); + riscv_timer_set_frequency(TIMER_FREQUENCY); + riscv_timer_init(); + + /* Heap, console, components and device initialization follow. */ +} +``` + +Validate startup, tick, exception handling and thread switching first. SMP +ports must additionally test a non-zero boot hart and every secondary CPU. diff --git a/libcpu/risc-v/common/SConscript b/libcpu/risc-v/common/SConscript index 814cfb01d5..fe50f4fa96 100644 --- a/libcpu/risc-v/common/SConscript +++ b/libcpu/risc-v/common/SConscript @@ -1,15 +1,93 @@ -# RT-Thread building script for component +# RT-Thread building script for the unified RISC-V CPU port +import os from building import * -cwd = GetCurrentDir() -src = Glob('*.c') + Glob('*.cpp') + Glob('*_gcc.S') -CPPPATH = [cwd] -ASFLAGS = ' -I ' + cwd +Import('rtconfig') -if not GetDepend('RT_USING_HW_ATOMIC'): - SrcRemove(src, 'atomic_riscv.c') +cwd = GetCurrentDir() +bsp_path = Dir('#').abspath +CPPPATH = [cwd + '/include', cwd] -group = DefineGroup('CPU', src, depend = [''], CPPPATH = CPPPATH, ASFLAGS = ASFLAGS) +if GetDepend('ARCH_RISCV_VECTOR'): + CPPPATH += [os.path.join(cwd, '..', 'vector', 'rvv-1.0')] +bsp_startup_obj = [] +builtin_fdt_obj = [] +src = [ + 'backtrace.c', + 'cpu.c', + 'cpuport.c', + 'isa_ext.c', + 'riscv_timer.c', + 'stack.c', + 'trap.c', +] + +if GetDepend('RT_USING_CACHE'): + src += ['cache.c'] + +if rtconfig.PLATFORM in ['gcc']: + src += [ + 'context_gcc.S', + 'cpu_gcc.S', + 'stack_gcc.S', + 'startup.c', + 'vector_gcc.S', + ] + + bsp_startup = os.path.join(bsp_path, 'startup_gcc.S') + if os.path.isfile(bsp_startup): + bsp_startup_obj = Env.Object('startup_gcc.o', '#startup_gcc.S') + else: + src += ['startup_gcc.S'] + + if GetDepend('RT_USING_BUILTIN_FDT'): + dtb_path = os.path.join(bsp_path, GetDepend('RT_BUILTIN_FDT_PATH').strip('"')) + builtin_fdt_obj = Env.Object('builtin_fdt_gcc.o', 'builtin_fdt_gcc.S') + Env.Depends(builtin_fdt_obj, dtb_path) + +if GetDepend('ARCH_MM_MMU'): + src += ['mmu.c'] + +if GetDepend('RT_USING_HW_ATOMIC'): + src += ['atomic_riscv.c'] + +if GetDepend('RT_USING_LWP'): + src += ['syscall.c'] + +if not GetDepend('ARCH_RISCV_M_MODE'): + src += ['sbi.c'] + + if not GetDepend('RT_USING_DM'): + src += ['console.c'] + +if GetDepend('RT_USING_DM'): + src += ['setup.c'] + + if GetDepend('RT_USING_SMP'): + src += ['cpu_spinwait.c'] + + if not GetDepend('ARCH_RISCV_M_MODE'): + src += ['cpu_sbi.c'] + +if not GetDepend('RT_USING_DM'): + src += ['interrupt.c'] + +if GetDepend('ARCH_RISCV_USING_IRQ_STACK'): + src += ['irq_stack.c'] + +group = DefineGroup('libcpu', src, depend = [''], CPPPATH = CPPPATH) +group += bsp_startup_obj +group += builtin_fdt_obj + +# build for sub-directory +list = os.listdir(cwd) +objs = [] + +for d in list: + path = os.path.join(cwd, d) + if os.path.isfile(os.path.join(path, 'SConscript')): + objs = objs + SConscript(os.path.join(d, 'SConscript')) +group = group + objs Return('group') diff --git a/libcpu/risc-v/common/atomic_riscv.c b/libcpu/risc-v/common/atomic_riscv.c index bc1561f2ee..3286c94d9a 100644 --- a/libcpu/risc-v/common/atomic_riscv.c +++ b/libcpu/risc-v/common/atomic_riscv.c @@ -10,150 +10,113 @@ #include <rtthread.h> +#include <asm-generic.h> + rt_atomic_t rt_hw_atomic_exchange(volatile rt_atomic_t *ptr, rt_atomic_t val) { rt_atomic_t result = 0; -#if __riscv_xlen == 32 - asm volatile ("amoswap.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); -#elif __riscv_xlen == 64 - asm volatile ("amoswap.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); -#endif + + __asm__ volatile ("amoswap." REG_WIDTH " %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); + return result; } rt_atomic_t rt_hw_atomic_add(volatile rt_atomic_t *ptr, rt_atomic_t val) { rt_atomic_t result = 0; -#if __riscv_xlen == 32 - asm volatile ("amoadd.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); -#elif __riscv_xlen == 64 - asm volatile ("amoadd.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); -#endif + + __asm__ volatile ("amoadd." REG_WIDTH " %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); + return result; } rt_atomic_t rt_hw_atomic_sub(volatile rt_atomic_t *ptr, rt_atomic_t val) { rt_atomic_t result = 0; + val = -val; -#if __riscv_xlen == 32 - asm volatile ("amoadd.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); -#elif __riscv_xlen == 64 - asm volatile ("amoadd.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); -#endif + __asm__ volatile ("amoadd." REG_WIDTH " %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); + return result; } rt_atomic_t rt_hw_atomic_xor(volatile rt_atomic_t *ptr, rt_atomic_t val) { rt_atomic_t result = 0; -#if __riscv_xlen == 32 - asm volatile ("amoxor.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); -#elif __riscv_xlen == 64 - asm volatile ("amoxor.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); -#endif + + __asm__ volatile ("amoxor." REG_WIDTH " %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); + return result; } rt_atomic_t rt_hw_atomic_and(volatile rt_atomic_t *ptr, rt_atomic_t val) { rt_atomic_t result = 0; -#if __riscv_xlen == 32 - asm volatile ("amoand.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); -#elif __riscv_xlen == 64 - asm volatile ("amoand.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); -#endif + + __asm__ volatile ("amoand." REG_WIDTH " %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); + return result; } rt_atomic_t rt_hw_atomic_or(volatile rt_atomic_t *ptr, rt_atomic_t val) { rt_atomic_t result = 0; -#if __riscv_xlen == 32 - asm volatile ("amoor.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); -#elif __riscv_xlen == 64 - asm volatile ("amoor.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); -#endif + + __asm__ volatile ("amoor." REG_WIDTH " %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); + return result; } rt_atomic_t rt_hw_atomic_load(volatile rt_atomic_t *ptr) { rt_atomic_t result = 0; -#if __riscv_xlen == 32 - asm volatile ("amoxor.w %0, x0, (%1)" : "=r"(result) : "r"(ptr) : "memory"); -#elif __riscv_xlen == 64 - asm volatile ("amoxor.d %0, x0, (%1)" : "=r"(result) : "r"(ptr) : "memory"); -#endif + + __asm__ volatile ("amoxor." REG_WIDTH " %0, x0, (%1)" : "=r"(result) : "r"(ptr) : "memory"); + return result; } void rt_hw_atomic_store(volatile rt_atomic_t *ptr, rt_atomic_t val) { rt_atomic_t result = 0; -#if __riscv_xlen == 32 - asm volatile ("amoswap.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); -#elif __riscv_xlen == 64 - asm volatile ("amoswap.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); -#endif + + __asm__ volatile ("amoswap." REG_WIDTH " %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); } rt_atomic_t rt_hw_atomic_flag_test_and_set(volatile rt_atomic_t *ptr) { - rt_atomic_t result = 0; - rt_atomic_t temp = 1; -#if __riscv_xlen == 32 - asm volatile ("amoor.w %0, %1, (%2)" : "=r"(result) : "r"(temp), "r"(ptr) : "memory"); -#elif __riscv_xlen == 64 - asm volatile ("amoor.d %0, %1, (%2)" : "=r"(result) : "r"(temp), "r"(ptr) : "memory"); -#endif + rt_atomic_t result = 0, temp = 1; + + __asm__ volatile ("amoor." REG_WIDTH " %0, %1, (%2)" : "=r"(result) : "r"(temp), "r"(ptr) : "memory"); + return result; } void rt_hw_atomic_flag_clear(volatile rt_atomic_t *ptr) { rt_atomic_t result = 0; -#if __riscv_xlen == 32 - asm volatile ("amoand.w %0, x0, (%1)" : "=r"(result) :"r"(ptr) : "memory"); -#elif __riscv_xlen == 64 - asm volatile ("amoand.d %0, x0, (%1)" : "=r"(result) :"r"(ptr) : "memory"); -#endif + + __asm__ volatile ("amoand." REG_WIDTH " %0, x0, (%1)" : "=r"(result) :"r"(ptr) : "memory"); } rt_atomic_t rt_hw_atomic_compare_exchange_strong(volatile rt_atomic_t *ptr, rt_atomic_t *old, rt_atomic_t desired) { - rt_atomic_t tmp = *old; - rt_atomic_t result = 0; -#if __riscv_xlen == 32 - asm volatile( - " fence iorw, ow\n" - "1: lr.w.aq %[result], (%[ptr])\n" - " bne %[result], %[tmp], 2f\n" - " sc.w.rl %[tmp], %[desired], (%[ptr])\n" - " bnez %[tmp], 1b\n" - " li %[result], 1\n" - " j 3f\n" - " 2:sw %[result], (%[old])\n" - " li %[result], 0\n" - " 3:\n" - : [result]"+r" (result), [tmp]"+r" (tmp), [ptr]"+r" (ptr) - : [desired]"r" (desired), [old]"r"(old) - : "memory"); -#elif __riscv_xlen == 64 - asm volatile( + rt_atomic_t tmp = *old, result = 0; + + __asm__ volatile ( " fence iorw, ow\n" - "1: lr.d.aq %[result], (%[ptr])\n" + "1: lr." REG_WIDTH ".aq %[result], (%[ptr])\n" " bne %[result], %[tmp], 2f\n" - " sc.d.rl %[tmp], %[desired], (%[ptr])\n" + " sc." REG_WIDTH ".rl %[tmp], %[desired], (%[ptr])\n" " bnez %[tmp], 1b\n" " li %[result], 1\n" " j 3f\n" - " 2:sd %[result], (%[old])\n" + " 2:" REG_S " %[result], (%[old])\n" " li %[result], 0\n" " 3:\n" : [result]"+r" (result), [tmp]"+r" (tmp), [ptr]"+r" (ptr) : [desired]"r" (desired), [old]"r"(old) : "memory"); -#endif + return result; } diff --git a/libcpu/risc-v/common/backtrace.c b/libcpu/risc-v/common/backtrace.c new file mode 100644 index 0000000000..50f417a587 --- /dev/null +++ b/libcpu/risc-v/common/backtrace.c @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-10-18 Shell Add backtrace support + */ + +#define DBG_TAG "hw.backtrace" +#define DBG_LVL DBG_INFO +#include <rtdbg.h> + +#include <stack.h> +#include <rtthread.h> +#ifdef ARCH_MM_MMU +#include <mmu.h> +#include <mm_aspace.h> +#endif + +#define WORD sizeof(rt_base_t) +#define ARCH_CONTEXT_FETCH(pctx, id) (*(((unsigned long *)pctx) + (id))) + +rt_inline rt_err_t _bt_kaddr(rt_ubase_t *fp, struct rt_hw_backtrace_frame *frame) +{ + rt_err_t rc; + frame->fp = *(fp - 2); + frame->pc = *(fp - 1); + + if ((rt_ubase_t)fp == frame->fp) + { + rc = -RT_ERROR; + } + else + { + rc = RT_EOK; + } + return rc; +} + +#ifndef ARCH_MM_MMU +static rt_bool_t _bt_thread_stack_contains(rt_thread_t thread, rt_ubase_t *fp) +{ + rt_ubase_t frame_start; + rt_ubase_t stack_start; + rt_ubase_t stack_end; + + if (!thread) + { + thread = rt_thread_self(); + } + if (!thread || (rt_ubase_t)fp < 2 * WORD) + { + return RT_FALSE; + } + + frame_start = (rt_ubase_t)(fp - 2); + stack_start = (rt_ubase_t)thread->stack_addr; + stack_end = stack_start + thread->stack_size; + + return frame_start >= stack_start && + frame_start + 2 * WORD <= stack_end; +} +#endif + +#ifdef RT_USING_SMART +#include <lwp_arch.h> +#include <lwp_user_mm.h> + +rt_inline rt_err_t _bt_uaddr(rt_lwp_t lwp, rt_ubase_t *fp, struct rt_hw_backtrace_frame *frame) +{ + rt_err_t rc; + if (lwp_data_get(lwp, &frame->fp, fp - 2, WORD) != WORD) + { + rc = -RT_EFAULT; + } + else if (lwp_data_get(lwp, &frame->pc, fp - 1, WORD) != WORD) + { + rc = -RT_EFAULT; + } + else if ((rt_ubase_t)fp == frame->fp) + { + rc = -RT_ERROR; + } + else + { + frame->pc -= 0; + rc = RT_EOK; + } + return rc; +} +#endif /* RT_USING_SMART */ + +rt_err_t rt_hw_backtrace_frame_unwind(rt_thread_t thread, struct rt_hw_backtrace_frame *frame) +{ + rt_err_t rc = -RT_ERROR; + rt_ubase_t *fp = (rt_ubase_t *)frame->fp; + + if (fp && !((long)fp & 0x7)) + { + #ifdef RT_USING_SMART + if (thread && thread->lwp) + { + void *lwp = thread->lwp; + void *this_lwp = lwp_self(); + if (this_lwp == lwp && rt_hw_mmu_v2p(((rt_lwp_t)lwp)->aspace, fp) != ARCH_MAP_FAILED) + { + rc = _bt_kaddr(fp, frame); + } + else if (lwp_user_accessible_ext(lwp, (void *)fp, WORD)) + { + rc = _bt_uaddr(lwp, fp, frame); + } + else + { + rc = -RT_EFAULT; + } + } + else + #endif /* RT_USING_SMART */ + #ifdef ARCH_MM_MMU + if (rt_kmem_v2p(fp) != ARCH_MAP_FAILED) + #else + if (_bt_thread_stack_contains(thread, fp)) + #endif + { + rc = _bt_kaddr(fp, frame); + } + else + { + rc = -RT_EINVAL; + } + } + else + { + rc = -RT_EFAULT; + } + return rc; +} + +rt_err_t rt_hw_backtrace_frame_get(rt_thread_t thread, struct rt_hw_backtrace_frame *frame) +{ + rt_err_t rc; + + if (!thread || !frame) + { + rc = -RT_EINVAL; + } + else + { + rt_hw_switch_frame_t sframe = thread->sp; + frame->pc = sframe->regs[RT_HW_SWITCH_CONTEXT_RA]; + frame->fp = sframe->regs[RT_HW_SWITCH_CONTEXT_S0]; + rc = RT_EOK; + } + return rc; +} diff --git a/libcpu/risc-v/common/builtin_fdt_gcc.S b/libcpu/risc-v/common/builtin_fdt_gcc.S new file mode 100644 index 0000000000..834f50f3f2 --- /dev/null +++ b/libcpu/risc-v/common/builtin_fdt_gcc.S @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2025-04-20 GuEe-GUI The first version + */ + +#include <asm-generic.h> + + .section ".data" + + .align RISCV_SZPTR + .globl rt_hw_builtin_fdt +rt_hw_builtin_fdt: +#ifdef RT_BUILTIN_FDT_PATH + .incbin RT_BUILTIN_FDT_PATH +#endif diff --git a/libcpu/risc-v/common/cache.c b/libcpu/risc-v/common/cache.c new file mode 100644 index 0000000000..57274287d8 --- /dev/null +++ b/libcpu/risc-v/common/cache.c @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2006-2021, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2021-01-29 lizhirui first version + * 2025-04-20 GuEe-GUI Port to the DM + * 2025-04-26 GuEe-GUI Follow the specific and vendor + */ + +#include <rthw.h> +#include <rtthread.h> +#include <rtcompiler.h> + +#include <cpu.h> +#include <sbi.h> +#include <cache.h> +#include <opcode.h> +#include <isa_ext.h> +#ifdef RT_USING_SMP +#include <smp_call.h> +#endif +#ifdef RT_USING_HWCACHE +#include <drivers/hwcache.h> +#endif + +static rt_uint32_t riscv_cbom_block_size; + +void riscv_cache_set_cbom_block_size(rt_uint32_t size) +{ + if (!riscv_isa_ext_test(ZICBOM)) + { + RT_ASSERT(0); + return; + } + + HWREG32(&riscv_cbom_block_size) = size; + rt_hw_wmb(); +} + +rt_uint32_t riscv_cache_get_cbom_block_size(void) +{ + rt_hw_rmb(); + return HWREG32(&riscv_cbom_block_size); +} + +rt_always_inline void local_flush_icache_all(void) +{ + __asm__ volatile (OPC_FENCE_I:::"memory"); +} + +rt_inline void ipi_remote_fence_i(void *data) +{ + RT_UNUSED(data); + + local_flush_icache_all(); +} + +rt_weak void rt_hw_cpu_icache_enable(void) +{ +#ifdef RT_USING_HWCACHE + rt_hwcache_icache_enable(); +#endif +} + +rt_weak void rt_hw_cpu_icache_disable(void) +{ +#ifdef RT_USING_HWCACHE + rt_hwcache_icache_disable(); +#endif +} + +rt_weak rt_base_t rt_hw_cpu_icache_status(void) +{ +#ifdef RT_USING_HWCACHE + return rt_hwcache_icache_status(); +#elif !defined(ARCH_RISCV_DISABLE_FENCE_I) + return 0; +#else /* !ARCH_RISCV_DISABLE_FENCE_I */ + return !0; +#endif /* RT_USING_HWCACHE */ +} + +rt_weak void rt_hw_cpu_icache_ops(int ops, void *addr, int size) +{ +#ifdef RT_USING_HWCACHE + rt_hwcache_icache_ops(ops, addr, size); +#elif !defined(ARCH_RISCV_DISABLE_FENCE_I) + /* + * RISC-V doesn't have an instruction to flush parts of the icache, + * so instead we just flush the whole thing. + */ + if (ops == RT_HW_CACHE_INVALIDATE) + { + local_flush_icache_all(); + + #if (defined(RT_USING_SMP) && RT_CPUS_NR > 1) || defined(RT_USING_AMP) + #ifndef ARCH_RISCV_M_MODE + if (has_ipi_extension) + { + sbi_remote_fence_i(RT_NULL); + } + else + #endif /* !ARCH_RISCV_M_MODE */ + { + rt_smp_call_each_cpu(ipi_remote_fence_i, RT_NULL, SMP_CALL_WAIT_ALL); + } + #endif /* RT_USING_SMP */ + } + RT_UNUSED(addr); + RT_UNUSED(size); +#else /* !ARCH_RISCV_DISABLE_FENCE_I */ + RT_UNUSED(ops); + RT_UNUSED(addr); + RT_UNUSED(size); +#endif /* RT_USING_HWCACHE */ +} + +rt_weak void rt_hw_cpu_dcache_enable(void) +{ +#ifdef RT_USING_HWCACHE + rt_hwcache_dcache_enable(); +#endif +} + +rt_weak void rt_hw_cpu_dcache_disable(void) +{ +#ifdef RT_USING_HWCACHE + rt_hwcache_dcache_disable(); +#endif +} + +rt_weak rt_base_t rt_hw_cpu_dcache_status(void) +{ +#ifdef RT_USING_HWCACHE + return rt_hwcache_dcache_status(); +#else + return riscv_cache_get_cbom_block_size() == 0; +#endif +} + +rt_weak void rt_hw_cpu_dcache_ops(int ops, void *addr, int size) +{ +#ifdef RT_USING_HWCACHE + rt_hwcache_dcache_ops(ops, addr, size); +#else + rt_uint32_t cbom_block_size; + + cbom_block_size = riscv_cache_get_cbom_block_size(); + + if (cbom_block_size) + { + if (ops == RT_HW_CACHE_FLUSH) + { + __asm__ volatile ( + " mv a0, %1\n\t" + " j 2f\n\t" + "3:\n\t" + OPC_CBO_CLEAN(a0) + " add a0, a0, %0\n\t" + "2:\n\t" + " bltu a0, %2, 3b\n\t" + ::"r"(cbom_block_size), + "r"((rt_ubase_t)addr & ~(cbom_block_size - 1UL)), + "r"((rt_ubase_t)addr + size) + : "a0"); + } + else if (ops == RT_HW_CACHE_INVALIDATE) + { + __asm__ volatile ( + " mv a0, %1\n\t" + " j 2f\n\t" + "3:\n\t" + OPC_CBO_INVAL(a0) + " add a0, a0, %0\n\t" + "2:\n\t" + " bltu a0, %2, 3b\n\t" + ::"r"(cbom_block_size), + "r"((rt_ubase_t)addr & ~(cbom_block_size - 1UL)), + "r"((rt_ubase_t)addr + size) + : "a0"); + } + } + else + { + __asm__ volatile (".rept 6\n nop\n .endr\n"); + } +#endif /* RT_USING_HWCACHE */ +} + +rt_weak void rt_hw_cpu_icache_invalidate_all(void) +{ + rt_hw_cpu_icache_ops(RT_HW_CACHE_INVALIDATE, RT_NULL, ~0U); +} + +rt_weak void rt_hw_cpu_icache_invalidate(void *addr, rt_size_t size) +{ + rt_hw_cpu_icache_ops(RT_HW_CACHE_INVALIDATE, addr, size); +} + +rt_weak void rt_hw_cpu_dcache_clean_and_invalidate(void *addr, rt_size_t size) +{ + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, addr, size); +} + +rt_weak void rt_hw_icache_invalidate_all(void) +{ + rt_hw_cpu_icache_ops(RT_HW_CACHE_INVALIDATE, RT_NULL, ~0U); +} + +rt_weak void rt_hw_sync_cache_local(void *addr, int size) +{ + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, addr, size); + rt_hw_cpu_icache_ops(RT_HW_CACHE_INVALIDATE, addr, size); +} diff --git a/libcpu/risc-v/common/console.c b/libcpu/risc-v/common/console.c new file mode 100644 index 0000000000..a2ce9f148b --- /dev/null +++ b/libcpu/risc-v/common/console.c @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2021-05-18 Bernard port from FreeBSD + */ + +#include <sbi.h> + +void rt_hw_console_output(const char *str) +{ + while (*str) + { + sbi_console_putchar(*str++); + } +} diff --git a/libcpu/risc-v/common/context_gcc.S b/libcpu/risc-v/common/context_gcc.S index 792ac8bd67..e0533c942c 100644 --- a/libcpu/risc-v/common/context_gcc.S +++ b/libcpu/risc-v/common/context_gcc.S @@ -1,319 +1,306 @@ /* - * Copyright (c) 2006-2018, RT-Thread Development Team + * Copyright (c) 2006-2025, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * * Change Logs: * Date Author Notes - * 2018/10/28 Bernard The unify RISC-V porting implementation - * 2018/12/27 Jesven Add SMP support - * 2020/11/20 BalanceTWK Add FPU support - * 2022/12/28 WangShun Add macro to distinguish whether FPU is supported - * 2023/03/19 Flyingcys Add riscv_32e support + * 2018-10-28 Bernard The unify RISC-V porting implementation + * 2018-12-27 Jesven Add SMP support + * 2021-02-02 lizhirui Add userspace support + * 2022-10-22 Shell Support User mode RVV; + * Trimming process switch context + * 2025-04-20 GuEe-GUI Support SMP */ -#define __ASSEMBLY__ -#include "cpuport.h" +#include "asm.h" +#include "csr.h" -#ifdef RT_USING_SMP -#define rt_hw_interrupt_disable rt_hw_local_irq_disable -#define rt_hw_interrupt_enable rt_hw_local_irq_enable +#define REG_IDX(name) RT_HW_SWITCH_CONTEXT_##name + +.macro SAVE_REG reg, index + REG_S \reg, \index * SZREG(sp) +.endm + +.macro LOAD_REG reg, index + REG_L \reg, \index * SZREG(sp) +.endm + +.macro SAVE_CONTEXT + addi sp, sp, -(RT_HW_SWITCH_CONTEXT_SIZE * SZREG) + SAVE_REG tp, REG_IDX(TP) + SAVE_REG ra, REG_IDX(RA) + SAVE_REG s0, REG_IDX(S0) + SAVE_REG s1, REG_IDX(S1) +#ifndef ARCH_RISCV_32E + SAVE_REG s2, REG_IDX(S2) + SAVE_REG s3, REG_IDX(S3) + SAVE_REG s4, REG_IDX(S4) + SAVE_REG s5, REG_IDX(S5) + SAVE_REG s6, REG_IDX(S6) + SAVE_REG s7, REG_IDX(S7) + SAVE_REG s8, REG_IDX(S8) + SAVE_REG s9, REG_IDX(S9) + SAVE_REG s10, REG_IDX(S10) + SAVE_REG s11, REG_IDX(S11) #endif + csrr s1, CSR_STATUS + li s0, (SR_PP) + or s1, s1, s0 + SAVE_REG s1, REG_IDX(XSTATUS) +.endm -/* - * rt_base_t rt_hw_interrupt_disable(void); - */ - .globl rt_hw_interrupt_disable -rt_hw_interrupt_disable: - csrrci a0, mstatus, 8 - ret +.macro RESTORE_CONTEXT + LOAD_REG s1, REG_IDX(XSTATUS) + csrw CSR_STATUS, s1 +#ifndef ARCH_RISCV_32E + LOAD_REG s11, REG_IDX(S11) + LOAD_REG s10, REG_IDX(S10) + LOAD_REG s9, REG_IDX(S9) + LOAD_REG s8, REG_IDX(S8) + LOAD_REG s7, REG_IDX(S7) + LOAD_REG s6, REG_IDX(S6) + LOAD_REG s5, REG_IDX(S5) + LOAD_REG s4, REG_IDX(S4) + LOAD_REG s3, REG_IDX(S3) + LOAD_REG s2, REG_IDX(S2) +#endif + LOAD_REG s1, REG_IDX(S1) + LOAD_REG s0, REG_IDX(S0) + LOAD_REG ra, REG_IDX(RA) + RESTORE_TP (REG_IDX(TP) * SZREG) + addi sp, sp, RT_HW_SWITCH_CONTEXT_SIZE * SZREG + csrw CSR_EPC, ra +.endm -/* - * void rt_hw_interrupt_enable(rt_base_t level); - */ - .globl rt_hw_interrupt_enable -rt_hw_interrupt_enable: - csrw mstatus, a0 - ret + .section ".text","ax" +#ifdef RT_USING_SMP /* - * #ifdef RT_USING_SMP - * void rt_hw_context_switch_to(rt_ubase_t to, stuct rt_thread *to_thread); - * #else - * void rt_hw_context_switch_to(rt_ubase_t to); - * #endif - * a0 --> to + * void rt_hw_context_switch_to(rt_ubase_t to, struct rt_thread *to_thread); + * a0 --> to (thread stack) * a1 --> to_thread */ - .globl rt_hw_context_switch_to +.globl rt_hw_context_switch_to rt_hw_context_switch_to: - la t0, __rt_rvstack -#ifdef SOC_RISCV_FAMILY_CH32 -/* - * if it is an assembly entry code, the SP offset value is determined by the assembly code, - * but the C code is determined by the compiler, so we subtract 512 here as a reservation. - * When entering the interrupt function of C code, the compiler automatically presses the stack - * into the task stack. We can only change the SP value used by the calling function after switching - * the interrupt stack.This problem can be solved by modifying the interrupt to the assembly entry, - * and there is no need to reserve 512 bytes. You only need to switch the interrupt stack at the - * beginning of the interrupt function -*/ - addi t0, t0, -512 // for ch32 -#endif /* SOC_RISCV_FAMILY_CH32 */ - csrw mscratch,t0 + REG_L sp, (a0) - LOAD sp, (a0) + /* Reserved to_thread */ + mv s1, a1 -#ifdef RT_USING_SMP - mv a0, a1 - call rt_cpus_lock_status_restore + mv a0, s1 + call rt_cpus_lock_status_restore +#ifdef RT_USING_SMART + mv a0, s1 + call lwp_user_setting_restore #endif - LOAD a0, 2 * REGBYTES(sp) - csrw mstatus, a0 - j rt_hw_context_switch_exit + + j _context_switch_exit /* - * #ifdef RT_USING_SMP * void rt_hw_context_switch(rt_ubase_t from, rt_ubase_t to, struct rt_thread *to_thread); - * #else - * void rt_hw_context_switch(rt_ubase_t from, rt_ubase_t to); - * #endif - * - * a0 --> from - * a1 --> to + * a0 --> from (from_thread stack) + * a1 --> to (to_thread stack) * a2 --> to_thread */ - .globl rt_hw_context_switch +.globl rt_hw_context_switch rt_hw_context_switch: - /* saved from thread context - * x1/ra -> sp(0) - * x1/ra -> sp(1) - * mstatus.mie -> sp(2) - * x(i) -> sp(i-4) - */ -#ifdef ARCH_RISCV_FPU - addi sp, sp, -32 * FREGBYTES + SAVE_CONTEXT + mv a3, sp + REG_S a3, (a0) /* Store sp in preempted tasks TCB */ + REG_L a0, (a1) /* Get new task stack pointer */ + mv sp, a0 - FSTORE f0, 0 * FREGBYTES(sp) - FSTORE f1, 1 * FREGBYTES(sp) - FSTORE f2, 2 * FREGBYTES(sp) - FSTORE f3, 3 * FREGBYTES(sp) - FSTORE f4, 4 * FREGBYTES(sp) - FSTORE f5, 5 * FREGBYTES(sp) - FSTORE f6, 6 * FREGBYTES(sp) - FSTORE f7, 7 * FREGBYTES(sp) - FSTORE f8, 8 * FREGBYTES(sp) - FSTORE f9, 9 * FREGBYTES(sp) - FSTORE f10, 10 * FREGBYTES(sp) - FSTORE f11, 11 * FREGBYTES(sp) - FSTORE f12, 12 * FREGBYTES(sp) - FSTORE f13, 13 * FREGBYTES(sp) - FSTORE f14, 14 * FREGBYTES(sp) - FSTORE f15, 15 * FREGBYTES(sp) - FSTORE f16, 16 * FREGBYTES(sp) - FSTORE f17, 17 * FREGBYTES(sp) - FSTORE f18, 18 * FREGBYTES(sp) - FSTORE f19, 19 * FREGBYTES(sp) - FSTORE f20, 20 * FREGBYTES(sp) - FSTORE f21, 21 * FREGBYTES(sp) - FSTORE f22, 22 * FREGBYTES(sp) - FSTORE f23, 23 * FREGBYTES(sp) - FSTORE f24, 24 * FREGBYTES(sp) - FSTORE f25, 25 * FREGBYTES(sp) - FSTORE f26, 26 * FREGBYTES(sp) - FSTORE f27, 27 * FREGBYTES(sp) - FSTORE f28, 28 * FREGBYTES(sp) - FSTORE f29, 29 * FREGBYTES(sp) - FSTORE f30, 30 * FREGBYTES(sp) - FSTORE f31, 31 * FREGBYTES(sp) + /* Backup thread self */ + mv s1, a2 + mv a0, s1 + call rt_cpus_lock_status_restore +#ifdef RT_USING_SMART + mv a0, s1 + call lwp_user_setting_restore #endif -#ifndef __riscv_32e - addi sp, sp, -32 * REGBYTES + j _context_switch_exit + +#define EXP_FRAME t0 +#define FROM_SPP t1 +#define TO_SPP t2 +#define TO_TCB a4 +#define TMP a5 + +/* + * void rt_hw_context_switch_interrupt(rt_ubase_t context, rt_ubase_t from, rt_ubase_t to, struct rt_thread *to_thread) + * a0 --> interrupt context + * a1 --> addr of from_thread's sp + * a2 --> addr of to_thread's sp + * a3 --> to_thread's tcb + */ +.globl rt_hw_context_switch_interrupt +rt_hw_context_switch_interrupt: +#ifdef RT_USING_DEBUG + /* Debug frame for backtrace */ +#if __riscv_xlen == 64 + addi sp, sp, -(48) + sd ra, 40(sp) + sd s0, 32(sp) +#elif __riscv_xlen == 32 + addi sp, sp, -(32) + sw ra, 28(sp) + sw s0, 24(sp) #else - addi sp, sp, -16 * REGBYTES +#error "Unexpected __riscv_xlen" #endif +#endif /* RT_USING_DEBUG */ + + /* We can discard all the previous ABI here */ + mv EXP_FRAME, a0 + mv FROM_SPP, a1 + mv TO_SPP, a2 + mv TO_TCB, a3 - STORE sp, (a0) +#ifdef RT_USING_SMART + PUSH EXP_FRAME + PUSH FROM_SPP + PUSH TO_SPP + PUSH TO_TCB - STORE x1, 0 * REGBYTES(sp) - STORE x1, 1 * REGBYTES(sp) + call rt_thread_self + call lwp_user_setting_save - csrr a0, mstatus - andi a0, a0, 8 - beqz a0, save_mpie - li a0, 0x80 -save_mpie: - STORE a0, 2 * REGBYTES(sp) + POP TO_TCB + POP TO_SPP + POP FROM_SPP + POP EXP_FRAME +#endif /* RT_USING_SMART */ - STORE x4, 4 * REGBYTES(sp) - STORE x5, 5 * REGBYTES(sp) - STORE x6, 6 * REGBYTES(sp) - STORE x7, 7 * REGBYTES(sp) - STORE x8, 8 * REGBYTES(sp) - STORE x9, 9 * REGBYTES(sp) - STORE x10, 10 * REGBYTES(sp) - STORE x11, 11 * REGBYTES(sp) - STORE x12, 12 * REGBYTES(sp) - STORE x13, 13 * REGBYTES(sp) - STORE x14, 14 * REGBYTES(sp) - STORE x15, 15 * REGBYTES(sp) -#ifndef __riscv_32e - STORE x16, 16 * REGBYTES(sp) - STORE x17, 17 * REGBYTES(sp) - STORE x18, 18 * REGBYTES(sp) - STORE x19, 19 * REGBYTES(sp) - STORE x20, 20 * REGBYTES(sp) - STORE x21, 21 * REGBYTES(sp) - STORE x22, 22 * REGBYTES(sp) - STORE x23, 23 * REGBYTES(sp) - STORE x24, 24 * REGBYTES(sp) - STORE x25, 25 * REGBYTES(sp) - STORE x26, 26 * REGBYTES(sp) - STORE x27, 27 * REGBYTES(sp) - STORE x28, 28 * REGBYTES(sp) - STORE x29, 29 * REGBYTES(sp) - STORE x30, 30 * REGBYTES(sp) - STORE x31, 31 * REGBYTES(sp) + /* Reset SP of from-thread */ + mv sp, EXP_FRAME + + addi sp, sp, -(RT_HW_SWITCH_CONTEXT_SIZE * SZREG) + /* Resume according to the privilege mode saved in the exception frame. */ + la ra, rt_hw_resume_interrupt + /* Save the entire context */ + li TMP, (K_XSTATUS_DEFAULT & ~SR_PIE) + REG_S TMP, REG_IDX(XSTATUS) * SZREG(sp) + REG_S ra, REG_IDX(RA) * SZREG(sp) + + /* Save SP of from-thread */ + mv a0, sp + REG_S a0, (FROM_SPP) + + /* Setup SP to to-thread's */ + REG_L a0, (TO_SPP) + mv sp, a0 + +#ifdef RT_USING_SMART + PUSH TO_TCB #endif - /* restore to thread context - * sp(0) -> epc; - * sp(1) -> ra; - * sp(i) -> x(i+2) - */ - LOAD sp, (a1) + mv a0, TO_TCB + call rt_cpus_lock_status_restore +#ifdef RT_USING_SMART + POP a0 /* TO_TCB */ + call lwp_user_setting_restore +#endif + j _context_switch_exit -#ifdef RT_USING_SMP - mv a0, a2 - call rt_cpus_lock_status_restore -#endif /*RT_USING_SMP*/ +#else /* !RT_USING_SMP */ + .section ".data" - j rt_hw_context_switch_exit + .align RISCV_SZPTR +/** + * @brief from thread used interrupt context switch + */ +.globl rt_interrupt_from_thread +rt_interrupt_from_thread: + RISCV_PTR 0 + +/** + * @brief to thread used interrupt context switch + */ +.globl rt_interrupt_to_thread +rt_interrupt_to_thread: + RISCV_PTR 0 + +/** + * @brief flag to indicate context switch in interrupt or not + */ +.globl rt_thread_switch_interrupt_flag +rt_thread_switch_interrupt_flag: + RISCV_PTR 0 + + .section ".text","ax" -#ifdef RT_USING_SMP /* - * void rt_hw_context_switch_interrupt(void *context, rt_ubase_t from, rt_ubase_t to, struct rt_thread *to_thread); + * void rt_hw_context_switch_to(rt_ubase_t to); * - * a0 --> context - * a1 --> from - * a2 --> to - * a3 --> to_thread + * a0 --> to SP pointer */ - .globl rt_hw_context_switch_interrupt -rt_hw_context_switch_interrupt: +.globl rt_hw_context_switch_to +rt_hw_context_switch_to: + REG_L sp, (a0) - STORE a0, 0(a1) + call rt_thread_self + mv s1, a0 - LOAD sp, 0(a2) - move a0, a3 - call rt_cpus_lock_status_restore +#ifdef RT_USING_SMART + call lwp_aspace_switch +#endif - j rt_hw_context_switch_exit + j _context_switch_exit -#endif +/* + * void rt_hw_context_switch(rt_ubase_t from, rt_ubase_t to); + * + * a0 --> from SP pointer + * a1 --> to SP pointer + * + * It should only be used on local interrupt disable + */ +.globl rt_hw_context_switch +rt_hw_context_switch: + SAVE_CONTEXT + REG_S sp, (a0) -.global rt_hw_context_switch_exit -rt_hw_context_switch_exit: -#ifdef RT_USING_SMP -#ifdef RT_USING_SIGNALS - mv a0, sp + /* Restore to thread SP */ + REG_L sp, (a1) - csrr t0, mhartid - /* switch interrupt stack of current cpu */ - la sp, __stack_start__ - addi t1, t0, 1 - li t2, __STACKSIZE__ - mul t1, t1, t2 - add sp, sp, t1 /* sp = (cpuid + 1) * __STACKSIZE__ + __stack_start__ */ + /* Restore Address Space */ + call rt_thread_self + mv s1, a0 - call rt_signal_check - mv sp, a0 -#endif +#ifdef RT_USING_SMART + call lwp_aspace_switch #endif - /* resw ra to mepc */ - LOAD a0, 0 * REGBYTES(sp) - csrw mepc, a0 - LOAD x1, 1 * REGBYTES(sp) -#ifdef ARCH_RISCV_FPU - li t0, 0x7800 -#else - li t0, 0x1800 -#endif - csrw mstatus, t0 - LOAD a0, 2 * REGBYTES(sp) - csrs mstatus, a0 + j _context_switch_exit + +/* + * void rt_hw_context_switch_interrupt(rt_ubase_t from, rt_ubase_t to, rt_thread_t from_thread, rt_thread_t to_thread) + * a0 --> from_thread's sp + * a1 --> to_thread's sp + * a2 --> from_thread's tcb + * a3 --> to_thread's tcb + */ +.globl rt_hw_context_switch_interrupt +rt_hw_context_switch_interrupt: + la t0, rt_thread_switch_interrupt_flag + REG_L t1, (t0) + bne t1, zero, _init_to_thread - LOAD x4, 4 * REGBYTES(sp) - LOAD x5, 5 * REGBYTES(sp) - LOAD x6, 6 * REGBYTES(sp) - LOAD x7, 7 * REGBYTES(sp) - LOAD x8, 8 * REGBYTES(sp) - LOAD x9, 9 * REGBYTES(sp) - LOAD x10, 10 * REGBYTES(sp) - LOAD x11, 11 * REGBYTES(sp) - LOAD x12, 12 * REGBYTES(sp) - LOAD x13, 13 * REGBYTES(sp) - LOAD x14, 14 * REGBYTES(sp) - LOAD x15, 15 * REGBYTES(sp) -#ifndef __riscv_32e - LOAD x16, 16 * REGBYTES(sp) - LOAD x17, 17 * REGBYTES(sp) - LOAD x18, 18 * REGBYTES(sp) - LOAD x19, 19 * REGBYTES(sp) - LOAD x20, 20 * REGBYTES(sp) - LOAD x21, 21 * REGBYTES(sp) - LOAD x22, 22 * REGBYTES(sp) - LOAD x23, 23 * REGBYTES(sp) - LOAD x24, 24 * REGBYTES(sp) - LOAD x25, 25 * REGBYTES(sp) - LOAD x26, 26 * REGBYTES(sp) - LOAD x27, 27 * REGBYTES(sp) - LOAD x28, 28 * REGBYTES(sp) - LOAD x29, 29 * REGBYTES(sp) - LOAD x30, 30 * REGBYTES(sp) - LOAD x31, 31 * REGBYTES(sp) + la t1, rt_interrupt_from_thread + REG_S a0, (t1) - addi sp, sp, 32 * REGBYTES -#else - addi sp, sp, 16 * REGBYTES -#endif +_init_to_thread: + la t1, rt_interrupt_to_thread + REG_S a1, (t1) -#ifdef ARCH_RISCV_FPU - FLOAD f0, 0 * FREGBYTES(sp) - FLOAD f1, 1 * FREGBYTES(sp) - FLOAD f2, 2 * FREGBYTES(sp) - FLOAD f3, 3 * FREGBYTES(sp) - FLOAD f4, 4 * FREGBYTES(sp) - FLOAD f5, 5 * FREGBYTES(sp) - FLOAD f6, 6 * FREGBYTES(sp) - FLOAD f7, 7 * FREGBYTES(sp) - FLOAD f8, 8 * FREGBYTES(sp) - FLOAD f9, 9 * FREGBYTES(sp) - FLOAD f10, 10 * FREGBYTES(sp) - FLOAD f11, 11 * FREGBYTES(sp) - FLOAD f12, 12 * FREGBYTES(sp) - FLOAD f13, 13 * FREGBYTES(sp) - FLOAD f14, 14 * FREGBYTES(sp) - FLOAD f15, 15 * FREGBYTES(sp) - FLOAD f16, 16 * FREGBYTES(sp) - FLOAD f17, 17 * FREGBYTES(sp) - FLOAD f18, 18 * FREGBYTES(sp) - FLOAD f19, 19 * FREGBYTES(sp) - FLOAD f20, 20 * FREGBYTES(sp) - FLOAD f21, 21 * FREGBYTES(sp) - FLOAD f22, 22 * FREGBYTES(sp) - FLOAD f23, 23 * FREGBYTES(sp) - FLOAD f24, 24 * FREGBYTES(sp) - FLOAD f25, 25 * FREGBYTES(sp) - FLOAD f26, 26 * FREGBYTES(sp) - FLOAD f27, 27 * FREGBYTES(sp) - FLOAD f28, 28 * FREGBYTES(sp) - FLOAD f29, 29 * FREGBYTES(sp) - FLOAD f30, 30 * FREGBYTES(sp) - FLOAD f31, 31 * FREGBYTES(sp) + li t1, 1 + REG_S t1, (t0) - addi sp, sp, 32 * FREGBYTES -#endif + ret +#endif /* RT_USING_SMP */ - mret +.local _context_switch_exit +_context_switch_exit: + RESTORE_CONTEXT + XRET diff --git a/libcpu/risc-v/common/cpu.c b/libcpu/risc-v/common/cpu.c new file mode 100644 index 0000000000..a74cd34438 --- /dev/null +++ b/libcpu/risc-v/common/cpu.c @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2025-04-20 GuEe-GUI The first version + */ + +#include <rthw.h> +#include <rtthread.h> +#include <rtcompiler.h> + +#include <csr.h> +#include <cpu.h> +#include <cpuport.h> +#ifdef ARCH_MM_MMU +#include <mmu.h> +#endif +#include <cache.h> + +#define RISCV_HARTID_INVALID (~(rt_ubase_t)0) + +rt_ubase_t boot_cpu_hartid; + +static rt_bool_t hartid_equ_cpuid = RT_FALSE; +static int mapped_cpus_nr; +rt_ubase_t cpu_id_to_hartid_map[RT_CPUS_NR] rt_align(L1_CACHE_BYTES); + +static void riscv_hartid_map_clear(void) +{ + for (int i = 0; i < RT_CPUS_NR; ++i) + { + cpu_id_to_hartid_map[i] = RISCV_HARTID_INVALID; + } + + mapped_cpus_nr = 0; + hartid_equ_cpuid = RT_FALSE; +} + +static void riscv_hartid_map_refresh_fastpath(void) +{ + hartid_equ_cpuid = RT_TRUE; + + for (int i = 0; i < mapped_cpus_nr; ++i) + { + if (cpu_id_to_hartid_map[i] != (rt_ubase_t)i) + { + hartid_equ_cpuid = RT_FALSE; + break; + } + } +} + +#if defined(RT_USING_SMP) && defined(ARCH_MM_MMU) +extern char __percpu_hartid; + +static rt_ubase_t *percpu_hartid rt_align(L1_CACHE_BYTES); + +static rt_bool_t percpu_hartid_ptr_valid(rt_ubase_t *ptr) +{ + rt_ubase_t addr = (rt_ubase_t)ptr; + rt_ubase_t start = (rt_ubase_t)&__percpu_start; + rt_ubase_t end = (rt_ubase_t)&__percpu_end; + + return addr >= start && addr + sizeof(*ptr) <= end; +} + +static rt_ubase_t *percpu_hartid_ptr(void) +{ + rt_ubase_t *ptr = percpu_hartid; + + if (!percpu_hartid_ptr_valid(ptr)) + { + ptr = (rt_ubase_t *)&__percpu_hartid; + } + + return percpu_hartid_ptr_valid(ptr) ? ptr : RT_NULL; +} + +void percpu_hartid_init(rt_ubase_t *percpu_ptr, rt_ubase_t hartid) +{ + rt_ubase_t *percpu_hartid_paddr; + int cpu_id = riscv_hartid_to_cpu_id(hartid); + rt_size_t percpu_size = (rt_size_t)(&__percpu_end - &__percpu_start); + + RT_ASSERT(cpu_id >= 0); + RT_ASSERT(cpu_id < RT_CPUS_NR); + if (cpu_id < 0 || cpu_id >= RT_CPUS_NR) + { + return; + } + + percpu_hartid_paddr = percpu_ptr; + percpu_hartid = (void *)((rt_ubase_t)percpu_ptr - rt_kmem_pvoff()); + + /* Save to the real area */ + *(rt_ubase_t *)((void *)percpu_hartid_paddr + cpu_id * percpu_size) = hartid; +} + +/* This is the current CPU stack, but it is not used in this call. */ +void percpu_write(rt_ubase_t offset, const void *data, rt_size_t size) +{ + rt_ubase_t *ptr = percpu_hartid_ptr(); + + if (ptr) + { + rt_memcpy((void *)((rt_ubase_t)(ptr + 1) + offset), data, size); + } +} + +void percpu_read(rt_ubase_t offset, void *data, rt_size_t size) +{ + rt_ubase_t *ptr = percpu_hartid_ptr(); + + if (ptr) + { + rt_memcpy(data, (void *)((rt_ubase_t)(ptr + 1) + offset), size); + } +} +#endif /* RT_USING_SMP && ARCH_MM_MMU */ + +rt_ubase_t riscv_cpu_id_to_hartid(int cpu_id) +{ + RT_ASSERT(cpu_id >= 0); + RT_ASSERT(cpu_id < RT_CPUS_NR); + RT_ASSERT(cpu_id_to_hartid_map[cpu_id] != RISCV_HARTID_INVALID); + + return cpu_id_to_hartid_map[cpu_id]; +} + +rt_bool_t riscv_cpu_id_available(int cpu_id) +{ + if (cpu_id < 0 || cpu_id >= RT_CPUS_NR) + { + return RT_FALSE; + } + + return cpu_id_to_hartid_map[cpu_id] != RISCV_HARTID_INVALID; +} + +int riscv_hartid_to_cpu_id(rt_ubase_t hartid) +{ + int cpu_id; + + if (hartid_equ_cpuid && hartid < (rt_ubase_t)mapped_cpus_nr) + { + return hartid; + } + + for (cpu_id = 0; cpu_id < mapped_cpus_nr; ++cpu_id) + { + if (hartid == cpu_id_to_hartid_map[cpu_id]) + { + return cpu_id; + } + } + + return -1; +} + +int riscv_hartid_register(rt_ubase_t hartid) +{ + int cpu_id = riscv_hartid_to_cpu_id(hartid); + + if (cpu_id >= 0) + { + return cpu_id; + } + + if (mapped_cpus_nr >= RT_CPUS_NR) + { + RT_ASSERT(0); + return -RT_EFULL; + } + + cpu_id = mapped_cpus_nr; + cpu_id_to_hartid_map[cpu_id] = hartid; + rt_hw_wmb(); + mapped_cpus_nr = cpu_id + 1; + riscv_hartid_map_refresh_fastpath(); + rt_hw_wmb(); + + return cpu_id; +} + +void riscv_cpu_id_set(int cpu_id) +{ +#ifdef RT_USING_SMP + RT_ASSERT(cpu_id >= 0 && cpu_id < RT_CPUS_NR); + if (cpu_id < 0 || cpu_id >= RT_CPUS_NR) + { + return; + } + + __asm__ volatile ("mv tp, %0" :: "r"(cpu_id) : "memory"); +#else + RT_UNUSED(cpu_id); +#endif +} + +#if defined(RT_USING_SMP) && !defined(RT_USING_DM) +rt_weak rt_size_t riscv_hartid_list_get(rt_ubase_t *hartids, rt_size_t max_harts) +{ + rt_size_t hart_count = max_harts; + + if (hart_count > RT_CPUS_NR) + { + hart_count = RT_CPUS_NR; + } + + for (rt_size_t i = 0; i < hart_count; ++i) + { + hartids[i] = i; + } + + return hart_count; +} +#endif + +void riscv_boot_hart_init(rt_ubase_t hartid) +{ +#if defined(RT_USING_SMP) && !defined(RT_USING_DM) + rt_ubase_t hartids[RT_CPUS_NR]; + rt_size_t hart_count = riscv_hartid_list_get(hartids, RT_CPUS_NR); +#endif + + boot_cpu_hartid = hartid; + riscv_hartid_map_clear(); + riscv_hartid_register(hartid); + riscv_cpu_id_set(0); + +#if defined(RT_USING_SMP) && !defined(RT_USING_DM) + RT_ASSERT(hart_count <= RT_CPUS_NR); + if (hart_count > RT_CPUS_NR) + { + hart_count = RT_CPUS_NR; + } + + for (rt_size_t i = 0; i < hart_count; ++i) + { + if (hartids[i] != hartid) + { + riscv_hartid_register(hartids[i]); + } + } +#endif +} + +int rt_hw_cpu_id(void) +{ +#ifdef RT_USING_SMP + int cpu_id; + +#if !defined(ARCH_RISCV_M_MODE) && defined(ARCH_MM_MMU) + if (ARCH_VADDR_WIDTH) + { + rt_ubase_t *ptr = percpu_hartid_ptr(); + + if (ptr) + { + cpu_id = riscv_hartid_to_cpu_id(*ptr); + if (cpu_id >= 0 && cpu_id < RT_CPUS_NR) + { + return cpu_id; + } + } + } +#endif /* !ARCH_RISCV_M_MODE && ARCH_MM_MMU */ + { + __asm__ volatile ("mv %0, tp":"=r"(cpu_id)); + + if (cpu_id >= 0 && cpu_id < RT_CPUS_NR) + { + return cpu_id; + } + } + + return 0; +#else + return 0; +#endif +} + +const char *rt_hw_cpu_arch(void) +{ +#if __riscv_xlen == 128 + return "riscv128"; +#elif __riscv_xlen == 64 + return "riscv64"; +#elif __riscv_xlen == 32 + return "riscv32"; +#else +#error "Unexpected __riscv_xlen" +#endif +} diff --git a/libcpu/risc-v/common/cpu_gcc.S b/libcpu/risc-v/common/cpu_gcc.S new file mode 100644 index 0000000000..1659e0ed2b --- /dev/null +++ b/libcpu/risc-v/common/cpu_gcc.S @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2025-04-20 GuEe-GUI The first version + */ + +#include <csr.h> + +#ifdef RT_USING_SMP +#define rt_hw_interrupt_disable rt_hw_local_irq_disable +#define rt_hw_interrupt_enable rt_hw_local_irq_enable +#endif + + .section ".text","ax" + +START_POINT(rt_hw_interrupt_is_disabled) + csrr a0, CSR_STATUS + andi a0, a0, SR_IE + seqz a0, a0 + ret +START_POINT_END(rt_hw_interrupt_is_disabled) + +START_POINT(rt_hw_interrupt_enable) + /* Restore to old csr */ + csrs CSR_STATUS, a0 + ret +START_POINT_END(rt_hw_interrupt_enable) + +START_POINT(rt_hw_interrupt_disable) + /* Clear IE */ + csrrci a0, CSR_STATUS, SR_IE + ret +START_POINT_END(rt_hw_interrupt_disable) diff --git a/libcpu/risc-v/common/cpu_sbi.c b/libcpu/risc-v/common/cpu_sbi.c new file mode 100644 index 0000000000..4ad4077387 --- /dev/null +++ b/libcpu/risc-v/common/cpu_sbi.c @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2025-04-20 GuEe-GUI The first version + */ + +#include <sbi.h> +#include <cpu.h> +#include <cpuport.h> + +static int sbi_cpu_boot(rt_uint32_t cpuid, rt_uint64_t entry) +{ + rt_ubase_t hartid; + + if (!riscv_cpu_id_available(cpuid)) + { + return -RT_EINVAL; + } + + hartid = riscv_cpu_id_to_hartid(cpuid); + + return sbi_hsm_hart_start(hartid, entry, entry); +} + +static void sbi_cpu_shutdown(void) +{ + sbi_hsm_hart_stop(); +} + +struct cpu_ops_t cpu_sbi_ops = +{ + .method = "sbi", + .cpu_boot = sbi_cpu_boot, + .cpu_shutdown = sbi_cpu_shutdown, +}; diff --git a/libcpu/risc-v/common/cpu_spinwait.c b/libcpu/risc-v/common/cpu_spinwait.c new file mode 100644 index 0000000000..628046ccde --- /dev/null +++ b/libcpu/risc-v/common/cpu_spinwait.c @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2025-04-20 GuEe-GUI The first version + */ + +#include <rthw.h> + +#include <cpu.h> + +rt_uint8_t riscv_spinwait_table = 0; + +static int spinwait_cpu_boot(rt_uint32_t cpuid, rt_uint64_t entry) +{ + rt_hw_barrier(); + + HWREG8(&riscv_spinwait_table) = RT_UINT8_MAX; + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, &riscv_spinwait_table, sizeof(riscv_spinwait_table)); + + return 0; +} + +struct cpu_ops_t cpu_spinwait_ops = +{ + .method = "spinwait", + .cpu_boot = spinwait_cpu_boot, +}; diff --git a/libcpu/risc-v/common/cpuport.c b/libcpu/risc-v/common/cpuport.c index 502a8fcbf3..d06134cf57 100644 --- a/libcpu/risc-v/common/cpuport.c +++ b/libcpu/risc-v/common/cpuport.c @@ -1,99 +1,120 @@ /* - * Copyright (c) 2006-2023, RT-Thread Development Team + * Copyright (c) 2006-2025, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * * Change Logs: * Date Author Notes - * 2018/10/28 Bernard The unify RISC-V porting code. - * 2020/11/20 BalanceTWK Add FPU support - * 2023/01/04 WangShun Adapt to CH32 + * 2018-10-28 Bernard The unify RISC-V porting code. + * 2025-04-20 GuEe-GUI Port ffsN and clz, spinlock */ #include <rthw.h> #include <rtthread.h> -#include "cpuport.h" -#include "rt_hw_stack_frame.h" +#include <asm-generic.h> -#ifndef RT_USING_SMP -volatile rt_ubase_t rt_interrupt_from_thread = 0; -volatile rt_ubase_t rt_interrupt_to_thread = 0; -volatile rt_uint32_t rt_thread_switch_interrupt_flag = 0; -#endif +#ifdef RT_USING_CPU_FFS -/** - * This function will initialize thread stack - * - * @param tentry the entry of thread - * @param parameter the parameter of entry - * @param stack_addr the beginning stack address - * @param texit the function will be called when thread exit - * - * @return stack address - */ -rt_uint8_t *rt_hw_stack_init(void *tentry, - void *parameter, - rt_uint8_t *stack_addr, - void *texit) +int __rt_ffs(int value) { - struct rt_hw_stack_frame *frame; - rt_uint8_t *stk; - int i; - - stk = stack_addr + sizeof(rt_ubase_t); - stk = (rt_uint8_t *)RT_ALIGN_DOWN((rt_ubase_t)stk, REGBYTES); - stk -= sizeof(struct rt_hw_stack_frame); - - frame = (struct rt_hw_stack_frame *)stk; - - for (i = 0; i < sizeof(struct rt_hw_stack_frame) / sizeof(rt_ubase_t); i++) - { - ((rt_ubase_t *)frame)[i] = 0xdeadbeef; - } +#ifdef __GNUC__ + return __builtin_ffs(value); +#else +#error Unsupport ffs +#endif +} - frame->ra = (rt_ubase_t)texit; - frame->a0 = (rt_ubase_t)parameter; - frame->epc = (rt_ubase_t)tentry; +unsigned long __rt_ffsl(unsigned long value) +{ +#ifdef __GNUC__ + return __builtin_ffsl(value); +#else +#error Unsupport ffsl +#endif +} - /* force to machine mode(MPP=11) and set MPIE to 1 */ -#ifdef ARCH_RISCV_FPU - frame->mstatus = 0x7880; +unsigned long __rt_clz(unsigned long value) +{ +#ifdef __GNUC__ + return __builtin_clzl(value); #else - frame->mstatus = 0x1880; +#error Unsupport clz #endif +} +#endif /* RT_USING_CPU_FFS */ + +#ifdef RT_USING_SMART +#if defined(RT_USING_SMP) && RT_CPUS_NR > 1 +extern void percpu_write(rt_ubase_t offset, const void *data, rt_size_t size); - return stk; +void rt_hw_set_process_id(int pid) +{ + percpu_write(0, &pid, sizeof(pid)); } +#else +static int process_id; -rt_weak void rt_trigger_software_interrupt(void) +void rt_hw_set_process_id(int pid) { - while (0); + process_id = pid; } +#endif /* RT_USING_SMP && RT_CPUS_NR > 1 */ +#endif /* RT_USING_SMART */ -rt_weak void rt_hw_do_after_save_above(void) +#ifdef RT_USING_SMP +void rt_hw_spin_lock_init(rt_hw_spinlock_t *lock) { - while (1); + lock->slock = 0; } -/* - * #ifdef RT_USING_SMP - * void rt_hw_context_switch_interrupt(void *context, rt_ubase_t from, rt_ubase_t to, struct rt_thread *to_thread); - * #else - * void rt_hw_context_switch_interrupt(rt_ubase_t from, rt_ubase_t to); - * #endif - */ -#ifndef RT_USING_SMP -rt_weak void rt_hw_context_switch_interrupt(rt_ubase_t from, rt_ubase_t to, rt_thread_t from_thread, rt_thread_t to_thread) +void rt_hw_spin_lock(rt_hw_spinlock_t *lock) { - if (rt_thread_switch_interrupt_flag == 0) - rt_interrupt_from_thread = from; + unsigned long tmp, new_val, mask; + unsigned short ticket; - rt_interrupt_to_thread = to; - rt_thread_switch_interrupt_flag = 1; + __asm__ volatile ( + "1:\n\t" + " lr." REG_WIDTH ".aq %0, (%3)\n\t" + " srli %1, %0, 16\n\t" + " addi %1, %1, 1\n\t" + " slli %1, %1, 16\n\t" + " li %2, 0xffff\n\t" + " and %2, %0, %2\n\t" + " or %1, %1, %2\n\t" + " sc." REG_WIDTH ".rl %1, %1, (%3)\n\t" + " bnez %1, 1b" + : "=&r" (tmp), "=&r" (new_val), "=&r" (mask) + : "r" (&lock->slock) + : "memory"); + + ticket = (tmp >> 16) & 0xffff; + + while (HWREG16(&lock->tickets.owner) != ticket) + { + __asm__ volatile (OPC_PAUSE_I); + } +} + +void rt_hw_spin_unlock(rt_hw_spinlock_t *lock) +{ + unsigned long tmp, new_val, mask; - rt_trigger_software_interrupt(); + __asm__ volatile ( + "1:\n\t" + " lr." REG_WIDTH ".aq %0, (%3)\n\t" + " addi %1, %0, 1\n\t" + " li %2, 0xffff\n\t" + " and %1, %1, %2\n\t" + " slli %2, %2, 16\n\t" + " and %0, %0, %2\n\t" + " or %1, %0, %1\n\t" + " sc." REG_WIDTH ".rl %1, %1, (%3)\n\t" + " bnez %1, 1b" + : "=&r" (tmp), "=&r" (new_val), "=&r" (mask) + : "r" (&lock->slock) + : "memory"); - return ; + rt_hw_barrier(); } -#endif /* end of RT_USING_SMP */ +#endif /* RT_USING_SMP */ diff --git a/libcpu/risc-v/common/hwcache/Kconfig b/libcpu/risc-v/common/hwcache/Kconfig new file mode 100644 index 0000000000..918b625343 --- /dev/null +++ b/libcpu/risc-v/common/hwcache/Kconfig @@ -0,0 +1,3 @@ +config RT_CACHE_SIFIVE_CCACHE + bool "Sifive Composable Cache controller" + depends on RT_USING_HWCACHE diff --git a/libcpu/risc-v/common/hwcache/SConscript b/libcpu/risc-v/common/hwcache/SConscript new file mode 100644 index 0000000000..d7b5123ca5 --- /dev/null +++ b/libcpu/risc-v/common/hwcache/SConscript @@ -0,0 +1,17 @@ +from building import * + +group = [] + +if not GetDepend(['RT_USING_HWCACHE']): + Return('group') + +src = [] +cwd = GetCurrentDir() +CPPPATH = [cwd] + +if GetDepend(['RT_CACHE_SIFIVE_CCACHE']): + src += ['hwcache-sifive_ccache.c'] + +group = DefineGroup('DeviceDrivers', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/libcpu/risc-v/common/hwcache/hwcache-sifive_ccache.c b/libcpu/risc-v/common/hwcache/hwcache-sifive_ccache.c new file mode 100644 index 0000000000..5ede267686 --- /dev/null +++ b/libcpu/risc-v/common/hwcache/hwcache-sifive_ccache.c @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-02-25 GuEe-GUI the first version + */ + +#include <rthw.h> +#include <rtthread.h> +#include <rtdevice.h> +#include <drivers/misc.h> + +#define DBG_TAG "cache.sifive_ccache" +#define DBG_LVL DBG_INFO +#include <rtdbg.h> + +#ifdef ARCH_MM_MMU +#include <mmu.h> +#endif + +#define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100 +#define SIFIVE_CCACHE_DIRECCFIX_HIGH 0x104 +#define SIFIVE_CCACHE_DIRECCFIX_COUNT 0x108 + +#define SIFIVE_CCACHE_DIRECCFAIL_LOW 0x120 +#define SIFIVE_CCACHE_DIRECCFAIL_HIGH 0x124 +#define SIFIVE_CCACHE_DIRECCFAIL_COUNT 0x128 + +#define SIFIVE_CCACHE_DATECCFIX_LOW 0x140 +#define SIFIVE_CCACHE_DATECCFIX_HIGH 0x144 +#define SIFIVE_CCACHE_DATECCFIX_COUNT 0x148 + +#define SIFIVE_CCACHE_DATECCFAIL_LOW 0x160 +#define SIFIVE_CCACHE_DATECCFAIL_HIGH 0x164 +#define SIFIVE_CCACHE_DATECCFAIL_COUNT 0x168 + +#define SIFIVE_CCACHE_CONFIG 0x00 +#define SIFIVE_CCACHE_CONFIG_BANK_MASK RT_GENMASK_ULL(7, 0) +#define SIFIVE_CCACHE_CONFIG_WAYS_MASK RT_GENMASK_ULL(15, 8) +#define SIFIVE_CCACHE_CONFIG_SETS_MASK RT_GENMASK_ULL(23, 16) +#define SIFIVE_CCACHE_CONFIG_BLKS_MASK RT_GENMASK_ULL(31, 24) + +#define SIFIVE_CCACHE_FLUSH64 0x200 +#define SIFIVE_CCACHE_FLUSH32 0x240 + +#define SIFIVE_CCACHE_WAYENABLE 0x08 +#define SIFIVE_CCACHE_ECCINJECTERR 0x40 + +#define SIFIVE_CCACHE_MAX_ECCINTR 4 +#define SIFIVE_CCACHE_LINE_SIZE 64 + +#define SIFIVE_CCACHE_TRUNKCLOCKGATE 0x1000 +#define SIFIVE_CCACHE_TRUNKCLOCKGATE_DISABLE RT_BIT(0) +#define SIFIVE_CCACHE_REGIONCLOCKGATE_DISABLE RT_BIT(1) + +enum +{ + DIR_CORR = 0, + DATA_CORR, + DATA_UNCORR, + DIR_UNCORR, +}; + +enum +{ + QUIRK_BROKEN_DATA_UNCORR = RT_BIT(0), + QUIRK_HAS_CG = RT_BIT(1), +}; + +static void *ccache_base; +static rt_ubase_t ccache_quirk; +static rt_uint32_t ccache_level; +static int ccache_irq[SIFIVE_CCACHE_MAX_ECCINTR]; +static struct rt_ofw_node *ccache_np; + +static void ccache_enable(void) +{ + rt_uint32_t config, ways; + + /* Enable all ways of composable cache */ + config = HWREG32(ccache_base + SIFIVE_CCACHE_CONFIG); + ways = RT_FIELD_GET(SIFIVE_CCACHE_CONFIG_WAYS_MASK, config); + + HWREG32(ccache_base + SIFIVE_CCACHE_WAYENABLE) = ways - 1; + + if (ccache_quirk & QUIRK_HAS_CG) + { + /* Enable clock gating bits */ + config = HWREG32(ccache_base + SIFIVE_CCACHE_TRUNKCLOCKGATE); + config &= ~(SIFIVE_CCACHE_TRUNKCLOCKGATE_DISABLE | SIFIVE_CCACHE_REGIONCLOCKGATE_DISABLE); + HWREG32(ccache_base + SIFIVE_CCACHE_TRUNKCLOCKGATE) = config; + } +} + +static void ccache_disable(void) +{ + rt_uint32_t config; + + HWREG32(ccache_base + SIFIVE_CCACHE_WAYENABLE) = 0; + + if (ccache_quirk & QUIRK_HAS_CG) + { + /* Disable clock gating bits */ + config = HWREG32(ccache_base + SIFIVE_CCACHE_TRUNKCLOCKGATE); + config |= SIFIVE_CCACHE_TRUNKCLOCKGATE_DISABLE | SIFIVE_CCACHE_REGIONCLOCKGATE_DISABLE; + HWREG32(ccache_base + SIFIVE_CCACHE_TRUNKCLOCKGATE) = config; + } +} + +static void ccache_flush_range(void *vaddr, rt_size_t size) +{ +#ifdef ARCH_MM_MMU + rt_ubase_t start = (rt_ubase_t)rt_kmem_v2p(vaddr); +#else + rt_ubase_t start = (rt_ubase_t)vaddr; +#endif + rt_ubase_t end = start + size; + + if (!size) + { + return; + } + + rt_hw_barrier(); + + for (rt_ubase_t line = RT_ALIGN_DOWN(start, SIFIVE_CCACHE_LINE_SIZE); + line < end; + line += SIFIVE_CCACHE_LINE_SIZE) + { +#if __riscv_xlen == 64 + HWREG64(ccache_base + SIFIVE_CCACHE_FLUSH64) = line; +#else + HWREG32(ccache_base + SIFIVE_CCACHE_FLUSH32) = line >> 4; +#endif + rt_hw_barrier(); + } +} + +static struct rt_hwcache_ops ccache_mgmt = +{ + .name = "ccache", + .enable = ccache_enable, + .disable = ccache_disable, + .flush = ccache_flush_range, + .invalidate = ccache_flush_range, +}; + +static void sifive_ccache_isr(int irq, void *param) +{ + rt_uint32_t add_h, add_l; + + if (irq == ccache_irq[DIR_CORR]) + { + add_h = HWREG32(ccache_base + SIFIVE_CCACHE_DIRECCFIX_HIGH); + add_l = HWREG32(ccache_base + SIFIVE_CCACHE_DIRECCFIX_LOW); + LOG_E("Dir error: 0x%08x%08x", add_h, add_l); + + /* Reading this register clears the DirError interrupt sig */ + HWREG32(ccache_base + SIFIVE_CCACHE_DIRECCFIX_COUNT); + } + else if (irq == ccache_irq[DIR_UNCORR]) + { + add_h = HWREG32(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_HIGH); + add_l = HWREG32(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_LOW); + + /* Reading this register clears the DirFail interrupt sig */ + HWREG32(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_COUNT); + LOG_E("Dir fail: 0x%08x%08x", add_h, add_l); + RT_ASSERT(0); + } + else if (irq == ccache_irq[DATA_CORR]) + { + add_h = HWREG32(ccache_base + SIFIVE_CCACHE_DATECCFIX_HIGH); + add_l = HWREG32(ccache_base + SIFIVE_CCACHE_DATECCFIX_LOW); + LOG_E("Data error: 0x%08x.%08x", add_h, add_l); + + /* Reading this register clears the DataError interrupt sig */ + HWREG32(ccache_base + SIFIVE_CCACHE_DATECCFIX_COUNT); + } + else if (irq == ccache_irq[DATA_UNCORR]) + { + add_h = HWREG32(ccache_base + SIFIVE_CCACHE_DATECCFAIL_HIGH); + add_l = HWREG32(ccache_base + SIFIVE_CCACHE_DATECCFAIL_LOW); + LOG_E("Data fail: 0x%08x%08x", add_h, add_l); + + /* Reading this register clears the DataFail interrupt sig */ + HWREG32(ccache_base + SIFIVE_CCACHE_DATECCFAIL_COUNT); + } +} + +static void ccache_irq_init(void) +{ + if (!ccache_np) + { + return; + } + + for (int i = 0; i < RT_ARRAY_SIZE(ccache_irq); ++i) + { + int irq; + + if (i == DATA_UNCORR && (ccache_quirk & QUIRK_BROKEN_DATA_UNCORR)) + { + continue; + } + + irq = rt_ofw_get_irq(ccache_np, i); + + if (irq >= 0) + { + rt_hw_interrupt_install(irq, sifive_ccache_isr, ccache_np, "ccache"); + rt_hw_interrupt_umask(irq); + + ccache_irq[i] = irq; + } + } +} +INIT_SUBSYS_EXPORT(ccache_irq_init); + +static rt_err_t sifive_ccache_ofw_init(struct rt_ofw_node *np, + const struct rt_ofw_node_id *id) +{ + ccache_quirk = *(rt_ubase_t *)id->data; + + ccache_base = rt_ofw_iomap(np, 0); + + if (!ccache_base) + { + return -RT_EIO; + } + + ccache_np = np; + + if (!rt_dm_cpu_dcache_ops) + { + rt_dm_cpu_dcache_ops = &ccache_mgmt; + } + + return RT_EOK; +} + +static rt_ubase_t ccache0_quirk = QUIRK_HAS_CG; +static rt_ubase_t jh7100_quirk = QUIRK_BROKEN_DATA_UNCORR; + +static const struct rt_ofw_node_id sifive_ccache_ofw_ids[] = +{ + { .compatible = "sifive,fu540-c000-ccache" }, + { .compatible = "sifive,fu740-c000-ccache" }, + { .compatible = "starfive,jh7100-ccache", .data = &jh7100_quirk }, + { .compatible = "sifive,ccache0", .data = &ccache0_quirk }, + { /* sentinel */ } +}; +RT_HWCACHE_OFW_DECLARE(sifive_ccache, sifive_ccache_ofw_ids, sifive_ccache_ofw_init); diff --git a/libcpu/risc-v/common/include/asm-generic.h b/libcpu/risc-v/common/include/asm-generic.h new file mode 100644 index 0000000000..6f29a6034d --- /dev/null +++ b/libcpu/risc-v/common/include/asm-generic.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2006-2025 RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-03-12 WangXiaoyao the first version + * 2025-04-20 GuEe-GUI Merge 64 and 32 mode + */ + +#ifndef __ASM_GENERIC_H__ +#define __ASM_GENERIC_H__ + +#if defined(__ASSEMBLER__) && !defined(__ASSEMBLY__) +/* For XuanTie GCC */ +#define __ASSEMBLY__ +#endif + +#include <rtconfig.h> + +#ifdef __ASSEMBLY__ +/* use to mark a start point where every task start from */ +#define START_POINT(funcname) \ + .global funcname; \ + .type funcname, %function; \ + funcname: \ + .cfi_sections .debug_frame, .eh_frame; \ + .cfi_startproc; \ + .cfi_undefined ra + +#define START_POINT_END(name) \ + .cfi_endproc; \ + .size name, .-name; + +#define _AC(X,Y) X +#define _AT(T,X) X +#define __ASM_STR(x) x +#else +#define _AC(X,Y) (X##Y) +#define _AT(T,X) ((T)(X)) +#define __ASM_STR(x) #x +#endif + +#define _UL(x) (_AC(x, UL)) +#define _ULL(x) (_AC(x, ULL)) + +#if __riscv_xlen == 64 +#define BITS_PER_LONG 64 +#define __REG_SEL(a, b) __ASM_STR(a) +#elif __riscv_xlen == 32 +#define BITS_PER_LONG 32 +#define __REG_SEL(a, b) __ASM_STR(b) +#else +#error "Unexpected __riscv_xlen" +#endif + +#ifndef RT_BITS_PER_LONG +#define RT_BITS_PER_LONG BITS_PER_LONG +#endif + +#ifndef BIT +#define BIT(x) (_UL(1) << (x)) +#endif + +#ifndef GENMASK +#define GENMASK(h, l) (((_UL(~0)) << (l)) & (_UL(~0) >> (BITS_PER_LONG - 1 - (h)))) +#endif + +#define REG_WIDTH __REG_SEL(d, w) +#define REG_L __REG_SEL(ld, lw) +#define REG_S __REG_SEL(sd, sw) +#define REG_SC __REG_SEL(sc.d, sc.w) +#define REG_AMOSWAP_AQ __REG_SEL(amoswap.d.aq, amoswap.w.aq) +#define REG_ASM __REG_SEL(.dword, .word) +#define SZREG __REG_SEL(8, 4) +#define LGREG __REG_SEL(3, 2) +#define FREG_L __REG_SEL(fld, flw) +#define FREG_S __REG_SEL(fsd, fsw) + +#if __riscv_xlen == 64 +#ifdef __ASSEMBLY__ +#define RISCV_PTR .dword +#define RISCV_SZPTR 8 +#define RISCV_LGPTR 3 +#else +#define RISCV_PTR ".dword" +#define RISCV_SZPTR "8" +#define RISCV_LGPTR "3" +#endif +#elif __riscv_xlen == 32 +#ifdef __ASSEMBLY__ +#define RISCV_PTR .word +#define RISCV_SZPTR 4 +#define RISCV_LGPTR 2 +#else +#define RISCV_PTR ".word" +#define RISCV_SZPTR "4" +#define RISCV_LGPTR "2" +#endif +#else +#error "Unexpected __riscv_xlen" +#endif + +#endif /* __ASM_GENERIC_H__ */ diff --git a/libcpu/risc-v/common/include/asm.h b/libcpu/risc-v/common/include/asm.h new file mode 100644 index 0000000000..500247a57e --- /dev/null +++ b/libcpu/risc-v/common/include/asm.h @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2006-2025 RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2021-02-02 lizhirui first version + * 2021-02-11 lizhirui fixed gp save/store bug + * 2021-11-18 JasonHu add fpu registers save/restore + * 2022-10-22 Shell Support kernel mode RVV + * 2025-04-20 GuEe-GUI Merge 64I and 32I/E + */ + +#ifndef __ASM_H__ +#define __ASM_H__ + +#include <rtconfig.h> + +#include <csr.h> +#include <stack.h> + +#ifdef __GNUC__ +.macro LOAD_GLOBAL_POINTER +.option push +.option norelax + la gp, __global_pointer$ +.option pop +.endm + +.macro XRET +#ifdef ARCH_RISCV_M_MODE + mret +#else + sret +#endif +.endm + +.macro PUSH reg + addi sp, sp, -SZREG + REG_S \reg, 0(sp) +.endm + +.macro POP reg + REG_L \reg, 0(sp) + addi sp, sp, SZREG +.endm + +.macro OPEN_INTERRUPT + csrsi CSR_STATUS, 2 +.endm + +.macro CLOSE_INTERRUPT + csrci CSR_STATUS, 2 +.endm + +.macro SAVE_ALL +#ifdef ARCH_RISCV_FPU + /* Reserve float registers */ + addi sp, sp, -CTX_FPU_REG_NR * SZREG +#endif +#ifdef ARCH_RISCV_VECTOR + /* Reserve float registers */ + addi sp, sp, -CTX_VECTOR_REG_NR * SZREG +#endif + + /* Save general registers */ + addi sp, sp, -CTX_GENERAL_REG_NR * SZREG + REG_S x1, 1 * SZREG(sp) + + csrr x1, CSR_STATUS + REG_S x1, FRAME_OFF_XSTATUS(sp) + + csrr x1, CSR_CAUSE + REG_S x1, FRAME_OFF_XCAUSE(sp) + + csrr x1, CSR_EPC + REG_S x1, 0 * SZREG(sp) + + REG_S x3, 4 * SZREG(sp) + REG_S x4, 5 * SZREG(sp) + REG_S x5, 6 * SZREG(sp) + REG_S x6, 7 * SZREG(sp) + REG_S x7, 8 * SZREG(sp) + REG_S x8, 9 * SZREG(sp) + REG_S x9, 10 * SZREG(sp) + REG_S x10, 11 * SZREG(sp) + REG_S x11, 12 * SZREG(sp) + REG_S x12, 13 * SZREG(sp) + REG_S x13, 14 * SZREG(sp) + REG_S x14, 15 * SZREG(sp) + REG_S x15, 16 * SZREG(sp) +#ifndef ARCH_RISCV_32E + REG_S x16, 17 * SZREG(sp) + REG_S x17, 18 * SZREG(sp) + REG_S x18, 19 * SZREG(sp) + REG_S x19, 20 * SZREG(sp) + REG_S x20, 21 * SZREG(sp) + REG_S x21, 22 * SZREG(sp) + REG_S x22, 23 * SZREG(sp) + REG_S x23, 24 * SZREG(sp) + REG_S x24, 25 * SZREG(sp) + REG_S x25, 26 * SZREG(sp) + REG_S x26, 27 * SZREG(sp) + REG_S x27, 28 * SZREG(sp) + REG_S x28, 29 * SZREG(sp) + REG_S x29, 30 * SZREG(sp) + REG_S x30, 31 * SZREG(sp) + REG_S x31, 32 * SZREG(sp) +#endif /* !ARCH_RISCV_32E */ + csrr t0, CSR_SCRATCH + REG_S t0, FRAME_OFF_SP(sp) + +#ifdef ARCH_RISCV_FPU + /* Backup sp and adjust sp to save float registers */ + mv t1, sp + addi t1, t1, CTX_GENERAL_REG_NR * SZREG + + li t0, SR_FS + csrs CSR_STATUS, t0 + CTX_FREG_S f0, FPU_CTX_F0_OFF(t1) + CTX_FREG_S f1, FPU_CTX_F1_OFF(t1) + CTX_FREG_S f2, FPU_CTX_F2_OFF(t1) + CTX_FREG_S f3, FPU_CTX_F3_OFF(t1) + CTX_FREG_S f4, FPU_CTX_F4_OFF(t1) + CTX_FREG_S f5, FPU_CTX_F5_OFF(t1) + CTX_FREG_S f6, FPU_CTX_F6_OFF(t1) + CTX_FREG_S f7, FPU_CTX_F7_OFF(t1) + CTX_FREG_S f8, FPU_CTX_F8_OFF(t1) + CTX_FREG_S f9, FPU_CTX_F9_OFF(t1) + CTX_FREG_S f10, FPU_CTX_F10_OFF(t1) + CTX_FREG_S f11, FPU_CTX_F11_OFF(t1) + CTX_FREG_S f12, FPU_CTX_F12_OFF(t1) + CTX_FREG_S f13, FPU_CTX_F13_OFF(t1) + CTX_FREG_S f14, FPU_CTX_F14_OFF(t1) + CTX_FREG_S f15, FPU_CTX_F15_OFF(t1) + CTX_FREG_S f16, FPU_CTX_F16_OFF(t1) + CTX_FREG_S f17, FPU_CTX_F17_OFF(t1) + CTX_FREG_S f18, FPU_CTX_F18_OFF(t1) + CTX_FREG_S f19, FPU_CTX_F19_OFF(t1) + CTX_FREG_S f20, FPU_CTX_F20_OFF(t1) + CTX_FREG_S f21, FPU_CTX_F21_OFF(t1) + CTX_FREG_S f22, FPU_CTX_F22_OFF(t1) + CTX_FREG_S f23, FPU_CTX_F23_OFF(t1) + CTX_FREG_S f24, FPU_CTX_F24_OFF(t1) + CTX_FREG_S f25, FPU_CTX_F25_OFF(t1) + CTX_FREG_S f26, FPU_CTX_F26_OFF(t1) + CTX_FREG_S f27, FPU_CTX_F27_OFF(t1) + CTX_FREG_S f28, FPU_CTX_F28_OFF(t1) + CTX_FREG_S f29, FPU_CTX_F29_OFF(t1) + CTX_FREG_S f30, FPU_CTX_F30_OFF(t1) + CTX_FREG_S f31, FPU_CTX_F31_OFF(t1) + + /* Clean FS domain */ + csrc CSR_STATUS, t0 + + /* Clean status would clr sr_sd; */ + li t0, SR_FS_CLEAN + csrs CSR_STATUS, t0 + +#endif /* ARCH_RISCV_FPU */ + +#ifdef ARCH_RISCV_VECTOR + csrr t0, CSR_STATUS + andi t0, t0, SR_VS + beqz t0, 0f + + /* push vector frame */ + addi t1, sp, (CTX_GENERAL_REG_NR + CTX_FPU_REG_NR) * SZREG + + SAVE_VECTOR t1 +0: +#endif /* ARCH_RISCV_VECTOR */ +.endm + +.macro RESTORE_TP offset +#if !defined(RT_USING_SMP) + REG_L tp, \offset(sp) +#elif defined(ARCH_MM_MMU) && !defined(ARCH_RISCV_M_MODE) + /* Bare SMP uses tp as a hart-local CPU ID, including interrupt returns. */ + la t0, rt_hw_arch_vaddr_width + lbu t0, 0(t0) + beqz t0, .Lkeep_tp\@ + REG_L tp, \offset(sp) +.Lkeep_tp\@: +#endif +.endm + +.macro RESTORE_ALL +#ifdef ARCH_RISCV_VECTOR + /* Skip on close */ + REG_L t0, FRAME_OFF_XSTATUS(sp) + /* Cannot use vector on initial */ + andi t0, t0, SR_VS_CLEAN + beqz t0, 0f + + /* Restore vector frame */ + addi t1, sp, (CTX_GENERAL_REG_NR + CTX_FPU_REG_NR) * SZREG + + RESTORE_VECTOR t1 +0: +#endif /* ARCH_RISCV_VECTOR */ + +#ifdef ARCH_RISCV_FPU + /* Restore float register */ + addi t2, sp, CTX_GENERAL_REG_NR * SZREG + + li t0, SR_FS + csrs CSR_STATUS, t0 + CTX_FREG_L f0, FPU_CTX_F0_OFF(t2) + CTX_FREG_L f1, FPU_CTX_F1_OFF(t2) + CTX_FREG_L f2, FPU_CTX_F2_OFF(t2) + CTX_FREG_L f3, FPU_CTX_F3_OFF(t2) + CTX_FREG_L f4, FPU_CTX_F4_OFF(t2) + CTX_FREG_L f5, FPU_CTX_F5_OFF(t2) + CTX_FREG_L f6, FPU_CTX_F6_OFF(t2) + CTX_FREG_L f7, FPU_CTX_F7_OFF(t2) + CTX_FREG_L f8, FPU_CTX_F8_OFF(t2) + CTX_FREG_L f9, FPU_CTX_F9_OFF(t2) + CTX_FREG_L f10, FPU_CTX_F10_OFF(t2) + CTX_FREG_L f11, FPU_CTX_F11_OFF(t2) + CTX_FREG_L f12, FPU_CTX_F12_OFF(t2) + CTX_FREG_L f13, FPU_CTX_F13_OFF(t2) + CTX_FREG_L f14, FPU_CTX_F14_OFF(t2) + CTX_FREG_L f15, FPU_CTX_F15_OFF(t2) + CTX_FREG_L f16, FPU_CTX_F16_OFF(t2) + CTX_FREG_L f17, FPU_CTX_F17_OFF(t2) + CTX_FREG_L f18, FPU_CTX_F18_OFF(t2) + CTX_FREG_L f19, FPU_CTX_F19_OFF(t2) + CTX_FREG_L f20, FPU_CTX_F20_OFF(t2) + CTX_FREG_L f21, FPU_CTX_F21_OFF(t2) + CTX_FREG_L f22, FPU_CTX_F22_OFF(t2) + CTX_FREG_L f23, FPU_CTX_F23_OFF(t2) + CTX_FREG_L f24, FPU_CTX_F24_OFF(t2) + CTX_FREG_L f25, FPU_CTX_F25_OFF(t2) + CTX_FREG_L f26, FPU_CTX_F26_OFF(t2) + CTX_FREG_L f27, FPU_CTX_F27_OFF(t2) + CTX_FREG_L f28, FPU_CTX_F28_OFF(t2) + CTX_FREG_L f29, FPU_CTX_F29_OFF(t2) + CTX_FREG_L f30, FPU_CTX_F30_OFF(t2) + CTX_FREG_L f31, FPU_CTX_F31_OFF(t2) + + /* Clean FS domain */ + csrc CSR_STATUS, t0 + + /* Clean status would clr sr_sd; */ + li t0, SR_FS_CLEAN + csrs CSR_STATUS, t0 + +#endif /* ARCH_RISCV_FPU */ + + /* Restore general register */ +#ifndef ARCH_RISCV_USING_IRQ_STACK + addi t0, sp, CTX_ALL_REG_NR * SZREG + csrw CSR_SCRATCH, t0 +#endif + + /* resw ra to CSR_EPC */ + REG_L x1, 0 * SZREG(sp) + csrw CSR_EPC, x1 + + REG_L x1, 2 * SZREG(sp) + csrw CSR_STATUS, x1 + + REG_L x1, 1 * SZREG(sp) + + REG_L x3, 4 * SZREG(sp) + RESTORE_TP (5 * SZREG) + REG_L x5, 6 * SZREG(sp) + REG_L x6, 7 * SZREG(sp) + REG_L x7, 8 * SZREG(sp) + REG_L x8, 9 * SZREG(sp) + REG_L x9, 10 * SZREG(sp) + REG_L x10, 11 * SZREG(sp) + REG_L x11, 12 * SZREG(sp) + REG_L x12, 13 * SZREG(sp) + REG_L x13, 14 * SZREG(sp) + REG_L x14, 15 * SZREG(sp) + REG_L x15, 16 * SZREG(sp) +#ifndef ARCH_RISCV_32E + REG_L x16, 17 * SZREG(sp) + REG_L x17, 18 * SZREG(sp) + REG_L x18, 19 * SZREG(sp) + REG_L x19, 20 * SZREG(sp) + REG_L x20, 21 * SZREG(sp) + REG_L x21, 22 * SZREG(sp) + REG_L x22, 23 * SZREG(sp) + REG_L x23, 24 * SZREG(sp) + REG_L x24, 25 * SZREG(sp) + REG_L x25, 26 * SZREG(sp) + REG_L x26, 27 * SZREG(sp) + REG_L x27, 28 * SZREG(sp) + REG_L x28, 29 * SZREG(sp) + REG_L x29, 30 * SZREG(sp) + REG_L x30, 31 * SZREG(sp) + REG_L x31, 32 * SZREG(sp) +#endif /* !ARCH_RISCV_32E */ + + /* Restore user sp */ + REG_L sp, FRAME_OFF_SP(sp) +.endm +#endif /* __GNUC__ */ + +#endif /* __ASM_H__ */ diff --git a/libcpu/risc-v/common/include/cache.h b/libcpu/risc-v/common/include/cache.h new file mode 100644 index 0000000000..a8224a9e59 --- /dev/null +++ b/libcpu/risc-v/common/include/cache.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2006-2025 RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2025-04-20 GuEe-GUI the first version + */ + +#ifndef __CACHE_H__ +#define __CACHE_H__ + +#define L1_CACHE_SHIFT 6 +#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) + +#ifndef __ASSEMBLY__ +#include <rtdef.h> + +void riscv_cache_set_cbom_block_size(rt_uint32_t size); +rt_uint32_t riscv_cache_get_cbom_block_size(void); + +void rt_hw_cpu_icache_invalidate_all(void); +void rt_hw_cpu_icache_invalidate(void *addr, rt_size_t size); +void rt_hw_cpu_dcache_clean_and_invalidate(void *addr, rt_size_t size); + +void rt_hw_icache_invalidate_all(void); + +void rt_hw_sync_cache_local(void *addr, int size); +#endif /* !__ASSEMBLY__ */ + +#endif /* __CACHE_H__ */ diff --git a/libcpu/risc-v/common/include/context_ext.h b/libcpu/risc-v/common/include/context_ext.h new file mode 100644 index 0000000000..2c2676e2b6 --- /dev/null +++ b/libcpu/risc-v/common/include/context_ext.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2006-2024, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-10-10 RT-Thread the first version + */ + +#ifndef __CONTEXT_EXT_H__ +#define __CONTEXT_EXT_H__ + +#include <asm-generic.h> + +#ifdef ARCH_RISCV_FPU +#ifndef __riscv_flen +#error "ARCH_RISCV_FPU requires a toolchain target with F or D extension" +#endif +/* Number of XLEN-sized stack slots occupied by 32 floating-point registers. */ +#define CTX_FPU_REG_NR (32 * __riscv_flen / __riscv_xlen) +#else +#define CTX_FPU_REG_NR 0 +#endif + +#ifdef __ASSEMBLY__ + +/** + * ================================== + * RISC-V D ISA (Floating) + * ================================== + */ + +#ifdef ARCH_RISCV_FPU +#define FPU_CTX_REG_OFF(n) ((__riscv_flen / 8) * (n)) +#define FPU_CTX_F0_OFF FPU_CTX_REG_OFF(0) +#define FPU_CTX_F1_OFF FPU_CTX_REG_OFF(1) +#define FPU_CTX_F2_OFF FPU_CTX_REG_OFF(2) +#define FPU_CTX_F3_OFF FPU_CTX_REG_OFF(3) +#define FPU_CTX_F4_OFF FPU_CTX_REG_OFF(4) +#define FPU_CTX_F5_OFF FPU_CTX_REG_OFF(5) +#define FPU_CTX_F6_OFF FPU_CTX_REG_OFF(6) +#define FPU_CTX_F7_OFF FPU_CTX_REG_OFF(7) +#define FPU_CTX_F8_OFF FPU_CTX_REG_OFF(8) +#define FPU_CTX_F9_OFF FPU_CTX_REG_OFF(9) +#define FPU_CTX_F10_OFF FPU_CTX_REG_OFF(10) +#define FPU_CTX_F11_OFF FPU_CTX_REG_OFF(11) +#define FPU_CTX_F12_OFF FPU_CTX_REG_OFF(12) +#define FPU_CTX_F13_OFF FPU_CTX_REG_OFF(13) +#define FPU_CTX_F14_OFF FPU_CTX_REG_OFF(14) +#define FPU_CTX_F15_OFF FPU_CTX_REG_OFF(15) +#define FPU_CTX_F16_OFF FPU_CTX_REG_OFF(16) +#define FPU_CTX_F17_OFF FPU_CTX_REG_OFF(17) +#define FPU_CTX_F18_OFF FPU_CTX_REG_OFF(18) +#define FPU_CTX_F19_OFF FPU_CTX_REG_OFF(19) +#define FPU_CTX_F20_OFF FPU_CTX_REG_OFF(20) +#define FPU_CTX_F21_OFF FPU_CTX_REG_OFF(21) +#define FPU_CTX_F22_OFF FPU_CTX_REG_OFF(22) +#define FPU_CTX_F23_OFF FPU_CTX_REG_OFF(23) +#define FPU_CTX_F24_OFF FPU_CTX_REG_OFF(24) +#define FPU_CTX_F25_OFF FPU_CTX_REG_OFF(25) +#define FPU_CTX_F26_OFF FPU_CTX_REG_OFF(26) +#define FPU_CTX_F27_OFF FPU_CTX_REG_OFF(27) +#define FPU_CTX_F28_OFF FPU_CTX_REG_OFF(28) +#define FPU_CTX_F29_OFF FPU_CTX_REG_OFF(29) +#define FPU_CTX_F30_OFF FPU_CTX_REG_OFF(30) +#define FPU_CTX_F31_OFF FPU_CTX_REG_OFF(31) + +#if __riscv_flen == 64 +#define CTX_FREG_L fld +#define CTX_FREG_S fsd +#elif __riscv_flen == 32 +#define CTX_FREG_L flw +#define CTX_FREG_S fsw +#else +#error "Unexpected __riscv_flen" +#endif +#endif /* ARCH_RISCV_FPU */ + +#endif /* __ASSEMBLY__ */ + +#ifdef ARCH_RISCV_VECTOR +#include "rvv_context.h" +#else /* !ARCH_RISCV_VECTOR */ +#define CTX_VECTOR_REG_NR 0 +#endif /* ARCH_RISCV_VECTOR */ + +#endif /* __CONTEXT_EXT_H__ */ diff --git a/libcpu/risc-v/common/include/cpu.h b/libcpu/risc-v/common/include/cpu.h new file mode 100644 index 0000000000..c78a574e20 --- /dev/null +++ b/libcpu/risc-v/common/include/cpu.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2025-04-20 GuEe-GUI The first version + */ + +#ifndef __CPU_H__ +#define __CPU_H__ + +#include <rtdef.h> + +#ifndef RT_CPUS_NR +#define RT_CPUS_NR 1 +#endif + +#if defined(RT_USING_DM) && defined(RT_USING_SMP) +struct cpu_ops_t +{ + const char *method; + int (*cpu_init)(rt_uint32_t id, void *param); + int (*cpu_boot)(rt_uint32_t id, rt_uint64_t entry); + void (*cpu_shutdown)(void); +}; +#endif + +extern rt_ubase_t boot_cpu_hartid; +extern void _secondary_cpu_entry(void); + +#if defined(RT_USING_SMP) && defined(ARCH_MM_MMU) +extern char __percpu_real_end, __percpu_end, __percpu_start; +#endif /* RT_USING_SMP && ARCH_MM_MMU */ + +rt_ubase_t riscv_cpu_id_to_hartid(int cpu_id); +rt_bool_t riscv_cpu_id_available(int cpu_id); +int riscv_hartid_to_cpu_id(rt_ubase_t hartid); +int riscv_hartid_register(rt_ubase_t hartid); +void riscv_cpu_id_set(int cpu_id); +#if defined(RT_USING_SMP) && !defined(RT_USING_DM) +rt_size_t riscv_hartid_list_get(rt_ubase_t *hartids, rt_size_t max_harts); +#endif +void riscv_boot_hart_init(rt_ubase_t hartid); + +#endif /* __CPU_H__ */ diff --git a/libcpu/risc-v/common/include/cpuport.h b/libcpu/risc-v/common/include/cpuport.h new file mode 100644 index 0000000000..10f265ed38 --- /dev/null +++ b/libcpu/risc-v/common/include/cpuport.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2018-10-03 Bernard The first version + */ + +#ifndef CPUPORT_H__ +#define CPUPORT_H__ + +#include <rtconfig.h> + +#ifdef RT_USING_CACHE +#include <cache.h> +#define RT_CPU_CACHE_LINE_SZ L1_CACHE_BYTES +#endif + +#include <opcode.h> + +#ifdef RT_USING_SMP +typedef union +{ + unsigned long slock; + struct __arch_tickets + { + unsigned short owner; + unsigned short next; + } tickets; +} rt_hw_spinlock_t; +#endif /* RT_USING_SMP */ + +#define RISCV_FENCE(p, s) __asm__ volatile ("fence " #p "," #s:::"memory") + +#define rt_hw_barrier(...) RISCV_FENCE(iorw, iorw) +#define rt_hw_wfi() __asm__ volatile ("wfi") +#define rt_hw_rmb() RISCV_FENCE(ir, ir) +#define rt_hw_wmb() RISCV_FENCE(ow, ow) + +#ifndef __ASSEMBLY__ +#ifdef RT_USING_CACHE +rt_inline void rt_hw_dsb(void) +{ + __asm__ volatile ("fence":::"memory"); +} + +rt_inline void rt_hw_dmb(void) +{ + __asm__ volatile ("fence":::"memory"); +} + +rt_inline void rt_hw_isb(void) +{ +#ifndef ARCH_RISCV_DISABLE_FENCE_I + __asm__ volatile (OPC_FENCE_I:::"memory"); +#endif +} +#endif /* RT_USING_CACHE */ + +rt_inline void rt_hw_cpu_relax(void) +{ + __asm__ volatile (OPC_PAUSE_I); + rt_hw_barrier(); +} + +void _thread_start(void); +#endif /* !__ASSEMBLY__ */ + +#endif /* CPUPORT_H__ */ diff --git a/libcpu/risc-v/common/include/csr.h b/libcpu/risc-v/common/include/csr.h new file mode 100644 index 0000000000..f5ddca9c40 --- /dev/null +++ b/libcpu/risc-v/common/include/csr.h @@ -0,0 +1,580 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2025-04-20 GuEe-GUI The first version + */ + +#ifndef __CSR_H__ +#define __CSR_H__ + +#include <asm-generic.h> + +/* Status register flags */ +#define SR_SIE _UL(0x00000002) /* Supervisor Interrupt Enable */ +#define SR_MIE _UL(0x00000008) /* Machine Interrupt Enable */ +#define SR_SPIE _UL(0x00000020) /* Previous Supervisor IE */ +#define SR_MPIE _UL(0x00000080) /* Previous Machine IE */ +#define SR_SPP _UL(0x00000100) /* Previously Supervisor */ +#define SR_MPP _UL(0x00001800) /* Previously Machine */ +#define SR_SUM _UL(0x00040000) /* Supervisor User Memory Access */ +#define SR_MXR _UL(0x00080000) /* Make eXecutable Readable */ + +#define SR_FS _UL(0x00006000) /* Floating-point Status */ +#define SR_FS_OFF _UL(0x00000000) +#define SR_FS_INITIAL _UL(0x00002000) +#define SR_FS_CLEAN _UL(0x00004000) +#define SR_FS_DIRTY _UL(0x00006000) + +#define SR_VS _UL(0x00000600) /* Vector Status */ +#define SR_VS_OFF _UL(0x00000000) +#define SR_VS_INITIAL _UL(0x00000200) +#define SR_VS_CLEAN _UL(0x00000400) +#define SR_VS_DIRTY _UL(0x00000600) + +#define SR_XS _UL(0x00018000) /* Extension Status */ +#define SR_XS_OFF _UL(0x00000000) +#define SR_XS_INITIAL _UL(0x00008000) +#define SR_XS_CLEAN _UL(0x00010000) +#define SR_XS_DIRTY _UL(0x00018000) + +#define SR_FS_VS (SR_FS | SR_VS) /* Vector and Floating-Point Unit */ + +#if __riscv_xlen == 32 +#define SR_SD _UL(0x80000000) /* FS/VS/XS dirty */ +#else +#define SR_SD _UL(0x8000000000000000) /* FS/VS/XS dirty */ +#endif + +#if __riscv_xlen == 64 +#define SR_UXL _UL(0x300000000) /* XLEN mask for U-mode */ +#define SR_UXL_32 _UL(0x100000000) /* XLEN = 32 for U-mode */ +#define SR_UXL_64 _UL(0x200000000) /* XLEN = 64 for U-mode */ +#endif + +/* SATP flags */ +#if __riscv_xlen == 32 +#define SATP_PPN _UL(0x003fffff) +#define SATP_MODE_BARE _UL(0x00000000) +#define SATP_MODE_32 _UL(0x80000000) +#define SATP_MODE_SHIFT 31 +#define SATP_MODE_MASK (_UL(0x1) << SATP_MODE_SHIFT) +#define SATP_ASID_BITS 9 +#define SATP_ASID_SHIFT 22 +#define SATP_ASID_MASK _UL(0x1ff) +#else +#define SATP_PPN _UL(0x00000fffffffffff) +#define SATP_MODE_BARE _UL(0x0000000000000000) +#define SATP_MODE_39 _UL(0x8000000000000000) +#define SATP_MODE_48 _UL(0x9000000000000000) +#define SATP_MODE_57 _UL(0xa000000000000000) +#define SATP_MODE_64 _UL(0xb000000000000000) +#define SATP_MODE_SHIFT 60 +#define SATP_MODE_MASK (_UL(0xf) << SATP_MODE_SHIFT) +#define SATP_ASID_BITS 16 +#define SATP_ASID_SHIFT 44 +#define SATP_ASID_MASK _UL(0xffff) +#endif + +/* Exception cause high bit - is an interrupt if set */ +#define CAUSE_IRQ_FLAG (_UL(1) << (__riscv_xlen - 1)) + +/* Interrupt causes (minus the high bit) */ +#define IRQ_S_SOFT 1 +#define IRQ_VS_SOFT 2 +#define IRQ_M_SOFT 3 +#define IRQ_S_TIMER 5 +#define IRQ_VS_TIMER 6 +#define IRQ_M_TIMER 7 +#define IRQ_S_EXT 9 +#define IRQ_VS_EXT 10 +#define IRQ_M_EXT 11 +#define IRQ_S_GEXT 12 +#define IRQ_PMU_OVF 13 +#define IRQ_LOCAL_MAX (IRQ_PMU_OVF + 1) +#define IRQ_LOCAL_MASK GENMASK((IRQ_LOCAL_MAX - 1), 0) + +/* Exception causes */ +#define EXC_INST_MISALIGNED 0 +#define EXC_INST_ACCESS 1 +#define EXC_INST_ILLEGAL 2 +#define EXC_BREAKPOINT 3 +#define EXC_LOAD_MISALIGNED 4 +#define EXC_LOAD_ACCESS 5 +#define EXC_STORE_MISALIGNED 6 +#define EXC_STORE_ACCESS 7 +#define EXC_SYSCALL 8 +#define EXC_HYPERVISOR_SYSCALL 9 +#define EXC_SUPERVISOR_SYSCALL 10 +#define EXC_INST_PAGE_FAULT 12 +#define EXC_LOAD_PAGE_FAULT 13 +#define EXC_STORE_PAGE_FAULT 15 +#define EXC_INST_GUEST_PAGE_FAULT 20 +#define EXC_LOAD_GUEST_PAGE_FAULT 21 +#define EXC_VIRTUAL_INST_FAULT 22 +#define EXC_STORE_GUEST_PAGE_FAULT 23 + +/* PMP configuration */ +#define PMP_R 0x01 +#define PMP_W 0x02 +#define PMP_X 0x04 +#define PMP_A 0x18 +#define PMP_A_TOR 0x08 +#define PMP_A_NA4 0x10 +#define PMP_A_NAPOT 0x18 +#define PMP_L 0x80 + +/* HSTATUS flags */ +#if __riscv_xlen == 64 +#define HSTATUS_VSXL _UL(0x300000000) +#define HSTATUS_VSXL_SHIFT 32 +#endif +#define HSTATUS_VTSR _UL(0x00400000) +#define HSTATUS_VTW _UL(0x00200000) +#define HSTATUS_VTVM _UL(0x00100000) +#define HSTATUS_VGEIN _UL(0x0003f000) +#define HSTATUS_VGEIN_SHIFT 12 +#define HSTATUS_HU _UL(0x00000200) +#define HSTATUS_SPVP _UL(0x00000100) +#define HSTATUS_SPV _UL(0x00000080) +#define HSTATUS_GVA _UL(0x00000040) +#define HSTATUS_VSBE _UL(0x00000020) + +/* HGATP flags */ +#define HGATP_MODE_OFF _UL(0) +#define HGATP_MODE_SV32X4 _UL(1) +#define HGATP_MODE_SV39X4 _UL(8) +#define HGATP_MODE_SV48X4 _UL(9) +#define HGATP_MODE_SV57X4 _UL(10) + +#define HGATP32_MODE_SHIFT 31 +#define HGATP32_VMID_SHIFT 22 +#define HGATP32_VMID GENMASK(28, 22) +#define HGATP32_PPN GENMASK(21, 0) + +#define HGATP64_MODE_SHIFT 60 +#define HGATP64_VMID_SHIFT 44 +#define HGATP64_VMID GENMASK(57, 44) +#define HGATP64_PPN GENMASK(43, 0) + +#define HGATP_PAGE_SHIFT 12 + +#if __riscv_xlen == 64 +#define HGATP_PPN HGATP64_PPN +#define HGATP_VMID_SHIFT HGATP64_VMID_SHIFT +#define HGATP_VMID HGATP64_VMID +#define HGATP_MODE_SHIFT HGATP64_MODE_SHIFT +#else +#define HGATP_PPN HGATP32_PPN +#define HGATP_VMID_SHIFT HGATP32_VMID_SHIFT +#define HGATP_VMID HGATP32_VMID +#define HGATP_MODE_SHIFT HGATP32_MODE_SHIFT +#endif + +/* VSIP & HVIP relation */ +#define VSIP_TO_HVIP_SHIFT (IRQ_VS_SOFT - IRQ_S_SOFT) +#define VSIP_VALID_MASK ((_UL(1) << IRQ_S_SOFT) | \ + (_UL(1) << IRQ_S_TIMER) | \ + (_UL(1) << IRQ_S_EXT) | \ + (_UL(1) << IRQ_PMU_OVF)) + +/* AIA CSR bits */ +#define TOPI_IID_SHIFT 16 +#define TOPI_IID_MASK GENMASK(11, 0) +#define TOPI_IPRIO_MASK GENMASK(7, 0) +#define TOPI_IPRIO_BITS 8 + +#define TOPEI_ID_SHIFT 16 +#define TOPEI_ID_MASK GENMASK(10, 0) +#define TOPEI_PRIO_MASK GENMASK(10, 0) + +#define ISELECT_IPRIO0 0x30 +#define ISELECT_IPRIO15 0x3f +#define ISELECT_MASK GENMASK(8, 0) + +#define HVICTL_VTI BIT(30) +#define HVICTL_IID GENMASK(27, 16) +#define HVICTL_IID_SHIFT 16 +#define HVICTL_DPR BIT(9) +#define HVICTL_IPRIOM BIT(8) +#define HVICTL_IPRIO GENMASK(7, 0) + +/* xENVCFG flags */ +#define ENVCFG_STCE (_ULL(1) << 63) +#define ENVCFG_PBMTE (_ULL(1) << 62) +#define ENVCFG_CBZE (_UL(1) << 7) +#define ENVCFG_CBCFE (_UL(1) << 6) +#define ENVCFG_CBIE_SHIFT 4 +#define ENVCFG_CBIE (_UL(0x3) << ENVCFG_CBIE_SHIFT) +#define ENVCFG_CBIE_ILL _UL(0x0) +#define ENVCFG_CBIE_FLUSH _UL(0x1) +#define ENVCFG_CBIE_INV _UL(0x3) +#define ENVCFG_FIOM _UL(0x1) + +/* Smstateen bits */ +#define SMSTATEEN0_AIA_IMSIC_SHIFT 58 +#define SMSTATEEN0_AIA_IMSIC (_ULL(1) << SMSTATEEN0_AIA_IMSIC_SHIFT) +#define SMSTATEEN0_AIA_SHIFT 59 +#define SMSTATEEN0_AIA (_ULL(1) << SMSTATEEN0_AIA_SHIFT) +#define SMSTATEEN0_AIA_ISEL_SHIFT 60 +#define SMSTATEEN0_AIA_ISEL (_ULL(1) << SMSTATEEN0_AIA_ISEL_SHIFT) +#define SMSTATEEN0_HSENVCFG_SHIFT 62 +#define SMSTATEEN0_HSENVCFG (_ULL(1) << SMSTATEEN0_HSENVCFG_SHIFT) +#define SMSTATEEN0_SSTATEEN0_SHIFT 63 +#define SMSTATEEN0_SSTATEEN0 (_ULL(1) << SMSTATEEN0_SSTATEEN0_SHIFT) + +/* symbolic CSR names: */ +#define CSR_CYCLE 0xc00 +#define CSR_TIME 0xc01 +#define CSR_INSTRET 0xc02 +#define CSR_HPMCOUNTER3 0xc03 +#define CSR_HPMCOUNTER4 0xc04 +#define CSR_HPMCOUNTER5 0xc05 +#define CSR_HPMCOUNTER6 0xc06 +#define CSR_HPMCOUNTER7 0xc07 +#define CSR_HPMCOUNTER8 0xc08 +#define CSR_HPMCOUNTER9 0xc09 +#define CSR_HPMCOUNTER10 0xc0a +#define CSR_HPMCOUNTER11 0xc0b +#define CSR_HPMCOUNTER12 0xc0c +#define CSR_HPMCOUNTER13 0xc0d +#define CSR_HPMCOUNTER14 0xc0e +#define CSR_HPMCOUNTER15 0xc0f +#define CSR_HPMCOUNTER16 0xc10 +#define CSR_HPMCOUNTER17 0xc11 +#define CSR_HPMCOUNTER18 0xc12 +#define CSR_HPMCOUNTER19 0xc13 +#define CSR_HPMCOUNTER20 0xc14 +#define CSR_HPMCOUNTER21 0xc15 +#define CSR_HPMCOUNTER22 0xc16 +#define CSR_HPMCOUNTER23 0xc17 +#define CSR_HPMCOUNTER24 0xc18 +#define CSR_HPMCOUNTER25 0xc19 +#define CSR_HPMCOUNTER26 0xc1a +#define CSR_HPMCOUNTER27 0xc1b +#define CSR_HPMCOUNTER28 0xc1c +#define CSR_HPMCOUNTER29 0xc1d +#define CSR_HPMCOUNTER30 0xc1e +#define CSR_HPMCOUNTER31 0xc1f +#define CSR_CYCLEH 0xc80 +#define CSR_TIMEH 0xc81 +#define CSR_INSTRETH 0xc82 +#define CSR_HPMCOUNTER3H 0xc83 +#define CSR_HPMCOUNTER4H 0xc84 +#define CSR_HPMCOUNTER5H 0xc85 +#define CSR_HPMCOUNTER6H 0xc86 +#define CSR_HPMCOUNTER7H 0xc87 +#define CSR_HPMCOUNTER8H 0xc88 +#define CSR_HPMCOUNTER9H 0xc89 +#define CSR_HPMCOUNTER10H 0xc8a +#define CSR_HPMCOUNTER11H 0xc8b +#define CSR_HPMCOUNTER12H 0xc8c +#define CSR_HPMCOUNTER13H 0xc8d +#define CSR_HPMCOUNTER14H 0xc8e +#define CSR_HPMCOUNTER15H 0xc8f +#define CSR_HPMCOUNTER16H 0xc90 +#define CSR_HPMCOUNTER17H 0xc91 +#define CSR_HPMCOUNTER18H 0xc92 +#define CSR_HPMCOUNTER19H 0xc93 +#define CSR_HPMCOUNTER20H 0xc94 +#define CSR_HPMCOUNTER21H 0xc95 +#define CSR_HPMCOUNTER22H 0xc96 +#define CSR_HPMCOUNTER23H 0xc97 +#define CSR_HPMCOUNTER24H 0xc98 +#define CSR_HPMCOUNTER25H 0xc99 +#define CSR_HPMCOUNTER26H 0xc9a +#define CSR_HPMCOUNTER27H 0xc9b +#define CSR_HPMCOUNTER28H 0xc9c +#define CSR_HPMCOUNTER29H 0xc9d +#define CSR_HPMCOUNTER30H 0xc9e +#define CSR_HPMCOUNTER31H 0xc9f + +#define CSR_SCOUNTOVF 0xda0 + +#define CSR_SSTATUS 0x100 +#define CSR_SIE 0x104 +#define CSR_STVEC 0x105 +#define CSR_SCOUNTEREN 0x106 +#define CSR_SENVCFG 0x10a +#define CSR_SSTATEEN0 0x10c +#define CSR_SSCRATCH 0x140 +#define CSR_SEPC 0x141 +#define CSR_SCAUSE 0x142 +#define CSR_STVAL 0x143 +#define CSR_SIP 0x144 +#define CSR_SATP 0x180 + +/* Supervisor-Level Interrupts (CLIC) */ +#define CSR_STVT 0x107 +#define CSR_SNXTI 0x145 + +#define CSR_STIMECMP 0x14d +#define CSR_STIMECMPH 0x15d + +/* Supervisor-Level Window to Indirectly Accessed Registers (AIA) */ +#define CSR_SISELECT 0x150 +#define CSR_SIREG 0x151 + +/* Supervisor-Level Interrupts (AIA) */ +#define CSR_STOPEI 0x15c +#define CSR_STOPI 0xdb0 + +/* Supervisor-Level High-Half CSRs (AIA) */ +#define CSR_SIEH 0x114 +#define CSR_SIPH 0x154 + +#define CSR_VSSTATUS 0x200 +#define CSR_VSIE 0x204 +#define CSR_VSTVEC 0x205 +#define CSR_VSSCRATCH 0x240 +#define CSR_VSEPC 0x241 +#define CSR_VSCAUSE 0x242 +#define CSR_VSTVAL 0x243 +#define CSR_VSIP 0x244 +#define CSR_VSATP 0x280 +#define CSR_VSTIMECMP 0x24d +#define CSR_VSTIMECMPH 0x25d + +#define CSR_HSTATUS 0x600 +#define CSR_HEDELEG 0x602 +#define CSR_HIDELEG 0x603 +#define CSR_HIE 0x604 +#define CSR_HTIMEDELTA 0x605 +#define CSR_HCOUNTEREN 0x606 +#define CSR_HGEIE 0x607 +#define CSR_HENVCFG 0x60a +#define CSR_HTIMEDELTAH 0x615 +#define CSR_HENVCFGH 0x61a +#define CSR_HTVAL 0x643 +#define CSR_HIP 0x644 +#define CSR_HVIP 0x645 +#define CSR_HTINST 0x64a +#define CSR_HGATP 0x680 +#define CSR_HGEIP 0xe12 + +/* Virtual Interrupts and Interrupt Priorities (H-extension with AIA) */ +#define CSR_HVIEN 0x608 +#define CSR_HVICTL 0x609 +#define CSR_HVIPRIO1 0x646 +#define CSR_HVIPRIO2 0x647 + +/* VS-Level Window to Indirectly Accessed Registers (H-extension with AIA) */ +#define CSR_VSISELECT 0x250 +#define CSR_VSIREG 0x251 + +/* VS-Level Interrupts (H-extension with AIA) */ +#define CSR_VSTOPEI 0x25c +#define CSR_VSTOPI 0xeb0 + +/* Hypervisor and VS-Level High-Half CSRs (H-extension with AIA) */ +#define CSR_HIDELEGH 0x613 +#define CSR_HVIENH 0x618 +#define CSR_HVIPH 0x655 +#define CSR_HVIPRIO1H 0x656 +#define CSR_HVIPRIO2H 0x657 +#define CSR_VSIEH 0x214 +#define CSR_VSIPH 0x254 + +/* Hypervisor stateen CSRs */ +#define CSR_HSTATEEN0 0x60c +#define CSR_HSTATEEN0H 0x61c + +#define CSR_MSTATUS 0x300 +#define CSR_MISA 0x301 +#define CSR_MIDELEG 0x303 +#define CSR_MIE 0x304 +#define CSR_MTVEC 0x305 +#define CSR_MENVCFG 0x30a +#define CSR_MENVCFGH 0x31a +#define CSR_MSCRATCH 0x340 +#define CSR_MEPC 0x341 +#define CSR_MCAUSE 0x342 +#define CSR_MTVAL 0x343 +#define CSR_MIP 0x344 +#define CSR_PMPCFG0 0x3a0 +#define CSR_PMPADDR0 0x3b0 +#define CSR_MVENDORID 0xf11 +#define CSR_MARCHID 0xf12 +#define CSR_MIMPID 0xf13 +#define CSR_MHARTID 0xf14 + +/* Machine-Level Interrupts (CLIC) */ +#define CSR_MTVT 0x307 +#define CSR_MNXTI 0x345 + +/* Machine-Level Window to Indirectly Accessed Registers (AIA) */ +#define CSR_MISELECT 0x350 +#define CSR_MIREG 0x351 + +/* Machine-Level Interrupts (AIA) */ +#define CSR_MTOPEI 0x35c +#define CSR_MTOPI 0xfb0 + +/* Virtual Interrupts for Supervisor Level (AIA) */ +#define CSR_MVIEN 0x308 +#define CSR_MVIP 0x309 + +/* Machine-Level High-Half CSRs (AIA) */ +#define CSR_MIDELEGH 0x313 +#define CSR_MIEH 0x314 +#define CSR_MVIENH 0x318 +#define CSR_MVIPH 0x319 +#define CSR_MIPH 0x354 + +#define CSR_VSTART 0x8 +#define CSR_VCSR 0xf +#define CSR_VL 0xc20 +#define CSR_VTYPE 0xc21 +#define CSR_VLENB 0xc22 + +/* Scalar Crypto Extension - Entropy */ +#define CSR_SEED 0x015 +#define SEED_OPST_MASK _UL(0xc0000000) +#define SEED_OPST_BIST _UL(0x00000000) +#define SEED_OPST_WAIT _UL(0x40000000) +#define SEED_OPST_ES16 _UL(0x80000000) +#define SEED_OPST_DEAD _UL(0xc0000000) +#define SEED_ENTROPY_MASK _UL(0xffff) + +#ifdef ARCH_RISCV_M_MODE +#define CSR_STATUS CSR_MSTATUS +#define CSR_IE CSR_MIE +#define CSR_TVEC CSR_MTVEC +#define CSR_ENVCFG CSR_MENVCFG +#define CSR_SCRATCH CSR_MSCRATCH +#define CSR_EPC CSR_MEPC +#define CSR_CAUSE CSR_MCAUSE +#define CSR_TVAL CSR_MTVAL +#define CSR_IP CSR_MIP +#define CSR_TVT CSR_MTVT +#define CSR_NXTI CSR_MNXTI + +#define CSR_IEH CSR_MIEH +#define CSR_ISELECT CSR_MISELECT +#define CSR_IREG CSR_MIREG +#define CSR_IPH CSR_MIPH +#define CSR_TOPEI CSR_MTOPEI +#define CSR_TOPI CSR_MTOPI + +#define SR_IE SR_MIE +#define SR_PIE SR_MPIE +#define SR_PP SR_MPP + +#define RV_IRQ_SOFT IRQ_M_SOFT +#define RV_IRQ_TIMER IRQ_M_TIMER +#define RV_IRQ_EXT IRQ_M_EXT +#else /* ARCH_RISCV_M_MODE */ +#define CSR_STATUS CSR_SSTATUS +#define CSR_IE CSR_SIE +#define CSR_TVEC CSR_STVEC +#define CSR_ENVCFG CSR_SENVCFG +#define CSR_SCRATCH CSR_SSCRATCH +#define CSR_EPC CSR_SEPC +#define CSR_CAUSE CSR_SCAUSE +#define CSR_TVAL CSR_STVAL +#define CSR_IP CSR_SIP +#define CSR_TVT CSR_STVT +#define CSR_NXTI CSR_SNXTI + +#define CSR_IEH CSR_SIEH +#define CSR_ISELECT CSR_SISELECT +#define CSR_IREG CSR_SIREG +#define CSR_IPH CSR_SIPH +#define CSR_TOPEI CSR_STOPEI +#define CSR_TOPI CSR_STOPI + +#define SR_IE SR_SIE +#define SR_PIE SR_SPIE +#define SR_PP SR_SPP + +#define RV_IRQ_SOFT IRQ_S_SOFT +#define RV_IRQ_TIMER IRQ_S_TIMER +#define RV_IRQ_EXT IRQ_S_EXT +#define RV_IRQ_PMU IRQ_PMU_OVF +#define SIP_LCOFIP (_AC(0x1, UL) << IRQ_PMU_OVF) +#endif /* !ARCH_RISCV_M_MODE */ + +/* IE/IP (Supervisor/Machine Interrupt Enable/Pending) flags */ +#define IE_SIE (_UL(0x1) << RV_IRQ_SOFT) +#define IE_TIE (_UL(0x1) << RV_IRQ_TIMER) +#define IE_EIE (_UL(0x1) << RV_IRQ_EXT) + +#ifndef __ASSEMBLY__ + +#ifdef __GNUC__ +#define csr_swap(csr, val) \ +({ \ + rt_ubase_t __v = (rt_ubase_t)(val); \ + __asm__ volatile ("csrrw %0, " __ASM_STR(csr) ", %1" \ + : "=r" (__v) : "rK" (__v) \ + : "memory"); \ + __v; \ +}) + +#define csr_read(csr) \ +({ \ + register rt_ubase_t __v; \ + __asm__ volatile ("csrr %0, " __ASM_STR(csr) \ + : "=r" (__v) : \ + : "memory"); \ + __v; \ +}) + +#define csr_write(csr, val) \ +({ \ + rt_ubase_t __v = (rt_ubase_t)(val); \ + __asm__ volatile ("csrw " __ASM_STR(csr) ", %0" \ + : : "rK" (__v) \ + : "memory"); \ +}) + +#define csr_read_set(csr, val) \ +({ \ + rt_ubase_t __v = (rt_ubase_t)(val); \ + __asm__ volatile ("csrrs %0, " __ASM_STR(csr) ", %1" \ + : "=r" (__v) : "rK" (__v) \ + : "memory"); \ + __v; \ +}) + +#define csr_read_or(csr, val) \ +({ \ + rt_ubase_t __v = (rt_ubase_t)(val); \ + __asm__ volatile ("csrrsi %0, " __ASM_STR(csr) ", %1" \ + : "=r" (__v) : "i" (__v) \ + : "memory"); \ + __v; \ +}) + +#define csr_set(csr, val) \ +({ \ + rt_ubase_t __v = (rt_ubase_t)(val); \ + __asm__ volatile ("csrs " __ASM_STR(csr) ", %0" \ + : : "rK" (__v) \ + : "memory"); \ +}) + +#define csr_read_clear(csr, val) \ +({ \ + rt_ubase_t __v = (rt_ubase_t)(val); \ + __asm__ volatile ("csrrc %0, " __ASM_STR(csr) ", %1" \ + : "=r" (__v) : "rK" (__v) \ + : "memory"); \ + __v; \ +}) + +#define csr_clear(csr, val) \ +({ \ + rt_ubase_t __v = (rt_ubase_t)(val); \ + __asm__ volatile ("csrc " __ASM_STR(csr) ", %0" \ + : : "rK" (__v) \ + : "memory"); \ +}) +#endif /* __GNUC__ */ + +#endif /* __ASSEMBLY__ */ + +#endif /* __CSR_H__ */ diff --git a/libcpu/risc-v/common/include/interrupt.h b/libcpu/risc-v/common/include/interrupt.h new file mode 100644 index 0000000000..d7cbb2a3f3 --- /dev/null +++ b/libcpu/risc-v/common/include/interrupt.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2018-10-01 Bernard The first version + * 2025-04-20 GuEe-GUI Port for not DM + */ + +#ifndef __INTERRUPT_H__ +#define __INTERRUPT_H__ + +#include <rthw.h> + +#ifndef RT_USING_DM +#ifdef RT_USING_ADT_BITMAP +#include <bitmap.h> + +#define RISCV_PLIC_QUIRK_EDGE_INTERRUPT RT_BIT(0) + +#define RISCV_CLIC_QUIRK_MODE_FIXED RT_BIT(0) +#define RISCV_CLIC_QUIRK_MINTTHRESH RT_BIT(1) +#define RISCV_CLIC_QUIRK_SHV RT_BIT(2) +#endif /* RT_USING_ADT_BITMAP */ + +#define INTC_TYPE_RISCV 0 +#define INTC_TYPE_ANDESTECH 1 +#define INTC_TYPE_CLIC 2 + +#define IRQ_INTC(vector) (vector) +#ifdef RT_USING_SMP +#define IPI_INTC(vector) (ipi_intc_base + (vector)) +#endif + +#define IRQ_PLIC(vector) (irq_plic_base + (vector)) + +#define IRQ_CLIC(vector) (vector) +#ifdef RT_USING_SMP +#define IPI_CLIC(vector) (ipi_clic_base + (vector)) +extern int ipi_intc_base; +extern int ipi_clic_base; +#endif +extern int irq_plic_base; + +rt_ubase_t platform_get_plic_base(void); +rt_size_t platform_get_plic_size(void); +rt_ubase_t platform_get_plic_quirks(void); +rt_ubase_t platform_get_clic_base(void); +rt_size_t platform_get_clic_size(void); +rt_ubase_t platform_get_clic_quirks(void); + +#ifdef ARCH_RISCV_M_MODE +rt_ubase_t platform_get_clint_base(void); +rt_size_t platform_get_clint_size(void); +#endif + +#endif /* !RT_USING_DM */ + +#ifdef RT_USING_SMP +void rt_hw_ipi_handler_install(int ipi_vector, rt_isr_handler_t ipi_isr_handler); +#endif + +void rt_hw_interrupt_handle(rt_uint32_t vector, void *param); + +#endif /* __INTERRUPT_H__ */ diff --git a/libcpu/risc-v/common/include/isa_ext.h b/libcpu/risc-v/common/include/isa_ext.h new file mode 100644 index 0000000000..f806421aba --- /dev/null +++ b/libcpu/risc-v/common/include/isa_ext.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2025-04-20 GuEe-GUI The first version + */ + +#ifndef __ISA_EXT_H__ +#define __ISA_EXT_H__ + +#include <rtdef.h> + +#define RISCV_ISA_EXT_I 0 +#define RISCV_ISA_EXT_M 1 +#define RISCV_ISA_EXT_A 2 +#define RISCV_ISA_EXT_F 3 +#define RISCV_ISA_EXT_D 4 +#define RISCV_ISA_EXT_Q 5 +#define RISCV_ISA_EXT_C 6 +#define RISCV_ISA_EXT_V 7 +#define RISCV_ISA_EXT_H 8 + +#define RISCV_ISA_EXT_BASE 26 + +#define RISCV_ISA_EXT_SSCOFPMF 26 +#define RISCV_ISA_EXT_SSTC 27 +#define RISCV_ISA_EXT_SVINVAL 28 +#define RISCV_ISA_EXT_SVPBMT 29 +#define RISCV_ISA_EXT_ZBB 30 +#define RISCV_ISA_EXT_ZICBOM 31 +#define RISCV_ISA_EXT_ZIHINTPAUSE 32 +#define RISCV_ISA_EXT_SVNAPOT 33 +#define RISCV_ISA_EXT_ZICBOZ 34 +#define RISCV_ISA_EXT_SMAIA 35 +#define RISCV_ISA_EXT_SSAIA 36 +#define RISCV_ISA_EXT_ZBA 37 +#define RISCV_ISA_EXT_ZBS 38 +#define RISCV_ISA_EXT_ZICNTR 39 +#define RISCV_ISA_EXT_ZICSR 40 +#define RISCV_ISA_EXT_ZIFENCEI 41 +#define RISCV_ISA_EXT_ZIHPM 42 +#define RISCV_ISA_EXT_SMSTATEEN 43 +#define RISCV_ISA_EXT_ZICOND 44 +#define RISCV_ISA_EXT_ZBC 45 +#define RISCV_ISA_EXT_ZBKB 46 +#define RISCV_ISA_EXT_ZBKC 47 +#define RISCV_ISA_EXT_ZBKX 48 +#define RISCV_ISA_EXT_ZKND 49 +#define RISCV_ISA_EXT_ZKNE 50 +#define RISCV_ISA_EXT_ZKNH 51 +#define RISCV_ISA_EXT_ZKR 52 +#define RISCV_ISA_EXT_ZKSED 53 +#define RISCV_ISA_EXT_ZKSH 54 +#define RISCV_ISA_EXT_ZKT 55 +#define RISCV_ISA_EXT_ZVBB 56 +#define RISCV_ISA_EXT_ZVBC 57 +#define RISCV_ISA_EXT_ZVKB 58 +#define RISCV_ISA_EXT_ZVKG 59 +#define RISCV_ISA_EXT_ZVKNED 60 +#define RISCV_ISA_EXT_ZVKNHA 61 +#define RISCV_ISA_EXT_ZVKNHB 62 +#define RISCV_ISA_EXT_ZVKSED 63 +#define RISCV_ISA_EXT_ZVKSH 64 +#define RISCV_ISA_EXT_ZVKT 65 +#define RISCV_ISA_EXT_ZFH 66 +#define RISCV_ISA_EXT_ZFHMIN 67 +#define RISCV_ISA_EXT_ZIHINTNTL 68 +#define RISCV_ISA_EXT_ZVFH 69 +#define RISCV_ISA_EXT_ZVFHMIN 70 +#define RISCV_ISA_EXT_ZFA 71 +#define RISCV_ISA_EXT_ZTSO 72 +#define RISCV_ISA_EXT_ZACAS 73 +#define RISCV_ISA_EXT_XANDESPMU 74 + +#define RISCV_ISA_EXT_MAX 75 +#define RISCV_ISA_EXT_INVALID RT_UINT32_MAX + +#ifdef ARCH_RISCV_M_MODE +#define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SMAIA +#else +#define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SSAIA +#endif + +void riscv_isa_ext_set_raw(rt_uint32_t isa); +void riscv_isa_ext_clear_raw(rt_uint32_t isa); +rt_bool_t riscv_isa_ext_test_raw(rt_uint32_t isa); + +#define riscv_isa_ext_set(isa) riscv_isa_ext_set_raw(RISCV_ISA_EXT_##isa) +#define riscv_isa_ext_clear(isa) riscv_isa_ext_clear_raw(RISCV_ISA_EXT_##isa) +#define riscv_isa_ext_test(isa) riscv_isa_ext_test_raw(RISCV_ISA_EXT_##isa) + +#endif /* __ISA_EXT_H__ */ diff --git a/libcpu/risc-v/common/include/mmu.h b/libcpu/risc-v/common/include/mmu.h new file mode 100644 index 0000000000..286143ecdf --- /dev/null +++ b/libcpu/risc-v/common/include/mmu.h @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2021-01-30 lizhirui first version + * 2023-10-12 Shell Add permission control API + * 2025-04-26 GuEe-GUI Follow the specific and vendor + */ + +#ifndef __MMU_H__ +#define __MMU_H__ + +#include <page_def.h> +#include <asm-generic.h> + +#ifndef __ASSEMBLY__ +#include <mm_aspace.h> + +/* Normal memory wra mapping type */ +#define NORMAL_MEM 0 +/* Normal nocache memory mapping type */ +#define NORMAL_NOCACHE_MEM 1 +/* Device mapping type */ +#define DEVICE_MEM 2 + +struct mem_desc +{ + unsigned long vaddr_start; + unsigned long vaddr_end; + unsigned long paddr_start; + unsigned long attr; + struct rt_varea varea; +}; + +#ifdef __HAS_MMU_VENDOR__ +#include <mmu_vendor.h> +#endif + +#endif /* !__ASSEMBLY__ */ + +#define RT_HW_MMU_PROT_READ 1 +#define RT_HW_MMU_PROT_WRITE 2 +#define RT_HW_MMU_PROT_EXECUTE 4 +#define RT_HW_MMU_PROT_KERNEL 8 +#define RT_HW_MMU_PROT_USER 16 +#define RT_HW_MMU_PROT_CACHE 32 + +/* Page table entry (PTE) fields */ +#define PTE_V BIT(0) /* Valid */ +#define PTE_R BIT(1) /* Read */ +#define PTE_W BIT(2) /* Write */ +#define PTE_X BIT(3) /* Execute */ +#define PTE_U BIT(4) /* User */ +#define PTE_G BIT(5) /* Global */ +#define PTE_A BIT(6) /* Accessed */ +#define PTE_D BIT(7) /* Dirty */ +#define PTE_SOFT GENMASK(9, 8) /* Reserved for Software */ + +#define PTE_BITS 10 +#if __riscv_xlen == 32 +#define PTE_PPN_MASK GENMASK(31, 10) +#else +#define PTE_PPN_MASK GENMASK(53, 10) +#endif + +#define PTE_ATTR_NONE _UL(0) +#define PTE_ATTR_RW (PTE_R | PTE_W) +#define PTE_ATTR_RWX (PTE_ATTR_RW | PTE_X) +#define PTE_ATTR_KERNEL (PTE_V | PTE_ATTR_RWX | PTE_A | PTE_D) +#define PTE_ATTR_USER (PTE_ATTR_KERNEL | PTE_U) + +#if !defined(__HAS_MMU_VENDOR__) && __riscv_xlen != 32 +#define SVPBMT_SHIFT 61 /* ISA SVPBMT */ + +#define PTE_VENDOR_PMA (0x0ULL << SVPBMT_SHIFT) +#define PTE_VENDOR_NOCACHE (0x1ULL << SVPBMT_SHIFT) +#define PTE_VENDOR_IO (0x2ULL << SVPBMT_SHIFT) +#define PTE_VENDOR_MTMASK (0x3ULL << SVPBMT_SHIFT) +#endif /* !__HAS_MMU_VENDOR__ && __riscv_xlen != 32 */ + +#ifndef PTE_VENDOR_PMA +#define PTE_VENDOR_PMA PTE_ATTR_NONE +#endif + +#ifndef PTE_VENDOR_NOCACHE +#define PTE_VENDOR_NOCACHE PTE_ATTR_NONE +#endif + +#ifndef PTE_VENDOR_IO +#define PTE_VENDOR_IO PTE_ATTR_NONE +#endif +#ifndef PTE_VENDOR_MTMASK +#define PTE_VENDOR_MTMASK PTE_ATTR_NONE +#endif + +#ifndef MMU_VENDOR_MAP_K_ROCB +#define MMU_VENDOR_MAP_K_ROCB ((PTE_ATTR_KERNEL & ~PTE_W) | PTE_VENDOR_PMA) +#endif +#ifndef MMU_VENDOR_MAP_K_RO +#define MMU_VENDOR_MAP_K_RO ((PTE_ATTR_KERNEL & ~PTE_W) | PTE_VENDOR_NOCACHE) +#endif +#ifndef MMU_VENDOR_MAP_K_RWCB +#define MMU_VENDOR_MAP_K_RWCB (PTE_ATTR_KERNEL | PTE_VENDOR_PMA) +#endif +#ifndef MMU_VENDOR_MAP_K_RW +#define MMU_VENDOR_MAP_K_RW (PTE_ATTR_KERNEL | PTE_VENDOR_NOCACHE) +#endif +#ifndef MMU_VENDOR_MAP_K_DEVICE +#define MMU_VENDOR_MAP_K_DEVICE ((PTE_ATTR_KERNEL & ~PTE_X) | PTE_VENDOR_IO) +#endif +#ifndef MMU_VENDOR_MAP_U_ROCB +#define MMU_VENDOR_MAP_U_ROCB ((PTE_ATTR_USER & ~PTE_W) | PTE_VENDOR_PMA) +#endif +#ifndef MMU_VENDOR_MAP_U_RO +#define MMU_VENDOR_MAP_U_RO ((PTE_ATTR_USER & ~PTE_W) | PTE_VENDOR_NOCACHE) +#endif +#ifndef MMU_VENDOR_MAP_U_RWCB +#define MMU_VENDOR_MAP_U_RWCB (PTE_ATTR_USER | PTE_VENDOR_PMA) +#endif +#ifndef MMU_VENDOR_MAP_U_RW +#define MMU_VENDOR_MAP_U_RW (PTE_ATTR_USER | PTE_VENDOR_NOCACHE) +#endif +#ifndef MMU_VENDOR_MAP_U_DEVICE +#define MMU_VENDOR_MAP_U_DEVICE ((PTE_ATTR_USER & ~PTE_X) | PTE_VENDOR_IO) +#endif +#ifndef MMU_VENDOR_MAP_EARLY +#define MMU_VENDOR_MAP_EARLY (PTE_RWX | PTE_G | PTE_V) +#endif + +#define MMU_MAP_K_ROCB MMU_VENDOR_MAP_K_ROCB +#define MMU_MAP_K_RO MMU_VENDOR_MAP_K_RO +#define MMU_MAP_K_RWCB MMU_VENDOR_MAP_K_RWCB +#define MMU_MAP_K_RW MMU_VENDOR_MAP_K_RW +#define MMU_MAP_K_DEVICE MMU_VENDOR_MAP_K_DEVICE +#define MMU_MAP_U_ROCB MMU_VENDOR_MAP_U_ROCB +#define MMU_MAP_U_RO MMU_VENDOR_MAP_U_RO +#define MMU_MAP_U_RWCB MMU_VENDOR_MAP_U_RWCB +#define MMU_MAP_U_RW MMU_VENDOR_MAP_U_RW +#define MMU_MAP_U_DEVICE MMU_VENDOR_MAP_U_DEVICE + +#define MMU_MAP_ERROR_VANOTALIGN -1 +#define MMU_MAP_ERROR_PANOTALIGN -2 +#define MMU_MAP_ERROR_NOPAGE -3 +#define MMU_MAP_ERROR_CONFLICT -4 + +#ifdef RT_PAGE_MPR_SIZE_DYNAMIC +#define ARCH_VADDR_WIDTH rt_hw_arch_vaddr_width +#elif __riscv_xlen == 32 +#define ARCH_VADDR_WIDTH 32 +#elif defined(ARCH_MM_MMU_SV48) +#define ARCH_VADDR_WIDTH 48 +#elif defined(ARCH_MM_MMU_SV57) +#define ARCH_VADDR_WIDTH 57 +#else +#define ARCH_VADDR_WIDTH 39 +#endif +#define ARCH_ADDRESS_WIDTH_BITS __riscv_xlen + +#define ARCH_MAP_FAILED _AT(void *, _ULL(-1)) + +#define ARCH_EARLY_MAP_SIZE 0x80000000 + +#ifndef __ASSEMBLY__ + +struct rt_aspace; + +extern rt_uint8_t rt_hw_arch_vaddr_width; + +void *rt_hw_mmu_tbl_get(void); +void rt_hw_mmu_tbl_set(rt_ubase_t tbl); +int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_ubase_t size, + rt_ubase_t *vtable, rt_ubase_t pv_off); +void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr); +void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size, + size_t attr); +void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size); +void rt_hw_aspace_switch(rt_aspace_t aspace); +void *rt_hw_mmu_v2p(rt_aspace_t aspace, void *vaddr); + +int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, + enum rt_mmu_cntl cmd); + +void *rt_hw_mmu_pgtbl_create(void); +void rt_hw_mmu_pgtbl_delete(void *pgtbl); + +#ifndef MMU_VENDOR_ATTR_RM_PERM_IMPL +/** + * @brief Remove permission from attribution + * + * @param attr architecture specified mmu attribution + * @param prot protect that will be removed + * @return size_t returned attribution + */ +rt_inline size_t rt_hw_mmu_attr_rm_perm(size_t attr, rt_base_t prot) +{ + switch (prot) + { + /* remove write permission for user */ + case RT_HW_MMU_PROT_WRITE | RT_HW_MMU_PROT_USER: + attr &= ~PTE_W; + break; + /* remove write permission for kernel */ + case RT_HW_MMU_PROT_WRITE | RT_HW_MMU_PROT_KERNEL: + attr &= ~PTE_W; + break; + default: + RT_ASSERT(0); + } + return attr; +} +#endif /* MMU_VENDOR_ATTR_RM_PERM_IMPL */ + +#ifndef MMU_VENDOR_ATTR_ADD_PERM_IMPL +/** + * @brief Add permission from attribution + * + * @param attr architecture specified mmu attribution + * @param prot protect that will be added + * @return size_t returned attribution + */ +rt_inline size_t rt_hw_mmu_attr_add_perm(size_t attr, rt_base_t prot) +{ + switch (prot) + { + /* add write permission for user */ + case RT_HW_MMU_PROT_WRITE | RT_HW_MMU_PROT_USER: + attr |= (PTE_R | PTE_W | PTE_U); + break; + default: + RT_ASSERT(0); + } + return attr; +} +#endif /* MMU_VENDOR_ATTR_ADD_PERM_IMPL */ + +#ifndef MMU_VENDOR_ATTR_TEST_PERM_IMPL +/** + * @brief Test permission from attribution + * + * @param attr architecture specified mmu attribution + * @param prot protect that will be test + * @return rt_bool_t RT_TRUE if the prot is allowed, otherwise RT_FALSE + */ +rt_inline rt_bool_t rt_hw_mmu_attr_test_perm(size_t attr, rt_base_t prot) +{ + rt_bool_t rc = 0; + + switch (prot & ~RT_HW_MMU_PROT_USER) + { + /* test write permission for user */ + case RT_HW_MMU_PROT_WRITE: + rc = ((attr & PTE_W) && (attr & PTE_R)); + break; + case RT_HW_MMU_PROT_READ: + rc = !!(attr & PTE_R); + break; + case RT_HW_MMU_PROT_EXECUTE: + rc = !!(attr & PTE_X); + break; + default: + RT_ASSERT(0); + } + + if (rc && (prot & RT_HW_MMU_PROT_USER)) + { + rc = !!(attr & PTE_U); + } + return rc; +} +#endif /* MMU_VENDOR_ATTR_TEST_PERM_IMPL */ + +#endif /* !__ASSEMBLY__ */ + +#endif /* __MMU_H__ */ diff --git a/libcpu/risc-v/common/include/opcode.h b/libcpu/risc-v/common/include/opcode.h new file mode 100644 index 0000000000..f98b529d48 --- /dev/null +++ b/libcpu/risc-v/common/include/opcode.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-11-09 Shell Add portable asm support + * 2025-04-26 GuEe-GUI Follow the specific and vendor + */ + +#ifndef __OPCODE_H__ +#define __OPCODE_H__ + +#include <rtcompiler.h> + +#include <asm-generic.h> +#ifdef __HAS_OPCODE_VENDOR__ +#include <opcode_vendor.h> +#endif + +/** + * @brief binary opcode pseudo operations + * Used to bypass toolchain restriction on extension ISA + * + */ + +/** + * @brief RISC-V instruction formats + */ + +#define OPCODE_MISC_MEM 15 +#define OPCODE_SYSTEM 115 + +#ifndef __OPC_INSN_FORMAT_R +/** + * R type: .insn r opcode6, func3, func7, rd, rs1, rs2 + * + * +-------+-----+-----+-------+----+---------+ + * | func7 | rs2 | rs1 | func3 | rd | opcode6 | + * +-------+-----+-----+-------+----+---------+ + * 31 25 20 15 12 7 0 + */ +#define __OPC_INSN_FORMAT_R(opcode, func3, func7, rd, rs1, rs2) \ + ".insn r "RT_STRINGIFY(opcode)","RT_STRINGIFY(func3)","RT_STRINGIFY(func7)","RT_STRINGIFY(rd)","RT_STRINGIFY(rs1)","RT_STRINGIFY(rs2)"\n\t" +#endif /* !__OPC_INSN_FORMAT_R */ + +#ifndef __OPC_INSN_FORMAT_I +/** + * I type: .insn i opcode6, func3, rd, rs1, simm12 + * + * +--------+-----+-------+----+---------+ + * | simm12 | rs1 | func3 | rd | opcode6 | + * +--------+-----+-------+----+---------+ + * 31 20 15 12 7 0 + */ +#define __OPC_INSN_FORMAT_I(opcode, func3, rd, rs1, simm12) \ + ".insn i "RT_STRINGIFY(opcode)","RT_STRINGIFY(func3)","RT_STRINGIFY(rd)","RT_STRINGIFY(rs1)","RT_STRINGIFY(simm12)"\n\t" +#endif /* !__OPC_INSN_FORMAT_I */ + +#ifndef OPC_CBO_INVAL +#define OPC_CBO_INVAL(base) __OPC_INSN_FORMAT_I(OPCODE_MISC_MEM, 2, x0, base, 0) +#endif + +#ifndef OPC_CBO_CLEAN +#define OPC_CBO_CLEAN(base) __OPC_INSN_FORMAT_I(OPCODE_MISC_MEM, 2, x0, base, 1) +#endif + +#ifndef OPC_FENCE_I +#ifdef ARCH_TOOLCHAIN_HAS_ZIFENCEI_ISA +#define OPC_FENCE_I __ASM_STR(fence.i) +#else /* !ARCH_TOOLCHAIN_HAS_ZIFENCEI_ISA */ +#define OPC_FENCE_I __ASM_STR(.long 0x0000100f) +#endif /* ARCH_TOOLCHAIN_HAS_ZIFENCEI_ISA */ +#endif /* !OPC_FENCE_I */ + +#ifndef OPC_PAUSE_I +#ifdef ARCH_TOOLCHAIN_HAS_ZIHINTPAUSE +#define OPC_PAUSE_I __ASM_STR(pause) +#else /* !ARCH_TOOLCHAIN_HAS_ZIHINTPAUSE */ +#define OPC_PAUSE_I __ASM_STR(.4byte 0x0100000f) +#endif /* ARCH_TOOLCHAIN_HAS_ZIHINTPAUSE */ +#endif /* !OPC_PAUSE_I */ + +#endif /* __OPCODE_H__ */
\ No newline at end of file diff --git a/libcpu/risc-v/common/include/page_def.h b/libcpu/risc-v/common/include/page_def.h new file mode 100644 index 0000000000..0b70d3084e --- /dev/null +++ b/libcpu/risc-v/common/include/page_def.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2025-04-20 GuEe-GUI The first version + */ + +#ifndef __PAGE_DEF_H__ +#define __PAGE_DEF_H__ + +#include <asm-generic.h> + +#ifndef ARCH_PAGE_SHIFT +#define ARCH_PAGE_SHIFT 12 +#elif ARCH_PAGE_SHIFT != 12 +#warning Make sure you fully understand what you are doing +#endif +#define ARCH_PAGE_SIZE (_UL(1) << ARCH_PAGE_SHIFT) +#define ARCH_PAGE_MASK (ARCH_PAGE_SIZE - 1) + +#if __riscv_xlen == 32 +#define SUPPER_PAGE_SIZE 0x400000 +#else +#define SUPPER_PAGE_SIZE 0x200000 +#endif +#define SUPPER_PAGE_MASK (SUPPER_PAGE_SIZE - 1) + +#endif /* __PAGE_DEF_H__ */ diff --git a/libcpu/risc-v/common/include/riscv_timer.h b/libcpu/risc-v/common/include/riscv_timer.h new file mode 100644 index 0000000000..d02e2170bf --- /dev/null +++ b/libcpu/risc-v/common/include/riscv_timer.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2018-10-28 Bernard The unify RISC-V porting code. + */ + +#ifndef __RISCV_TIMER_H__ +#define __RISCV_TIMER_H__ + +#include <rtdef.h> + +void riscv_timer_set_frequency(rt_uint32_t freq); +rt_uint32_t riscv_timer_get_frequency(void); + +rt_uint64_t riscv_timer_rdtime(void); +void riscv_timer_set_compare(rt_uint64_t value); + +void riscv_timer_isr(void); + +void riscv_timer_init(void); +void riscv_timer_probe(void *clint_timer_cmp, void *clint_timer_val); + +void riscv_timer_us_delay(rt_uint32_t us); + +#endif /* __RISCV_TIMER_H__ */ diff --git a/libcpu/risc-v/common/include/sbi.h b/libcpu/risc-v/common/include/sbi.h new file mode 100644 index 0000000000..2ed5aa3daa --- /dev/null +++ b/libcpu/risc-v/common/include/sbi.h @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2006-2021, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2021-05-18 Bernard port from FreeBSD + */ + +/*- + * Copyright (c) 2016-2017 Ruslan Bukin <[email protected]> + * All rights reserved. + * Copyright (c) 2019 Mitchell Horne <[email protected]> + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_SBI_H_ +#define _MACHINE_SBI_H_ + +#include <stdint.h> +#include <stdbool.h> +#include <rtdef.h> + +/* SBI Specification Version */ +#define SBI_SPEC_VERS_MAJOR_OFFSET 24 +#define SBI_SPEC_VERS_MAJOR_MASK (0x7F << SBI_SPEC_VERS_MAJOR_OFFSET) +#define SBI_SPEC_VERS_MINOR_OFFSET 0 +#define SBI_SPEC_VERS_MINOR_MASK (0xFFFFFF << SBI_SPEC_VERS_MINOR_OFFSET) + +/* SBI Implementation IDs */ +#define SBI_IMPL_ID_BBL 0 +#define SBI_IMPL_ID_OPENSBI 1 +#define SBI_IMPL_ID_XVISOR 2 +#define SBI_IMPL_ID_KVM 3 +#define SBI_IMPL_ID_RUSTSBI 4 +#define SBI_IMPL_ID_DIOSIX 5 + +/* SBI Error Codes */ +#define SBI_SUCCESS 0 +#define SBI_ERR_FAILURE -1 +#define SBI_ERR_NOT_SUPPORTED -2 +#define SBI_ERR_INVALID_PARAM -3 +#define SBI_ERR_DENIED -4 +#define SBI_ERR_INVALID_ADDRESS -5 +#define SBI_ERR_ALREADY_AVAILABLE -6 + +/* SBI Base Extension */ +#define SBI_EXT_ID_BASE 0x10 +#define SBI_BASE_GET_SPEC_VERSION 0 +#define SBI_BASE_GET_IMPL_ID 1 +#define SBI_BASE_GET_IMPL_VERSION 2 +#define SBI_BASE_PROBE_EXTENSION 3 +#define SBI_BASE_GET_MVENDORID 4 +#define SBI_BASE_GET_MARCHID 5 +#define SBI_BASE_GET_MIMPID 6 + +/* Timer (TIME) Extension */ +#define SBI_EXT_ID_TIME 0x54494D45 +#define SBI_TIME_SET_TIMER 0 + +/* IPI (IPI) Extension */ +#define SBI_EXT_ID_IPI 0x735049 +#define SBI_IPI_SEND_IPI 0 + +/* RFENCE (RFNC) Extension */ +#define SBI_EXT_ID_RFNC 0x52464E43 +#define SBI_RFNC_REMOTE_FENCE_I 0 +#define SBI_RFNC_REMOTE_SFENCE_VMA 1 +#define SBI_RFNC_REMOTE_SFENCE_VMA_ASID 2 +#define SBI_RFNC_REMOTE_HFENCE_GVMA_VMID 3 +#define SBI_RFNC_REMOTE_HFENCE_GVMA 4 +#define SBI_RFNC_REMOTE_HFENCE_VVMA_ASID 5 +#define SBI_RFNC_REMOTE_HFENCE_VVMA 6 + +#define SBI_EXT_ID_SRST 0x53525354 +#define SBI_SRST_FID_RESET 0 +#define SBI_SRST_RESET_TYPE_SHUTDOWN 0 +#define SBI_SRST_RESET_TYPE_COLD_REBOOT 1 +#define SBI_SRST_RESET_TYPE_WARM_REBOOT 2 +#define SBI_SRST_RESET_REASON_NONE 0 +#define SBI_SRST_RESET_REASON_SYS_FAILURE 1 + +/* Hart State Management (HSM) Extension */ +#define SBI_EXT_ID_HSM 0x48534D +#define SBI_HSM_HART_START 0 +#define SBI_HSM_HART_STOP 1 +#define SBI_HSM_HART_STATUS 2 +#define SBI_HSM_STATUS_STARTED 0 +#define SBI_HSM_STATUS_STOPPED 1 +#define SBI_HSM_STATUS_START_PENDING 2 +#define SBI_HSM_STATUS_STOP_PENDING 3 + +#define SBI_HART_MASK_BASE_ALL (~0UL) + +/* Legacy Extensions */ +#define SBI_SET_TIMER 0 +#define SBI_CONSOLE_PUTCHAR 1 +#define SBI_CONSOLE_GETCHAR 2 +#define SBI_CLEAR_IPI 3 +#define SBI_SEND_IPI 4 +#define SBI_REMOTE_FENCE_I 5 +#define SBI_REMOTE_SFENCE_VMA 6 +#define SBI_REMOTE_SFENCE_VMA_ASID 7 +#define SBI_SHUTDOWN 8 + +#define SBI_CALL0(e, f) SBI_CALL5(e, f, 0, 0, 0, 0, 0) +#define SBI_CALL1(e, f, p1) SBI_CALL5(e, f, p1, 0, 0, 0, 0) +#define SBI_CALL2(e, f, p1, p2) SBI_CALL5(e, f, p1, p2, 0, 0, 0) +#define SBI_CALL3(e, f, p1, p2, p3) SBI_CALL5(e, f, p1, p2, p3, 0, 0) +#define SBI_CALL4(e, f, p1, p2, p3, p4) SBI_CALL5(e, f, p1, p2, p3, p4, 0) +#define SBI_CALL5(e, f, p1, p2, p3, p4, p5) sbi_call(e, f, p1, p2, p3, p4, p5) + +/* + * Documentation available at + * https://github.com/riscv/riscv-sbi-doc/blob/master/riscv-sbi.adoc + */ + +struct sbi_ret +{ + long error; + long value; +}; + +rt_inline struct sbi_ret +sbi_call(uint64_t arg7, uint64_t arg6, uint64_t arg0, uint64_t arg1, + uint64_t arg2, uint64_t arg3, uint64_t arg4) +{ + struct sbi_ret ret; + + register uintptr_t a0 __asm("a0") = (uintptr_t)(arg0); + register uintptr_t a1 __asm("a1") = (uintptr_t)(arg1); + register uintptr_t a2 __asm("a2") = (uintptr_t)(arg2); + register uintptr_t a3 __asm("a3") = (uintptr_t)(arg3); + register uintptr_t a4 __asm("a4") = (uintptr_t)(arg4); + register uintptr_t a6 __asm("a6") = (uintptr_t)(arg6); + register uintptr_t a7 __asm("a7") = (uintptr_t)(arg7); + + __asm __volatile(\ + "ecall" \ + : "+r"(a0), "+r"(a1) \ + : "r"(a2), "r"(a3), "r"(a4), "r"(a6), "r"(a7) \ + : "memory"); + + ret.error = a0; + ret.value = a1; + return (ret); +} + +/* Base extension functions and variables. */ +extern unsigned long sbi_spec_version; +extern unsigned long sbi_impl_id; +extern unsigned long sbi_impl_version; + +extern bool has_time_extension; +extern bool has_ipi_extension; +extern bool has_rfnc_extension; +extern bool has_srst_extension; + +static __inline long +sbi_probe_extension(long id) +{ + return (SBI_CALL1(SBI_EXT_ID_BASE, SBI_BASE_PROBE_EXTENSION, id).value); +} + +/* TIME extension functions. */ +void sbi_set_timer(uint64_t val); + +/* IPI extension functions. */ +void sbi_send_ipi(const unsigned long *cpu_mask); + +/* RFENCE extension functions. */ +void sbi_remote_fence_i(const unsigned long *cpu_mask); +int sbi_remote_sfence_vma(const unsigned long *hart_mask, + const unsigned long hart_mask_base, + unsigned long start, unsigned long size); +void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start, + unsigned long size, unsigned long asid); + +/* Hart State Management extension functions. */ + +/* + * Start execution on the specified hart at physical address start_addr. The + * register a0 will contain the hart's ID, and a1 will contain the value of + * priv. + */ +int sbi_hsm_hart_start(unsigned long hart, unsigned long start_addr, unsigned long priv); + +/* + * Stop execution on the current hart. Interrupts should be disabled, or this + * function may return. + */ +void sbi_hsm_hart_stop(void); + +/* + * Get the execution status of the specified hart. The status will be one of: + * - SBI_HSM_STATUS_STARTED + * - SBI_HSM_STATUS_STOPPED + * - SBI_HSM_STATUS_START_PENDING + * - SBI_HSM_STATUS_STOP_PENDING + */ +int sbi_hsm_hart_status(unsigned long hart); + +/* + * Reboot the system. The type will be one of: + * - SBI_SRST_RESET_TYPE_WARM_REBOOT + * - SBI_SRST_RESET_TYPE_COLD_REBOOT + */ +void sbi_srst_reboot(unsigned long reset_type); + +/* + * Power off the system. + */ +void sbi_srst_power_off(void); + +/* Legacy extension functions. */ +static __inline void +sbi_console_putchar(int ch) +{ + (void)SBI_CALL1(SBI_CONSOLE_PUTCHAR, 0, ch); +} + +static __inline int +sbi_console_getchar(void) +{ + /* + * XXX: The "error" is returned here because legacy SBI functions + * continue to return their value in a0. + */ + return (SBI_CALL0(SBI_CONSOLE_GETCHAR, 0).error); +} + +static __inline void +sbi_shutdown(void) +{ + (void)SBI_CALL0(SBI_SHUTDOWN, 0); +} + +void sbi_print_version(void); +void sbi_init(void); + +#endif /* !_MACHINE_SBI_H_ */ diff --git a/libcpu/risc-v/common/include/setup.h b/libcpu/risc-v/common/include/setup.h new file mode 100644 index 0000000000..8f9f432c76 --- /dev/null +++ b/libcpu/risc-v/common/include/setup.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-02-21 GuEe-GUI first version + */ + +#ifndef __SETUP_H__ +#define __SETUP_H__ + +void rt_hw_fdt_install_early(void *fdt); +void rt_hw_common_setup(void); + +#endif /* __SETUP_H__ */ diff --git a/libcpu/risc-v/common/include/stack.h b/libcpu/risc-v/common/include/stack.h new file mode 100644 index 0000000000..c7d531eea0 --- /dev/null +++ b/libcpu/risc-v/common/include/stack.h @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2006-2021, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2021-01-30 lizhirui first version + * 2021-11-18 JasonHu add fpu member + * 2022-10-22 Shell Support kernel mode RVV + */ + +#ifndef __STACK_H__ +#define __STACK_H__ + +#include <rtconfig.h> + +#include <context_ext.h> + +#ifndef __ASSEMBLY__ +#include <rtdef.h> +#endif + +#ifdef ARCH_RISCV_FPU + #define K_XSTATUS_EXT SR_FS +#else + #define K_XSTATUS_EXT 0 +#endif + +#ifdef ARCH_RISCV_M_MODE + #define K_XSTATUS_DEFAULT_BASE (SR_PP | SR_PIE | K_XSTATUS_EXT) +#else + #define K_XSTATUS_DEFAULT_BASE (SR_PP | SR_PIE | SR_SUM | K_XSTATUS_EXT) +#endif + +#ifdef ARCH_RISCV_VECTOR + #define K_XSTATUS_DEFAULT (K_XSTATUS_DEFAULT_BASE | SR_VS) +#else + #define K_XSTATUS_DEFAULT K_XSTATUS_DEFAULT_BASE +#endif + +#ifdef ARCH_RISCV_32E + #define CTX_GENERAL_REG_NR 18 + #define FRAME_OFF_SP (17 * SZREG) +#else + #define CTX_GENERAL_REG_NR 34 + #define FRAME_OFF_SP (33 * SZREG) +#endif +#define CTX_ALL_REG_NR (CTX_GENERAL_REG_NR + CTX_FPU_REG_NR + CTX_VECTOR_REG_NR) + +#define FRAME_OFF_XSTATUS (2 * SZREG) +#define FRAME_OFF_XCAUSE (3 * SZREG) +#define FRAME_OFF_GP (4 * SZREG) +#define FRAME_OFF_TP (5 * SZREG) +#define FRAME_OFF_A7 (18 * SZREG) + +/* Switch frame */ +#define RT_HW_SWITCH_CONTEXT_XSTATUS 0 +#define RT_HW_SWITCH_CONTEXT_S1 1 +#define RT_HW_SWITCH_CONTEXT_S0 2 +#define RT_HW_SWITCH_CONTEXT_RA 3 +#define RT_HW_SWITCH_CONTEXT_TP 4 +#ifdef ARCH_RISCV_32E +#define RT_HW_SWITCH_CONTEXT_SIZE 8 /* Padding for alignment */ +#else +#define RT_HW_SWITCH_CONTEXT_S2 5 +#define RT_HW_SWITCH_CONTEXT_S3 6 +#define RT_HW_SWITCH_CONTEXT_S4 7 +#define RT_HW_SWITCH_CONTEXT_S5 8 +#define RT_HW_SWITCH_CONTEXT_S6 9 +#define RT_HW_SWITCH_CONTEXT_S7 10 +#define RT_HW_SWITCH_CONTEXT_S8 11 +#define RT_HW_SWITCH_CONTEXT_S9 12 +#define RT_HW_SWITCH_CONTEXT_S10 13 +#define RT_HW_SWITCH_CONTEXT_S11 14 +#define RT_HW_SWITCH_CONTEXT_SIZE 16 /* Padding for alignment */ +#endif + +#ifndef __ASSEMBLY__ +typedef struct rt_hw_switch_frame +{ + rt_ubase_t regs[RT_HW_SWITCH_CONTEXT_SIZE]; +} *rt_hw_switch_frame_t; + +RT_STATIC_ASSERT(riscv_switch_frame_size, + sizeof(struct rt_hw_switch_frame) == RT_HW_SWITCH_CONTEXT_SIZE * sizeof(rt_ubase_t)); + +#ifdef ARCH_RISCV_USING_IRQ_STACK +void riscv_irq_stack_init(void); +void *riscv_irq_stack_top(void); +#endif + +struct rt_hw_stack_frame +{ + rt_ubase_t epc; /* epc - epc - program counter */ + rt_ubase_t ra; /* x1 - ra - return address for jumps */ + rt_ubase_t xstatus; /* - supervisor/machine status register */ + rt_ubase_t xcause; /* - supervisor/machine cause register */ + rt_ubase_t gp; /* x3 - gp - global pointer */ + rt_ubase_t tp; /* x4 - tp - thread pointer */ + rt_ubase_t t0; /* x5 - t0 - temporary register 0 */ + rt_ubase_t t1; /* x6 - t1 - temporary register 1 */ + rt_ubase_t t2; /* x7 - t2 - temporary register 2 */ + rt_ubase_t s0_fp; /* x8 - s0/fp - saved register 0 or frame pointer */ + rt_ubase_t s1; /* x9 - s1 - saved register 1 */ + rt_ubase_t a0; /* x10 - a0 - return value or function argument 0 */ + rt_ubase_t a1; /* x11 - a1 - return value or function argument 1 */ + rt_ubase_t a2; /* x12 - a2 - function argument 2 */ + rt_ubase_t a3; /* x13 - a3 - function argument 3 */ + rt_ubase_t a4; /* x14 - a4 - function argument 4 */ + rt_ubase_t a5; /* x15 - a5 - function argument 5 */ +#ifndef ARCH_RISCV_32E + rt_ubase_t a6; /* x16 - a6 - function argument 6 */ + rt_ubase_t a7; /* x17 - s7 - function argument 7 */ + rt_ubase_t s2; /* x18 - s2 - saved register 2 */ + rt_ubase_t s3; /* x19 - s3 - saved register 3 */ + rt_ubase_t s4; /* x20 - s4 - saved register 4 */ + rt_ubase_t s5; /* x21 - s5 - saved register 5 */ + rt_ubase_t s6; /* x22 - s6 - saved register 6 */ + rt_ubase_t s7; /* x23 - s7 - saved register 7 */ + rt_ubase_t s8; /* x24 - s8 - saved register 8 */ + rt_ubase_t s9; /* x25 - s9 - saved register 9 */ + rt_ubase_t s10; /* x26 - s10 - saved register 10 */ + rt_ubase_t s11; /* x27 - s11 - saved register 11 */ + rt_ubase_t t3; /* x28 - t3 - temporary register 3 */ + rt_ubase_t t4; /* x29 - t4 - temporary register 4 */ + rt_ubase_t t5; /* x30 - t5 - temporary register 5 */ + rt_ubase_t t6; /* x31 - t6 - temporary register 6 */ +#endif /* !ARCH_RISCV_32E */ + + rt_ubase_t user_sp_exc_stack; /* sscratch - user mode sp/exception stack */ + +#ifdef ARCH_RISCV_FPU + rt_ubase_t f[CTX_FPU_REG_NR]; /* f0~f31 */ +#endif +#ifdef ARCH_RISCV_VECTOR + rt_ubase_t v[CTX_VECTOR_REG_NR]; +#endif +}; + +RT_STATIC_ASSERT(riscv_trap_frame_size, + sizeof(struct rt_hw_stack_frame) == CTX_ALL_REG_NR * sizeof(rt_ubase_t)); +#endif /* __ASSEMBLY__ */ + +#endif /* __STACK_H__ */ diff --git a/libcpu/risc-v/common/include/tlb.h b/libcpu/risc-v/common/include/tlb.h new file mode 100644 index 0000000000..72e3029e08 --- /dev/null +++ b/libcpu/risc-v/common/include/tlb.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2006-2022, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-11-28 WangXiaoyao the first version + * 2025-04-26 GuEe-GUI Follow the specific and vendor + */ + +#ifndef __TLB_H__ +#define __TLB_H__ + +#include <rtthread.h> + +#include <csr.h> +#include <sbi.h> +#include <page_def.h> +#include <mm_aspace.h> + +#ifdef ARCH_USING_ASID +/* Implemented ASID bits, discovered before secondary CPUs are started. */ +extern rt_ubase_t riscv_asid_mask; +#endif + +#define SFENCE_VMA_ASID(asid) \ + __asm__ volatile ("sfence.vma x0, %0"::"r"(asid) : "memory"); + +#define SFENCE_VMA_ADDR_INV(start) \ + __asm__ volatile ("sfence.vma %0, zero" ::"r"(start) : "memory"); + +rt_inline void rt_hw_tlb_invalidate_all(void) +{ +#ifdef RT_USING_SMP + unsigned long mask = 0; +#endif + + __asm__ volatile ("sfence.vma" ::: "memory"); +#ifdef RT_USING_SMP + /* RFENCE completes on all target harts before returning. */ + if (sbi_remote_sfence_vma(&mask, SBI_HART_MASK_BASE_ALL, 0, -1UL) != SBI_SUCCESS) + { + /* Continuing could reuse memory still reachable through a stale TLB. */ + RT_ASSERT(0); + while (1) + { + } + } +#endif +} + +rt_inline void rt_hw_tlb_invalidate_all_local(void) +{ + __asm__ volatile ("sfence.vma" ::: "memory"); +} + +rt_inline void rt_hw_tlb_invalidate_aspace(rt_aspace_t aspace) +{ +#if defined(RT_USING_SMP) + rt_hw_tlb_invalidate_all(); +#elif defined(ARCH_RISCV_DISABLE_SFENCE_VMA_ASID) || !defined(ARCH_USING_ASID) + rt_hw_tlb_invalidate_all_local(); +#else + if (riscv_asid_mask) + { + rt_ubase_t asid_value = aspace->asid & riscv_asid_mask; + + SFENCE_VMA_ASID(asid_value); + } + else + { + rt_hw_tlb_invalidate_all_local(); + } +#endif +} + +rt_inline void rt_hw_tlb_invalidate_page(rt_aspace_t aspace, void *start) +{ + RT_UNUSED(aspace); +#ifdef RT_USING_SMP + RT_UNUSED(start); + rt_hw_tlb_invalidate_all(); +#else + SFENCE_VMA_ADDR_INV(start); +#endif +} + +rt_inline void rt_hw_tlb_invalidate_range(rt_aspace_t aspace, void *start, + size_t size, size_t stride) +{ + /* huge page is taking as normal page */ + if (size <= ARCH_PAGE_SIZE) + { + rt_hw_tlb_invalidate_page(aspace, start); + } + else + { + rt_hw_tlb_invalidate_aspace(aspace); + } +} + +#endif /* __TLB_H__ */ diff --git a/libcpu/risc-v/common/interrupt.c b/libcpu/risc-v/common/interrupt.c new file mode 100644 index 0000000000..c571616e18 --- /dev/null +++ b/libcpu/risc-v/common/interrupt.c @@ -0,0 +1,551 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2018-10-01 Bernard The first version + * 2025-04-20 GuEe-GUI Port for not DM + */ + +#include <rthw.h> +#include <rtthread.h> + +#include <cpu.h> +#include <interrupt.h> +#include <board.h> + +#ifndef INTC_TYPE +#define INTC_TYPE INTC_TYPE_RISCV +#endif + +#ifndef MAX_HANDLERS +#define MAX_HANDLERS 128 +#endif +#ifdef ARCH_MM_MMU +#include <ioremap.h> +#else +static void *riscv_ioremap_identity(void *addr, rt_size_t size) +{ + RT_UNUSED(size); + + return addr; +} + +static void riscv_iounmap_identity(void *addr) +{ + RT_UNUSED(addr); +} + +#define rt_ioremap(addr, size) riscv_ioremap_identity((addr), (size)) +#define rt_iounmap(addr) riscv_iounmap_identity(addr) +#endif + +#define PIC_DISABLE_DM + +/* + * The DM and non-DM paths share the controller register logic. PIC_DISABLE_DM + * keeps the included drivers free of OFW and rt_pic dependencies; this file + * supplies the classic rt_hw_interrupt_* adapter around their raw helpers. + */ +#ifdef ARCH_RISCV_M_MODE +#include "pic/pic-riscv-clint.c" +#undef DBG_TAG +#undef DBG_LVL +#endif + +#if INTC_TYPE != INTC_TYPE_CLIC +/* --------------------------- PLIC and Local INTC -------------------------- */ + +#include "pic/pic-sifive-plic.c" +#undef DBG_TAG +#undef DBG_LVL +#include "pic/pic-riscv-intc.c" +#undef DBG_TAG +#undef DBG_LVL + +#ifdef RT_USING_SMP +int ipi_intc_base; +#endif +int irq_plic_base; + +static struct plic *system_plic; +size_t plic_base; + +rt_weak rt_ubase_t platform_get_plic_base(void) +{ + return 0; +} + +rt_weak rt_size_t platform_get_plic_size(void) +{ + return 0; +} + +rt_weak rt_ubase_t platform_get_plic_quirks(void) +{ + return 0; +} + +#else /* INTC_TYPE == INTC_TYPE_CLIC */ +/* ---------------------------------- CLIC ---------------------------------- */ + +/* + * CLIC can be used alongside INTC and PLIC in a SoC. + * However, we typically assume that NOT DM systems are MCUs, + * and they only use CLIC for interrupt handling. + */ +#include "pic/pic-riscv-clic.c" +#undef DBG_TAG +#undef DBG_LVL + +#ifdef RT_USING_SMP +int ipi_clic_base; +#endif +static struct riscv_clic *system_clic; + +rt_weak rt_ubase_t platform_get_clic_base(void) +{ + return 0; +} + +rt_weak rt_size_t platform_get_clic_size(void) +{ + return 0; +} + +rt_weak rt_ubase_t platform_get_clic_quirks(void) +{ + return 0; +} + +#endif /* INTC_TYPE != INTC_TYPE_CLIC */ + +#ifdef ARCH_RISCV_M_MODE +rt_weak rt_ubase_t platform_get_clint_base(void) +{ + return 0; +} + +rt_weak rt_size_t platform_get_clint_size(void) +{ + return 0; +} +#endif + +#define DBG_TAG "hw.irq" +#define DBG_LVL DBG_INFO + +static struct rt_irq_desc irq_desc[MAX_HANDLERS]; + +static rt_bool_t riscv_is_boot_cpu(void) +{ +#ifdef RT_USING_SMP + return rt_hw_cpu_id() == riscv_hartid_to_cpu_id(boot_cpu_hartid); +#else + return RT_TRUE; +#endif +} + +#ifdef ARCH_RISCV_M_MODE +static void riscv_clint_init(void) +{ + rt_ubase_t mmio_range[2] = + { + platform_get_clint_base(), + platform_get_clint_size(), + }; + + if (!mmio_range[0] && !mmio_range[1]) + { + return; + } + + if (!mmio_range[0] || !mmio_range[1] || + riscv_clint_common_init(mmio_range, 1) != RT_EOK) + { + LOG_E("Failed to initialize CLINT at %p", (void *)mmio_range[0]); + } +} +#endif + +#if INTC_TYPE != INTC_TYPE_CLIC +static rt_err_t riscv_irq_controller_init(void) +{ + rt_ubase_t base; + rt_size_t size; + + riscv_intc_common_init(&system_rv_intc, INTC_TYPE == INTC_TYPE_ANDESTECH); + + irq_plic_base = system_rv_intc.irqs_def_count + system_rv_intc.irqs_count; +#ifdef RT_USING_SMP + ipi_intc_base = irq_plic_base - RT_MAX_IPI; +#endif + +#ifdef ARCH_RISCV_M_MODE + riscv_clint_init(); +#endif + + RT_ASSERT(irq_plic_base < MAX_HANDLERS); + if (irq_plic_base >= MAX_HANDLERS) + { + LOG_E("MAX_HANDLERS is too small for local interrupts"); + return -RT_EINVAL; + } + + base = platform_get_plic_base(); + size = platform_get_plic_size(); + if (!base || !size) + { + LOG_E("PLIC MMIO range is not configured"); + return -RT_EINVAL; + } + + system_plic = plic_common_init(base, size, + MAX_HANDLERS - irq_plic_base, + platform_get_plic_quirks()); + + RT_ASSERT(system_plic != RT_NULL); + return system_plic ? RT_EOK : -RT_EIO; +} + +static void riscv_irq_controller_percpu_init(void) +{ + if (!system_plic) + { + return; + } + + plic_percpu_common_init(system_plic); + rt_hw_interrupt_umask(IRQ_INTC(RV_IRQ_EXT)); +} + +#else /* INTC_TYPE == INTC_TYPE_CLIC */ +static rt_err_t riscv_irq_controller_init(void) +{ + rt_ubase_t base; + rt_size_t size; + +#ifdef ARCH_RISCV_M_MODE + riscv_clint_init(); +#endif + + base = platform_get_clic_base(); + size = platform_get_clic_size(); + if (!base || !size) + { + LOG_E("CLIC MMIO range is not configured"); + return -RT_EINVAL; + } + + system_clic = riscv_clic_common_init(base, size, + platform_get_clic_quirks()); + + RT_ASSERT(system_clic != RT_NULL); + if (!system_clic) + { + return -RT_EIO; + } + +#ifdef RT_USING_SMP + ipi_clic_base = system_clic->ipi_base; +#endif + + return RT_EOK; +} + +static void riscv_irq_controller_percpu_init(void) +{ + if (system_clic) + { + riscv_clic_percpu_common_init(system_clic); + } +} +#endif /* INTC_TYPE != INTC_TYPE_CLIC */ + +rt_weak void rt_hw_interrupt_init(void) +{ + if (riscv_is_boot_cpu() && riscv_irq_controller_init() != RT_EOK) + { + return; + } + + riscv_irq_controller_percpu_init(); +} + +rt_weak void rt_hw_interrupt_mask(int vector) +{ + if (vector < 0 || vector >= MAX_HANDLERS) + { + return; + } + +#if INTC_TYPE != INTC_TYPE_CLIC +/* --------------------------- PLIC and Local INTC -------------------------- */ + + if (vector >= irq_plic_base) + { + if (!system_plic) + { + return; + } + + vector -= irq_plic_base; + + LOG_D("PLIC IRQ %d %s on CPU#%d", vector, "disabled", rt_hw_cpu_id()); + plic_irq_mask_raw(rt_hw_cpu_id(), vector); + } + else + { + #if INTC_TYPE == INTC_TYPE_RISCV + riscv_intc_irq_mask_raw(&system_rv_intc, vector); + #else + andes_intc_irq_mask_raw(&system_rv_intc, vector); + #endif + } + +#else /* INTC_TYPE == INTC_TYPE_CLIC */ +/* ---------------------------------- CLIC ---------------------------------- */ + + if (system_clic) + { + riscv_clic_irq_mask_raw(system_clic, vector); + } + +#endif /* INTC_TYPE != INTC_TYPE_CLIC */ +} + +rt_weak void rt_hw_interrupt_umask(int vector) +{ + if (vector < 0 || vector >= MAX_HANDLERS) + { + return; + } + +#if INTC_TYPE != INTC_TYPE_CLIC +/* --------------------------- PLIC and Local INTC -------------------------- */ + + if (vector >= irq_plic_base) + { + if (!system_plic) + { + return; + } + + vector -= irq_plic_base; + + LOG_D("PLIC IRQ %d %s on CPU#%d", vector, "enabled", rt_hw_cpu_id()); + plic_irq_unmask_raw(rt_hw_cpu_id(), vector); + } + else + { + #if INTC_TYPE == INTC_TYPE_RISCV + riscv_intc_irq_unmask_raw(&system_rv_intc, vector); + #else + andes_intc_irq_unmask_raw(&system_rv_intc, vector); + #endif + } + +#else /* INTC_TYPE == INTC_TYPE_CLIC */ +/* ---------------------------------- CLIC ---------------------------------- */ + + if (system_clic) + { + riscv_clic_irq_unmask_raw(system_clic, vector); + } + +#endif /* INTC_TYPE != INTC_TYPE_CLIC */ +} + +rt_weak rt_isr_handler_t rt_hw_interrupt_install(int vector, rt_isr_handler_t handler, + void *param, const char *name) +{ + rt_isr_handler_t old_handler = RT_NULL; + + if (vector >= 0 && vector < MAX_HANDLERS) + { + old_handler = irq_desc[vector].handler; + + if (handler != RT_NULL) + { + irq_desc[vector].handler = (rt_isr_handler_t)handler; + irq_desc[vector].param = param; + #ifdef RT_USING_INTERRUPT_INFO + rt_snprintf(irq_desc[vector].name, RT_NAME_MAX - 1, "%s", + name ? name : "irq"); + irq_desc[vector].counter = 0; + #endif + } + } + + return old_handler; +} + +#ifdef RT_USING_SMP +rt_weak void rt_hw_ipi_handler_install(int ipi_hwirq, rt_isr_handler_t ipi_isr_handler) +{ + rt_hw_interrupt_install(ipi_hwirq, ipi_isr_handler, RT_NULL, "IPI_HANDLER"); +} + +rt_weak void rt_hw_ipi_send(int ipi_vector, unsigned int cpu_mask) +{ + RT_BITMAP_DECLARE(cpu_masks, RT_CPUS_NR) = { cpu_mask }; + +#if INTC_TYPE != INTC_TYPE_CLIC + riscv_intc_irq_send_ipi_raw(&system_rv_intc, + ipi_intc_base + ipi_vector, cpu_masks); +#else + if (system_clic) + { + riscv_clic_irq_send_ipi_raw(system_clic, + ipi_clic_base + ipi_vector, cpu_masks); + } +#endif +} +#endif /* RT_USING_SMP */ + +static rt_bool_t handler_irq(int hwirq, void *data) +{ + rt_ubase_t irq = hwirq + (rt_ubase_t)data; + struct rt_irq_desc *desc; + + if (irq >= MAX_HANDLERS) + { + return RT_FALSE; + } + + desc = &irq_desc[irq]; + + if (desc->handler) + { + desc->handler(hwirq, desc->param); + + return RT_TRUE; + } + + return RT_FALSE; +} + +#if INTC_TYPE != INTC_TYPE_CLIC +static rt_bool_t handler_plic_irq(int hwirq, void *data) +{ + plic_irq_ack_or_eoi(rt_hw_cpu_id(), hwirq); + + return handler_irq(hwirq, data); +} + +void plic_init(void) +{ + plic_base = platform_get_plic_base(); +} + +void plic_set_priority(int irq, int priority) +{ + if (system_plic) + { + HWREG32(system_plic->regs + PRIORITY_BASE + irq * PRIORITY_PER_ID) = priority; + } +} + +void plic_irq_enable(int irq) +{ + if (system_plic) + { + plic_irq_unmask_raw(rt_hw_cpu_id(), irq); + } + + csr_set(CSR_IE, BIT(RV_IRQ_EXT)); +} + +void plic_irq_disable(int irq) +{ + if (system_plic) + { + plic_irq_mask_raw(rt_hw_cpu_id(), irq); + } +} + +void plic_set_thresh(rt_uint32_t val) +{ + if (system_plic) + { + plic_set_threshold_raw(&plic_percpu[rt_hw_cpu_id()], val); + } +} + +void plic_set_threshold(int threshold) +{ + plic_set_thresh(threshold); +} + +int plic_claim(void) +{ + if (!system_plic) + { + return 0; + } + + return HWREG32(plic_percpu[rt_hw_cpu_id()].hart_base + CONTEXT_CLAIM); +} + +void plic_complete(int irq) +{ + if (system_plic) + { + plic_irq_ack_or_eoi(rt_hw_cpu_id(), irq); + } +} + +void plic_set_ie(rt_uint32_t word_index, rt_uint32_t val) +{ + if (system_plic) + { + HWREG32(plic_percpu[rt_hw_cpu_id()].enable_base + word_index * sizeof(rt_uint32_t)) = val; + } +} + +void plic_handle_irq(void) +{ + if (system_plic) + { + plic_handler_isr_raw(system_plic, handler_plic_irq, (void *)(rt_ubase_t)irq_plic_base); + } +} +#endif /* INTC_TYPE != INTC_TYPE_CLIC */ + +rt_weak void rt_hw_interrupt_handle(rt_uint32_t vector, void *param) +{ + /* vector is equal to scause from trap */ + RT_UNUSED(param); + +#if INTC_TYPE != INTC_TYPE_CLIC +/* --------------------------- PLIC and Local INTC -------------------------- */ + + if (vector == RV_IRQ_EXT) + { + if (system_plic) + { + plic_handler_isr_raw(system_plic, handler_plic_irq, + (void *)(rt_ubase_t)irq_plic_base); + } + } + else + { + #if INTC_TYPE == INTC_TYPE_RISCV + riscv_intc_handler_raw(&system_rv_intc, vector, handler_irq, RT_NULL); + #else + riscv_intc_aia_handler_raw(&system_rv_intc, handler_irq, RT_NULL); + #endif + } + +#else /* INTC_TYPE == INTC_TYPE_CLIC */ +/* ---------------------------------- CLIC ---------------------------------- */ + + if (system_clic) + { + riscv_clic_handler_isr_raw(system_clic, vector, handler_irq, RT_NULL); + } + +#endif /* INTC_TYPE != INTC_TYPE_CLIC */ +} diff --git a/libcpu/risc-v/common/irq_stack.c b/libcpu/risc-v/common/irq_stack.c new file mode 100644 index 0000000000..1d323a8488 --- /dev/null +++ b/libcpu/risc-v/common/irq_stack.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2006-2026, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include <rthw.h> +#include <rtthread.h> + +#include <cpu.h> +#include <csr.h> +#include <stack.h> + +#define RISCV_IRQ_STACK_ALIGN 16 +#define RISCV_IRQ_STACK_STRIDE RT_ALIGN(ARCH_RISCV_IRQ_STACK_SIZE, RISCV_IRQ_STACK_ALIGN) + +static rt_uint8_t irq_stacks[RT_CPUS_NR][RISCV_IRQ_STACK_STRIDE] + rt_align(RISCV_IRQ_STACK_ALIGN); + +void riscv_irq_stack_init(void) +{ + csr_write(CSR_SCRATCH, riscv_irq_stack_top()); +} + +void *riscv_irq_stack_top(void) +{ + int cpu_id = rt_hw_cpu_id(); + + RT_ASSERT(cpu_id >= 0 && cpu_id < RT_CPUS_NR); + if (cpu_id < 0 || cpu_id >= RT_CPUS_NR) + { + cpu_id = 0; + } + + return &irq_stacks[cpu_id][RISCV_IRQ_STACK_STRIDE]; +} diff --git a/libcpu/risc-v/common/isa_ext.c b/libcpu/risc-v/common/isa_ext.c new file mode 100644 index 0000000000..907adfc488 --- /dev/null +++ b/libcpu/risc-v/common/isa_ext.c @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2025-04-20 GuEe-GUI The first version + */ + +#include <rthw.h> + +#include <cpuport.h> +#include <isa_ext.h> + +#define RISCV_ISA_WORD_BITS (sizeof(rt_ubase_t) * 8) +#define RISCV_ISA_WORDS ((RISCV_ISA_EXT_MAX + RISCV_ISA_WORD_BITS - 1) / \ + RISCV_ISA_WORD_BITS) + +static rt_ubase_t isa_map[RISCV_ISA_WORDS]; + +void riscv_isa_ext_set_raw(rt_uint32_t isa) +{ + if (isa < RISCV_ISA_EXT_MAX) + { + isa_map[isa / RISCV_ISA_WORD_BITS] |= + (rt_ubase_t)1 << (isa % RISCV_ISA_WORD_BITS); + rt_hw_wmb(); + } +} + +void riscv_isa_ext_clear_raw(rt_uint32_t isa) +{ + if (isa < RISCV_ISA_EXT_MAX) + { + isa_map[isa / RISCV_ISA_WORD_BITS] &= + ~((rt_ubase_t)1 << (isa % RISCV_ISA_WORD_BITS)); + rt_hw_wmb(); + } +} + +rt_bool_t riscv_isa_ext_test_raw(rt_uint32_t isa) +{ + rt_hw_rmb(); + + return isa < RISCV_ISA_EXT_MAX && + !!(isa_map[isa / RISCV_ISA_WORD_BITS] & + ((rt_ubase_t)1 << (isa % RISCV_ISA_WORD_BITS))); +} diff --git a/libcpu/risc-v/common/mmu.c b/libcpu/risc-v/common/mmu.c new file mode 100644 index 0000000000..8a15c24f2d --- /dev/null +++ b/libcpu/risc-v/common/mmu.c @@ -0,0 +1,1423 @@ +/* + * Copyright (c) 2006-2025 RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2021-01-30 lizhirui first version + * 2022-12-13 WangXiaoyao Port to new mm + * 2023-10-12 Shell Add permission control API + * 2025-04-26 GuEe-GUI Follow the specific and vendor + */ + +#include <rthw.h> +#include <rtthread.h> + +#define DBG_TAG "hw.mmu" +#define DBG_LVL DBG_INFO +#include <rtdbg.h> + +#include <cpu.h> +#include <csr.h> +#include <mmu.h> +#include <tlb.h> +#include <cache.h> +#include <ioremap.h> +#include <isa_ext.h> +#include <mm_page.h> +#include <mm_aspace.h> +#include <board.h> +#ifdef RT_USING_SMART +#include <lwp_user_mm.h> +#endif + +#ifndef ARCH_TEXT_OFFSET +#if __riscv_xlen == 64 +#define ARCH_TEXT_OFFSET 0x200000 +#elif __riscv_xlen == 32 +#define ARCH_TEXT_OFFSET 0x400000 +#else +#error "Unexpected __riscv_xlen" +#endif +#endif + +#ifndef ARCH_RAM_OFFSET +#define ARCH_RAM_OFFSET 0 +#endif + +#ifndef KERNEL_VADDR_START +#ifdef KERNEL_ASPACE_START +#define KERNEL_VADDR_START KERNEL_ASPACE_START +#else +#define KERNEL_VADDR_START (ARCH_RAM_OFFSET + ARCH_TEXT_OFFSET) +#endif +#endif + +#if defined(RT_USING_SMP) && RT_CPUS_NR > 1 +#define MMU_PERCPU +#endif + +#if __riscv_xlen == 32 +#define VPN_BITS 10 +#define VPN_MASK 0x3ffUL +#define PPN_MASK 0xfff00000UL +#define LEVEL_MAX 1 +rt_weak rt_uint8_t rt_hw_arch_vaddr_width = 32; +#else +#define VPN_BITS 9 +#define VPN_MASK 0x1ffUL +#define PPN_MASK 0x3ffffffffffc00UL +#define LEVEL_MAX 5 +rt_weak rt_uint8_t rt_hw_arch_vaddr_width = 39; +#endif + +struct mmu_level_info +{ + rt_ubase_t *pos; + void *page; +}; + +#ifdef MMU_PERCPU +struct mmu_percpu_level_info +{ + rt_ubase_t offset; + rt_ubase_t *page; +}; + +#define PERCPU_OFFSET rt_hw_cpu_id() * ARCH_PAGE_SIZE +static struct mmu_percpu_level_info percpu_level[RT_CPUS_NR][LEVEL_MAX + 1]; +#else +#define PERCPU_OFFSET 0 +#endif + +rt_ubase_t MMUTable[(ARCH_PAGE_SIZE / sizeof(rt_ubase_t)) * RT_CPUS_NR] + rt_align(ARCH_PAGE_SIZE); + +static rt_uint8_t satp_level_4k, satp_level_2M; +static rt_ubase_t satp_mode; + +struct page_table +{ + rt_ubase_t page[ARCH_PAGE_SIZE / sizeof(rt_ubase_t)]; +}; +static struct page_table *__init_page_array; + +rt_inline rt_ubase_t mmu_attr_fixup(rt_ubase_t attr) +{ +#if !defined(__HAS_MMU_VENDOR__) && __riscv_xlen != 32 + if (!riscv_isa_ext_test(SVPBMT)) + { + attr &= ~PTE_VENDOR_MTMASK; + } +#endif + + return attr; +} + +void set_free_page(void *page_array) +{ + __init_page_array = page_array; +} + +rt_ubase_t get_free_page(void) +{ + static rt_ubase_t page_off; + + if (page_off < ARCH_PAGE_SIZE / sizeof(rt_ubase_t)) + { + return (rt_ubase_t)__init_page_array[page_off++].page; + } + + return 0; +} + +static int mmu_map_early(rt_ubase_t *tbl, rt_uint8_t max_level, + rt_ubase_t va, rt_ubase_t pa, rt_ubase_t attr, rt_bool_t flush) +{ + rt_ubase_t page, *table, off, level_shift; + + attr = mmu_attr_fixup(attr); + + if (max_level == satp_level_2M) + { + if ((va & SUPPER_PAGE_MASK) || (pa & SUPPER_PAGE_MASK)) + { + return MMU_MAP_ERROR_VANOTALIGN; + } + } + else + { + if ((va & ARCH_PAGE_MASK) || (pa & ARCH_PAGE_MASK)) + { + return MMU_MAP_ERROR_VANOTALIGN; + } + } + + table = tbl; + level_shift = rt_hw_arch_vaddr_width - VPN_BITS; + + for (int level = 0; level < max_level; ++level) + { + off = (va >> level_shift) & VPN_MASK; + + if (!(table[off] & PTE_V)) + { + if (!(page = get_free_page())) + { + return MMU_MAP_ERROR_NOPAGE; + } + + rt_memset((void *)page, 0, ARCH_PAGE_SIZE); + table[off] = ((page >> ARCH_PAGE_SHIFT) << PTE_BITS) | PTE_V; + + if (flush) + { + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *)); + } + } + + if ((table[off] & PTE_ATTR_RWX) != 0) + { + /* No a page! */ + return MMU_MAP_ERROR_CONFLICT; + } + + /* Step into the next level page table */ + page = (table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT; + table = (rt_ubase_t *)page; + + level_shift -= VPN_BITS; + } + + off = (va >> level_shift) & VPN_MASK; + table[off] = ((pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | attr; + + if (flush) + { + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *)); + } + + return 0; +} + +#ifdef MMU_PERCPU +static int mmu_map_percpu_early(rt_ubase_t *tbl, rt_ubase_t *tbl_shadow, + rt_ubase_t va, rt_ubase_t pa) +{ + rt_ubase_t page, off, level_shift; + + if (tbl == tbl_shadow) + { + return mmu_map_early(tbl, satp_level_4k, va, pa, MMU_MAP_K_RWCB, RT_FALSE); + } + + level_shift = rt_hw_arch_vaddr_width - VPN_BITS; + + for (int level = 0; level < satp_level_4k; ++level) + { + off = (va >> level_shift) & VPN_MASK; + + if (!(tbl[off] & PTE_V) || (tbl[off] & PTE_ATTR_RWX)) + { + return MMU_MAP_ERROR_CONFLICT; + } + + /* Copy a shared branch before changing its leaf mapping. */ + if (tbl_shadow[off] == tbl[off]) + { + if (!(page = get_free_page())) + { + return MMU_MAP_ERROR_NOPAGE; + } + + rt_memcpy((void *)page, + (void *)((tbl[off] >> PTE_BITS) << ARCH_PAGE_SHIFT), + ARCH_PAGE_SIZE); + tbl_shadow[off] = ((page >> ARCH_PAGE_SHIFT) << PTE_BITS) | PTE_V; + } + else if (!(tbl_shadow[off] & PTE_V) || (tbl_shadow[off] & PTE_ATTR_RWX)) + { + return MMU_MAP_ERROR_CONFLICT; + } + + /* Step into the next level page table */ + tbl = (rt_ubase_t *)((tbl[off] >> PTE_BITS) << ARCH_PAGE_SHIFT); + tbl_shadow = (rt_ubase_t *)((tbl_shadow[off] >> PTE_BITS) << ARCH_PAGE_SHIFT); + + level_shift -= VPN_BITS; + } + + off = (va >> level_shift) & VPN_MASK; + tbl_shadow[off] = ((pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | MMU_MAP_K_RWCB; + + return 0; +} +#endif /* MMU_PERCPU */ + +/** + * @brief Switch the current address space to the specified one. + * + * This function is responsible for switching the address space by updating the page table + * and related hardware state. The behavior depends on whether the architecture supports + * Address Space Identifiers (ASIDs), devided by macro definition of ARCH_USING_ASID. + * + * @param aspace Pointer to the address space structure containing the new page table. + * + * @note If ASID is supported (`ARCH_USING_ASID` is defined), the ASID is encoded + * into `satp`. Otherwise, switching address spaces invalidates the local TLB. + */ +#ifdef ARCH_USING_ASID +/* Generation bits stay above the architectural ASID field. */ +#define ASID_FIRST_GENERATION (_ULL(1) << SATP_ASID_BITS) + +rt_ubase_t riscv_asid_mask; + +static rt_spinlock_t asid_lock = RT_SPINLOCK_INIT; + +static void asid_init(void) +{ + rt_ubase_t old_satp = csr_read(CSR_SATP); + + /* Probe with the early page table active, preserving MODE and PPN. */ + csr_write(CSR_SATP, old_satp | (SATP_ASID_MASK << SATP_ASID_SHIFT)); + riscv_asid_mask = (csr_read(CSR_SATP) >> SATP_ASID_SHIFT) & SATP_ASID_MASK; + csr_write(CSR_SATP, old_satp); + + /* The probe may have populated translations under the temporary ASID. */ + rt_hw_tlb_invalidate_all_local(); +} + +static rt_uint64_t _asid_acquire(rt_aspace_t aspace) +{ + static rt_uint32_t next_asid = 1; + static rt_uint64_t global_asid_generation = ASID_FIRST_GENERATION; + rt_ubase_t level; + + level = rt_spin_lock_irqsave(&asid_lock); + + if ((aspace->asid ^ global_asid_generation) >> SATP_ASID_BITS) + { + if (next_asid <= riscv_asid_mask) + { + aspace->asid = global_asid_generation | next_asid++; + } + else + { + global_asid_generation += ASID_FIRST_GENERATION; + next_asid = 1; + rt_hw_tlb_invalidate_all(); + + aspace->asid = global_asid_generation | next_asid++; + } + } + + rt_spin_unlock_irqrestore(&asid_lock, level); + + return aspace->asid & riscv_asid_mask; +} + +void rt_hw_aspace_switch(rt_aspace_t aspace) +{ + rt_ubase_t asid, page_table; + + page_table = (rt_ubase_t)rt_kmem_v2p(aspace->page_table + PERCPU_OFFSET); + + asid = riscv_asid_mask ? _asid_acquire(aspace) : 0; + + csr_write(CSR_SATP, satp_mode | (asid << SATP_ASID_SHIFT) | + (page_table >> ARCH_PAGE_SHIFT)); + +#ifdef ARCH_RISCV_DISABLE_SFENCE_VMA_ASID + rt_hw_tlb_invalidate_all_local(); +#else + if (riscv_asid_mask) + { + SFENCE_VMA_ASID(asid); + } + else + { + rt_hw_tlb_invalidate_all_local(); + } +#endif +} + +#else /* !ARCH_USING_ASID */ +void rt_hw_aspace_switch(rt_aspace_t aspace) +{ + rt_ubase_t page_table; + + page_table = (rt_ubase_t)rt_kmem_v2p(aspace->page_table + PERCPU_OFFSET); + + csr_write(CSR_SATP, satp_mode | ((rt_ubase_t)page_table >> ARCH_PAGE_SHIFT)); + + rt_hw_tlb_invalidate_all_local(); +} +#endif /* ARCH_USING_ASID */ + +void rt_hw_mmu_tbl_set(rt_ubase_t tbl) +{ + tbl += PERCPU_OFFSET; + tbl += rt_kmem_pvoff(); + + csr_write(CSR_SATP, satp_mode | (tbl >> ARCH_PAGE_SHIFT)); + + rt_hw_tlb_invalidate_all_local(); +} + +void *rt_hw_mmu_tbl_get(void) +{ + rt_ubase_t satp = csr_read(CSR_SATP); + + return (void *)((satp & SATP_PPN) << ARCH_PAGE_SHIFT) - rt_kmem_pvoff(); +} + +void *rt_ioremap_early(void *paddr, size_t size) +{ + rt_ubase_t base; + volatile size_t count; + static void *tbl = RT_NULL; + + if (!size) + { + return RT_NULL; + } + + if (!tbl) + { + tbl = rt_hw_mmu_tbl_get(); + } + + base = RT_ALIGN_DOWN((rt_size_t)paddr, SUPPER_PAGE_SIZE); + count = RT_ALIGN((rt_ubase_t)paddr + size - base, SUPPER_PAGE_SIZE) / SUPPER_PAGE_SIZE; + + while (count --> 0) + { + if (mmu_map_early(tbl, satp_level_2M, base, base, MMU_MAP_K_DEVICE, RT_TRUE)) + { + return RT_NULL; + } + + base += SUPPER_PAGE_SIZE; + } + + return paddr; +} + +#ifdef MMU_PERCPU +static int _map_percpu_area(rt_ubase_t *table, void *va, void *pa, int cpu_id) +{ + rt_ubase_t page, off, level_shift; + + level_shift = rt_hw_arch_vaddr_width - VPN_BITS; + + for (int level = 0; level < satp_level_4k; ++level) + { + off = ((rt_ubase_t)va >> level_shift) & VPN_MASK; + + if (!(table[off] & PTE_V)) + { + if (!(page = (rt_ubase_t)rt_pages_alloc(0))) + { + return MMU_MAP_ERROR_NOPAGE; + } + + rt_memset((void *)page, 0, ARCH_PAGE_SIZE); + + table[off] = (((page + rt_kmem_pvoff()) >> ARCH_PAGE_SHIFT) << PTE_BITS) | PTE_V; + + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *)); + } + else if (table[off] & PTE_ATTR_RWX) + { + return MMU_MAP_ERROR_CONFLICT; + } + + percpu_level[cpu_id][level].page = table; + percpu_level[cpu_id][level].offset = off; + + /* Step into the next level page table */ + table = (rt_ubase_t *)(((table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT) - rt_kmem_pvoff()); + + level_shift -= VPN_BITS; + } + + off = ((rt_ubase_t)va >> level_shift) & VPN_MASK; + table[off] = (((rt_ubase_t)pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | MMU_MAP_K_RWCB; + + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *)); + + percpu_level[cpu_id][satp_level_4k].page = table; + percpu_level[cpu_id][satp_level_4k].offset = off; + + return ARCH_PAGE_SIZE; +} + +static void percpu_map_sync(struct rt_aspace *aspace, int level, rt_ubase_t *table, rt_ubase_t off) +{ + if (aspace == &rt_kernel_space && + percpu_level[0][level].page == table && + percpu_level[0][level].offset != off) + { + rt_ubase_t *table_shadow; + + for (int i = 1; i < RT_CPUS_NR; ++i) + { + table_shadow = percpu_level[i][level].page; + table_shadow[off] = table[off]; + + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table_shadow + off, sizeof(void *)); + } + } +} + +static void user_root_sync(struct rt_aspace *aspace, int level, + rt_ubase_t *table, rt_ubase_t off) +{ + if (aspace != &rt_kernel_space && level == 0 && table == aspace->page_table) + { + for (int i = 1; i < RT_CPUS_NR; ++i) + { + rt_ubase_t *table_shadow = (void *)aspace->page_table + i * ARCH_PAGE_SIZE; + + table_shadow[off] = table[off]; + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table_shadow + off, + sizeof(*table_shadow)); + } + } +} + +static void pgtbl_entry_sync(struct rt_aspace *aspace, int level, + rt_ubase_t *table, rt_ubase_t off) +{ + percpu_map_sync(aspace, level, table, off); + user_root_sync(aspace, level, table, off); +} + +static void percpu_mmu_init_check(void) +{ + size_t mapped, size; + void *page_table, *vaddr, *paddr; + static rt_bool_t inited = RT_FALSE; + + if (inited) + { + return; + } + inited = RT_TRUE; + + page_table = rt_kernel_space.page_table; + + for (int cpu_id = 0; cpu_id < RT_CPUS_NR; ++cpu_id) + { + vaddr = &__percpu_start; + paddr = vaddr + rt_kmem_pvoff(); + size = (size_t)(&__percpu_end - &__percpu_start); + /* Offset to per-CPU partition for current CPU */ + paddr += size * cpu_id; + + while (size > 0) + { + MM_PGTBL_LOCK(&rt_kernel_space); + mapped = _map_percpu_area(page_table, vaddr, paddr, cpu_id); + MM_PGTBL_UNLOCK(&rt_kernel_space); + + RT_ASSERT(mapped > 0); + + size -= mapped; + vaddr += mapped; + paddr += mapped; + } + + page_table += ARCH_PAGE_SIZE; + } +} +#endif /* MMU_PERCPU */ + +static size_t _map_area(struct rt_aspace *aspace, void *va, void *pa, + size_t attr, size_t max_size) +{ + rt_uint8_t max_level; + rt_ubase_t page, *table, off, level_shift; + + attr = mmu_attr_fixup(attr); + + if (max_size >= SUPPER_PAGE_SIZE && + !((rt_ubase_t)va & SUPPER_PAGE_MASK) && + !((rt_ubase_t)pa & SUPPER_PAGE_MASK)) + { + max_level = satp_level_2M; + max_size = SUPPER_PAGE_SIZE; + } + else + { + max_level = satp_level_4k; + max_size = ARCH_PAGE_SIZE; + } + + table = aspace->page_table; + level_shift = rt_hw_arch_vaddr_width - VPN_BITS; + + for (int level = 0; level < max_level; ++level) + { + off = ((rt_ubase_t)va >> level_shift) & VPN_MASK; + + if (!(table[off] & PTE_V)) + { + if (!(page = (rt_ubase_t)rt_pages_alloc(0))) + { + return MMU_MAP_ERROR_NOPAGE; + } + + rt_memset((void *)page, 0, ARCH_PAGE_SIZE); + table[off] = (((page + rt_kmem_pvoff()) >> ARCH_PAGE_SHIFT) << PTE_BITS) | PTE_V; + + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *)); + } + else + { + page = (table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT; + page -= rt_kmem_pvoff(); + + rt_page_ref_inc((void *)page, 0); + } + + if ((table[off] & PTE_ATTR_RWX) != 0) + { + /* No a page! */ + return MMU_MAP_ERROR_CONFLICT; + } + + #ifdef MMU_PERCPU + pgtbl_entry_sync(aspace, level, table, off); + #endif + + /* Step into the next level page table */ + page = (table[off] >> PTE_BITS) << ARCH_PAGE_SHIFT; + table = (rt_ubase_t *)(page - rt_kmem_pvoff()); + + level_shift -= VPN_BITS; + } + + off = ((rt_ubase_t)va >> level_shift) & VPN_MASK; + table[off] = (((rt_ubase_t)pa >> ARCH_PAGE_SHIFT) << PTE_BITS) | attr; + + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, table + off, sizeof(void *)); + +#ifdef MMU_PERCPU + pgtbl_entry_sync(aspace, max_level, table, off); +#endif + + return max_size; +} + +static size_t _unmap_area(struct rt_aspace *aspace, void *va, size_t max_size) +{ + int level, ref; + size_t unmapped; + rt_ubase_t entry, page, *table, *pos, off, level_shift; + struct mmu_level_info level_info[LEVEL_MAX + 1]; + + rt_memset(level_info, 0, sizeof(level_info)); + + table = aspace->page_table; + level_shift = rt_hw_arch_vaddr_width - VPN_BITS; + + /* Max level is 4k */ + for (level = 0; level <= satp_level_4k; ++level) + { + off = ((rt_ubase_t)va >> level_shift) & VPN_MASK; + pos = table + off; + entry = *pos; + unmapped = 1UL << (ARCH_PAGE_SHIFT + + VPN_BITS * (satp_level_4k - level)); + + if (!(entry & PTE_V)) + { + return max_size < unmapped ? max_size : unmapped; + } + + level_info[level].pos = pos; + + if ((entry & PTE_ATTR_RWX) != 0) + { + break; + } + + if (level == satp_level_4k) + { + return 0; + } + + /* Step into the next level page table */ + page = (entry >> PTE_BITS) << ARCH_PAGE_SHIFT; + table = (rt_ubase_t *)(page - rt_kmem_pvoff()); + + level_info[level].page = table; + + level_shift -= VPN_BITS; + } + + if (unmapped > max_size) + { + return 0; + } + + pos = level_info[level].pos; + *pos = 0; + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, pos, sizeof(void *)); +#ifdef MMU_PERCPU + pgtbl_entry_sync(aspace, level, pos - off, off); +#endif + --level; + + while (level >= 0) + { + pos = level_info[level].pos; + + if (pos) + { + void *cur_page = level_info[level].page; + + ref = rt_page_ref_get(cur_page, 0); + + if (ref == 1) + { + *pos = 0; + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, pos, sizeof(void *)); + + #ifdef MMU_PERCPU + if (level == 0) + { + table = aspace->page_table; + } + else + { + table = percpu_level[0][level].page; + } + + if (pos >= table && pos < table + (ARCH_PAGE_SIZE / sizeof(rt_ubase_t))) + { + pgtbl_entry_sync(aspace, level, table, pos - table); + } + #endif /* MMU_PERCPU */ + } + + if (ref == 1) + { + /* Finish hardware page walks before recycling this table. */ + rt_hw_tlb_invalidate_all(); + } + rt_pages_free(cur_page, 0); + } + else + { + break; + } + + --level; + } + + return unmapped; +} + +/** + * @brief Maps a virtual address space to a physical address space. + * + * This function maps a specified range of virtual addresses to a range of physical addresses + * and sets the attributes of the page table entries (PTEs). If an error occurs during the + * mapping process, the function will automatically roll back any partially completed mappings. + * + * @param aspace Pointer to the address space structure containing the page table information. + * @param v_addr The starting virtual address to be mapped. + * @param p_addr The starting physical address to be mapped. + * @param size The size of the memory to be mapped (in bytes). + * @param attr The attributes of the page table entries (e.g., read/write permissions, cache policies). + * + * @return On success, returns the starting virtual address `v_addr`; + * On failure, returns `NULL`. + * + * @note This function will not override existing page table entries. + * @warning The caller must ensure that `v_addr` and `p_addr` are page-aligned, + * and `size` is a multiple of the page size. + * + */ +void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr, + size_t size, size_t attr) +{ + size_t mapped; + void *unmap_va = v_addr; + +#ifdef MMU_PERCPU + percpu_mmu_init_check(); +#endif + + while (size > 0) + { + #ifdef MMU_PERCPU + if (v_addr < (void *)&__percpu_start || + v_addr >= (void *)&__percpu_real_end) + #endif + { + MM_PGTBL_LOCK(aspace); + mapped = _map_area(aspace, v_addr, p_addr, attr, size); + MM_PGTBL_UNLOCK(aspace); + } + #ifdef MMU_PERCPU + else + { + /* Make the kernel skip the per-CPU as soon as possible */ + mapped = size > SUPPER_PAGE_SIZE ? SUPPER_PAGE_SIZE : size; + } + #endif /* MMU_PERCPU */ + + if (mapped && mapped <= size) + { + size -= mapped; + v_addr += mapped; + p_addr += mapped; + + continue; + } + + /* Error, undo map */ + while (unmap_va != v_addr) + { + MM_PGTBL_LOCK(aspace); + mapped = _unmap_area(aspace, unmap_va, + (rt_ubase_t)v_addr - (rt_ubase_t)unmap_va); + MM_PGTBL_UNLOCK(aspace); + + if (!mapped) + { + RT_ASSERT(mapped != 0); + break; + } + unmap_va += mapped; + } + + rt_hw_tlb_invalidate_all(); + return RT_NULL; + } + + return unmap_va; +} + +/** + * @brief Unmaps a range of virtual memory addresses from the specified address space. + * + * This function is responsible for unmapping a contiguous region of virtual memory + * from the given address space. It handles multiple pages and ensures thread safety + * by locking the page table during the unmapping operation. + * + * @param aspace Pointer to the address space structure from which the memory will be unmapped. + * @param v_addr Starting virtual address to unmap. Must be page-aligned. + * @param size Size of the memory region to unmap. Must be page-aligned. + * + * @note The caller must ensure that both `v_addr` and `size` are page-aligned. + * + * @details The function operates in a loop, unmapping memory in chunks. It uses the + * `_unmap_area` function to perform the actual unmapping, which is called within a + * locked section to ensure thread safety. The loop continues until the entire region + * is unmapped. + * + * @see _unmap_area + * @note unmap is different from map that it can handle multiple pages + */ +void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size) +{ + size_t unmapped; + + /* caller guarantee that v_addr & size are page aligned */ + if (!aspace->page_table) + { + return; + } + + while (size > 0) + { + MM_PGTBL_LOCK(aspace); + unmapped = _unmap_area(aspace, v_addr, size); + MM_PGTBL_UNLOCK(aspace); + + if (unmapped && unmapped <= size) + { + size -= unmapped; + v_addr += unmapped; + + continue; + } + + /* Region not exist in pgtbl */ + break; + } +} + +rt_inline void _init_region(void *vaddr, size_t size) +{ + rt_ioremap_start = vaddr; + rt_ioremap_size = size; + rt_mpr_start = rt_ioremap_start - rt_mpr_size; + LOG_D("rt_ioremap_start: %p, rt_mpr_start: %p", rt_ioremap_start, + rt_mpr_start); +} + +rt_inline rt_ubase_t get_top_vpn(rt_ubase_t va, rt_uint8_t satp_level) +{ + return (va >> (VPN_BITS * satp_level + 12)) & VPN_MASK; +} + +/** + * @brief Initialize the MMU (Memory Management Unit) mapping. + * + * This function initializes the MMU mapping, incluing these steps as follows: + * 1. Check the validity of the input parameters, + * 2. Calculate the start and end virtual addresses based on the input virtual address and size. + * 3. Convert the virtual addresses to PPN2 indices. + * 4. Check the initialization of the page table. If any entry in the page table within + * the specified range is non-zero, it returns -1. + * 5. It initializes the kernel address space using rt_aspace_init() and initializes the specified region + * using _init_region. + * + * @param aspace Pointer to the address space. Must not be NULL. + * @param v_address The starting virtual address. + * @param size The size of the virtual address space. + * @param vtable Pointer to the page table. Must not be NULL. + * @param pv_off The page table offset. + * + * @return Returns 0 if the initialization is successful. Returns -1 if any input parameter is invalid + * or the page table initialization check fails. + */ +int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_ubase_t size, + rt_ubase_t *vtable, rt_ubase_t pv_off) +{ + rt_size_t off, va_s, va_e; + rt_ubase_t aspace_start = KERNEL_VADDR_START; + +#if !defined(RT_USING_SMART) && ARCH_RAM_OFFSET + aspace_start = ARCH_RAM_OFFSET; +#endif + + if ((!aspace) || (!vtable)) + { + return -1; + } + + va_s = (rt_ubase_t)v_address; + va_e = (rt_ubase_t)v_address + size - 1; + + if (va_e < va_s) + { + return -1; + } + + /* convert address to top-level index */ + va_s = get_top_vpn(va_s, satp_level_4k); + va_e = get_top_vpn(va_e, satp_level_4k); + + if (va_s == 0) + { + return -1; + } + + /* vtable initialization check */ + for (off = va_s; off <= va_e; off++) + { + rt_size_t v = vtable[off]; + + if (v) + { + return -1; + } + } + +#if __riscv_xlen == 32 && !defined(RT_USING_SMART) + /* The RV32 ioremap and MPR regions are placed below the kernel image. */ + aspace_start = (rt_ubase_t)v_address - rt_mpr_size; +#endif + + rt_aspace_init(aspace, (void *)aspace_start, 0UL - aspace_start, vtable); + + _init_region(v_address, size); + + return 0; +} + +static void *mmu_v2p(void *tbl, void *vaddr, int *out_level, rt_base_t **out_pte) +{ + rt_ubase_t page, entry, off, level_shift, pa; + rt_ubase_t va = (rt_ubase_t)vaddr; + rt_ubase_t *table = tbl; + +#ifdef MMU_PERCPU + if (tbl == rt_kernel_space.page_table) + { + table = (rt_ubase_t *)(tbl + PERCPU_OFFSET); + } +#endif /* MMU_PERCPU */ + + /* Calculate initial shift based on virtual address width */ + level_shift = rt_hw_arch_vaddr_width - VPN_BITS; + + /* 4k is the max level, translate to the last level */ + for (*out_level = 0; *out_level <= satp_level_4k; ++(*out_level)) + { + /* Extract VPN (Virtual Page Number) for this level */ + off = (va >> level_shift) & VPN_MASK; + /* Fetch the page table entry */ + entry = table[off]; + + /* If entry is invalid (V bit not set), address translation fails */ + if (!(entry & PTE_V)) + { + goto _translate_fail; + } + + /* If entry has any RWX permission bits set, it is a leaf entry */ + if (entry & PTE_ATTR_RWX) + { + if (out_pte) + { + *out_pte = (rt_base_t *)table + off; + } + + /* Extract physical page base address and add page offset */ + pa = ((entry & PTE_PPN_MASK) >> PTE_BITS) << ARCH_PAGE_SHIFT; + + if (*out_level >= satp_level_4k) + { + pa |= va & ARCH_PAGE_MASK; + } + else + { + pa |= va & ((1UL << level_shift) - 1); + } + + return (void *)pa; + } + + /* Otherwise, walk to the next level page table */ + page = (entry >> PTE_BITS) << ARCH_PAGE_SHIFT; + table = (rt_ubase_t *)(page - rt_kmem_pvoff()); + + level_shift -= VPN_BITS; + } + +_translate_fail: + /* If reached here, it is not a valid mapped address */ + LOG_D("%s: failed at %p", __func__, vaddr); + return ARCH_MAP_FAILED; +} + +/** + * @brief Translate a virtual address to a physical address. + * + * This function translates a given virtual address (`vaddr`) to its corresponding + * physical address (`paddr`) using the page table in the specified address space (`aspace`). + * + * @param aspace Pointer to the address space structure containing the page table. + * @param vaddr The virtual address to be translated. + * + * @return The translated physical address. If the translation fails, `ARCH_MAP_FAILED` is returned. + * + * @note The function queries the page table entry (PTE) for the virtual address using `mmu_v2p`. + * If a valid PTE is found, the physical address is extracted and combined with the offset + * from the virtual address. If no valid PTE is found, a debug log is recorded, and + * `ARCH_MAP_FAILED` is returned. + */ +void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr) +{ + int level; + + return mmu_v2p(aspace->page_table, vaddr, &level, RT_NULL); +} + +static int _noncache(rt_base_t *pte) +{ + rt_ubase_t entry; + + if (!riscv_isa_ext_test(SVPBMT)) + { + return -RT_ENOSYS; + } + + entry = *pte; + *pte = (entry & ~PTE_VENDOR_MTMASK) | PTE_VENDOR_NOCACHE; + + return 0; +} + +static int _cache(rt_base_t *pte) +{ + rt_ubase_t entry; + + if (!riscv_isa_ext_test(SVPBMT)) + { + return -RT_ENOSYS; + } + + entry = *pte; + *pte = (entry & ~PTE_VENDOR_MTMASK) | PTE_VENDOR_PMA; + + return 0; +} + +static int (*control_handler[MMU_CNTL_DUMMY_END])(rt_base_t *pte)= +{ + [MMU_CNTL_CACHE] = _cache, + [MMU_CNTL_NONCACHE] = _noncache, +}; + +/** + * @brief Control the page table entries (PTEs) for a specified virtual address range. + * + * This function applies a control command (e.g., cache control) to the page table entries + * (PTEs) corresponding to the specified virtual address range (`vaddr` to `vaddr + size`). + * + * @param aspace Pointer to the address space structure containing the page table. + * @param vaddr The starting virtual address of the range. + * @param size The size of the virtual address range. + * @param cmd The control command to apply (e.g., `MMU_CNTL_CACHE`, `MMU_CNTL_NONCACHE`.etc.). + * + * @return `RT_EOK` on success, or an error code (`-RT_EINVAL` or `-RT_ENOSYS`) on failure. + * + * @note The function uses the `control_handler` array to map the command to a handler function. + * It iterates over the virtual address range, queries the PTEs, and applies the handler + * to each valid PTE. If the command is invalid, `-RT_ENOSYS` is returned. + */ +int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, + enum rt_mmu_cntl cmd) +{ + int level; + int err = -RT_EINVAL; + void *vend = vaddr + size; + + int (*handler)(rt_base_t *pte); + if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END) + { + handler = control_handler[cmd]; + + while (vaddr < vend) + { + void *range_end; + rt_base_t *pte = RT_NULL; + + mmu_v2p(aspace->page_table, vaddr, &level, &pte); + + if (level == satp_level_4k) + { + range_end = vaddr + ARCH_PAGE_SIZE; + } + else if (level == satp_level_2M) + { + range_end = vaddr + SUPPER_PAGE_SIZE; + } + else + { + /* 1G or larger is not supported. */ + RT_ASSERT(0); + } + + if (pte) + { + err = handler(pte); + RT_ASSERT(err == RT_EOK); + } + vaddr = range_end; + } + } + else + { + err = -RT_ENOSYS; + } + + return err; +} + +/** + * @brief setup Page Table for kernel space. It's a fixed map + * and all mappings cannot be changed after initialization. + * + * Memory region in struct mem_desc must be page aligned, + * otherwise is a failure and no report will be + * returned. + * + * @param aspace Pointer to the address space structure. + * @param mdesc Pointer to the array of memory descriptors. + * @param desc_nr Number of memory descriptors in the array. + */ +void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr) +{ + void *err; + + for (size_t i = 0; i < desc_nr; ++i) + { + size_t attr; + struct rt_mm_va_hint hint = + { + .flags = MMF_MAP_FIXED, + .limit_start = aspace->start, + .limit_range_size = aspace->size, + .map_size = mdesc->vaddr_end - mdesc->vaddr_start + 1, + .prefer = (void *)mdesc->vaddr_start + }; + + switch (mdesc->attr) + { + case NORMAL_MEM: + attr = MMU_MAP_K_RWCB; + break; + case NORMAL_NOCACHE_MEM: + attr = MMU_MAP_K_RWCB; + break; + case DEVICE_MEM: + attr = MMU_MAP_K_DEVICE; + break; + default: + attr = MMU_MAP_K_DEVICE; + } + + if (mdesc->paddr_start == (rt_uintptr_t)ARCH_MAP_FAILED) + { + mdesc->paddr_start = mdesc->vaddr_start + rt_kmem_pvoff(); + } + + rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr, + mdesc->paddr_start >> MM_PAGE_SHIFT, &err); + mdesc++; + } + +#ifdef ARCH_USING_ASID + asid_init(); +#endif + rt_hw_mmu_tbl_set((rt_ubase_t)rt_kernel_space.page_table); + rt_page_cleanup(); +} + +/** + * @brief Early memory setup function for hardware initialization. + * + * This function performs early memory setup tasks, including: + * - Calculating the physical-to-virtual (PV) offset. + * - Setting up initial page tables for identity mapping and text region relocation. + * - Applying new memory mappings by updating the SATP register. + * + * @note This function is typically called during the early stages of system initialization (startup_gcc.S), + * before the memory management system is fully operational. + */ +void rt_hw_mem_setup_early(rt_ubase_t vaddr_start, rt_ubase_t paddr_start, void *pgtbl) +{ + extern char __text_start, __text_end; + + rt_size_t size = ARCH_EARLY_MAP_SIZE; + rt_size_t map_pages; + rt_ubase_t *early_pgtbl = pgtbl, paddr_end; + rt_ubase_t kernel_paddr; + + rt_kmem_pvoff_set(paddr_start - vaddr_start); + + switch (rt_hw_arch_vaddr_width) + { + case 0: + satp_mode = SATP_MODE_BARE; + satp_level_4k = 0; + satp_level_2M = 0; + #ifdef RT_USING_SMART + RT_ASSERT(0); + #else + RT_ASSERT(paddr_start == vaddr_start); + #endif + /* Do nothing */ + return; +#if __riscv_xlen != 32 + case 64: + satp_mode = SATP_MODE_64; + satp_level_4k = 5; + satp_level_2M = 4; + break; + case 57: + satp_mode = SATP_MODE_57; + satp_level_4k = 4; + satp_level_2M = 3; + break; + case 48: + satp_mode = SATP_MODE_48; + satp_level_4k = 3; + satp_level_2M = 2; + break; + case 39: + default: + satp_mode = SATP_MODE_39; + satp_level_4k = 2; + satp_level_2M = 1; + rt_hw_arch_vaddr_width = 39; +#else + case 32: + default: + satp_mode = SATP_MODE_32; + satp_level_4k = 1; + satp_level_2M = 0; + rt_hw_arch_vaddr_width = 32; +#endif /* __riscv_xlen != 32 */ + break; + } + + map_pages = (size + (paddr_start & SUPPER_PAGE_MASK) + SUPPER_PAGE_MASK) / + SUPPER_PAGE_SIZE; + vaddr_start = RT_ALIGN_DOWN(vaddr_start, SUPPER_PAGE_SIZE); + paddr_start = RT_ALIGN_DOWN(paddr_start, SUPPER_PAGE_SIZE); + kernel_paddr = paddr_start; + + rt_memset(early_pgtbl, 0, RT_CPUS_NR * ARCH_PAGE_SIZE); + + if (!rt_kmem_pvoff()) + { + goto _map_vaddress; + } + + /* Identical mapping, only map for 'text' */ + size = RT_ALIGN((rt_ubase_t)(&__text_end - &__text_start), SUPPER_PAGE_SIZE); + + /* Check whether the virtual address area and identical area overlap */ + RT_ASSERT(size < rt_kmem_pvoff()); + + for (rt_ssize_t pages = size / SUPPER_PAGE_SIZE; pages > 0; --pages) + { + mmu_map_early(early_pgtbl, satp_level_2M, paddr_start, paddr_start, MMU_MAP_K_RWCB, RT_FALSE); + + paddr_start += SUPPER_PAGE_SIZE; + } + + /* Page array for map early */ + paddr_start = RT_ALIGN_DOWN((rt_ubase_t)early_pgtbl, SUPPER_PAGE_SIZE); + mmu_map_early(early_pgtbl, satp_level_2M, paddr_start, paddr_start, MMU_MAP_K_RWCB, RT_FALSE); + + if ((rt_ubase_t)__init_page_array - paddr_start >= SUPPER_PAGE_SIZE) + { + paddr_start = RT_ALIGN_DOWN((rt_ubase_t)__init_page_array, SUPPER_PAGE_SIZE); + mmu_map_early(early_pgtbl, satp_level_2M, paddr_start, paddr_start, MMU_MAP_K_RWCB, RT_FALSE); + } + +_map_vaddress: + /* Virtual mapping */ + paddr_start = kernel_paddr; + while (map_pages--) + { + #ifdef MMU_PERCPU + if (paddr_start < (rt_ubase_t)&__percpu_start || + paddr_start >= (rt_ubase_t)&__percpu_real_end) + #endif + { + mmu_map_early(early_pgtbl, satp_level_2M, vaddr_start, paddr_start, MMU_MAP_K_RWCB, RT_FALSE); + } + + rt_ubase_t next_paddr = paddr_start + SUPPER_PAGE_SIZE; + rt_ubase_t next_vaddr = vaddr_start + SUPPER_PAGE_SIZE; + + /* RV32 cannot represent an exclusive end address at 4 GiB. */ + if (next_paddr < paddr_start || next_vaddr < vaddr_start) + { + break; + } + + paddr_start = next_paddr; + vaddr_start = next_vaddr; + } + +#ifdef MMU_PERCPU + for (int cpu_id = 0; cpu_id < RT_CPUS_NR; ++cpu_id) + { + rt_ubase_t *tbl_shadow = early_pgtbl + + cpu_id * (ARCH_PAGE_SIZE / sizeof(*early_pgtbl)); + + if (cpu_id) + { + rt_memcpy(tbl_shadow, early_pgtbl, ARCH_PAGE_SIZE); + } + + paddr_start = (rt_ubase_t)&__percpu_start; + vaddr_start = paddr_start - rt_kmem_pvoff(); + size = (rt_size_t)(&__percpu_end - &__percpu_start); + /* Offset to per-CPU partition for current CPU */ + paddr_start += cpu_id * size; + paddr_end = paddr_start + size; + + while (paddr_start < paddr_end) + { + /* Map physical address per-CPU partition */ + RT_ASSERT(mmu_map_percpu_early(early_pgtbl, tbl_shadow, + vaddr_start, paddr_start) == 0); + + paddr_start += ARCH_PAGE_SIZE; + vaddr_start += ARCH_PAGE_SIZE; + } + } +#endif /* MMU_PERCPU */ +} + +rt_ubase_t mmu_satp_load(void *pgtbl, rt_ubase_t hartid) +{ +#ifdef MMU_PERCPU + pgtbl += riscv_hartid_to_cpu_id(hartid) * ARCH_PAGE_SIZE; +#endif + + return satp_mode | ((rt_ubase_t)pgtbl >> ARCH_PAGE_SHIFT); +} + +/** + * @brief Creates and initializes a new MMU page table. + * + * This function allocates a new MMU page table, copies the kernel space + * page table into it, and flushes the data cache to ensure consistency. + * + * @return + * - A pointer to the newly allocated MMU page table on success. + * - RT_NULL if the allocation fails. + */ +void *rt_hw_mmu_pgtbl_create(void) +{ + rt_size_t alloc_bits = 0; + rt_ubase_t *mmu_table; + +#ifdef MMU_PERCPU + alloc_bits = rt_page_bits(RT_CPUS_NR * ARCH_PAGE_SIZE); +#endif + mmu_table = (rt_ubase_t *)rt_pages_alloc_ext(alloc_bits, PAGE_ANY_AVAILABLE); + if (!mmu_table) + { + return RT_NULL; + } + +#ifdef MMU_PERCPU + for (int i = 0; i < RT_CPUS_NR; ++i) + { + rt_memcpy((void *)mmu_table + i * ARCH_PAGE_SIZE, + (void *)rt_kernel_space.page_table + i * ARCH_PAGE_SIZE, + ARCH_PAGE_SIZE); + } + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_table, + RT_CPUS_NR * ARCH_PAGE_SIZE); +#else + rt_memcpy(mmu_table, rt_kernel_space.page_table, ARCH_PAGE_SIZE); + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_table, ARCH_PAGE_SIZE); +#endif + + return mmu_table; +} + +/** + * @brief Deletes an MMU page table. + * + * This function frees the memory allocated for the given MMU page table. + * + * @param pgtbl Pointer to the MMU page table to be deleted. + */ +void rt_hw_mmu_pgtbl_delete(void *pgtbl) +{ + rt_size_t alloc_bits = 0; + +#ifdef MMU_PERCPU + alloc_bits = rt_page_bits(RT_CPUS_NR * ARCH_PAGE_SIZE); +#endif + rt_pages_free(pgtbl, alloc_bits); +} diff --git a/libcpu/risc-v/common/pic/Kconfig b/libcpu/risc-v/common/pic/Kconfig new file mode 100644 index 0000000000..19d42bcd55 --- /dev/null +++ b/libcpu/risc-v/common/pic/Kconfig @@ -0,0 +1,24 @@ +config RT_PIC_RISCV_AIA + bool "RISC-V AIA (Advanced Interrupt Architecture)" + depends on RT_USING_PIC + select RT_USING_OFW + select RT_PIC_RISCV_INTC + help + with APLIC (Advanced Platform-Level Interrupt Controller) and + IMSIC (Incoming MSI Controller) + +config RT_PIC_RISCV_CLIC + bool "RISC-V CLIC (Core-Local Interrupt Controller)" + depends on RT_USING_PIC + select RT_USING_OFW + +config RT_PIC_RISCV_INTC + bool "RISC-V CPU interrupt controller" + depends on RT_USING_PIC + select RT_USING_OFW + +config RT_PIC_SIFIVE_PLIC + bool "SiFive PLIC (Platform-Level Interrupt Controller)" + depends on RT_USING_PIC + select RT_USING_OFW + select RT_PIC_RISCV_INTC diff --git a/libcpu/risc-v/common/pic/SConscript b/libcpu/risc-v/common/pic/SConscript new file mode 100644 index 0000000000..98a20a7868 --- /dev/null +++ b/libcpu/risc-v/common/pic/SConscript @@ -0,0 +1,29 @@ +from building import * + +group = [] + +if not GetDepend(['RT_USING_PIC']): + Return('group') + +src = [] +cwd = GetCurrentDir() +CPPPATH = [cwd] + +if GetDepend(['RT_PIC_RISCV_AIA']): + src += ['pic-riscv-aplic.c', 'pic-riscv-imsic.c'] + +if GetDepend(['RT_PIC_RISCV_CLIC']): + src += ['pic-riscv-clic.c'] + +if GetDepend(['RT_PIC_RISCV_INTC']): + src += ['pic-riscv-intc.c'] + +if GetDepend(['RT_PIC_RISCV_CLIC']) or GetDepend(['RT_PIC_RISCV_INTC']): + src += ['pic-riscv-clint.c'] + +if GetDepend(['RT_PIC_SIFIVE_PLIC']): + src += ['pic-sifive-plic.c'] + +group = DefineGroup('DeviceDrivers', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/libcpu/risc-v/common/pic/pic-riscv-aplic.c b/libcpu/risc-v/common/pic/pic-riscv-aplic.c new file mode 100644 index 0000000000..8cc1d39997 --- /dev/null +++ b/libcpu/risc-v/common/pic/pic-riscv-aplic.c @@ -0,0 +1,792 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-01-30 GuEe-GUI first version + */ + +#include <rthw.h> +#include <rtthread.h> +#include <rtdevice.h> + +#define DBG_TAG "pic.aplic" +#define DBG_LVL DBG_INFO +#include <rtdbg.h> + +#include <csr.h> +#include <cpu.h> +#include <drivers/pic.h> +#include "pic-riscv-imsic.h" + +#define APLIC_MAX_IDC RT_BIT(14) +#define APLIC_MAX_SOURCE 1024 + +#define APLIC_DOMAINCFG 0x0000 +#define APLIC_DOMAINCFG_RDONLY 0x80000000 +#define APLIC_DOMAINCFG_IE RT_BIT(8) +#define APLIC_DOMAINCFG_DM RT_BIT(2) +#define APLIC_DOMAINCFG_BE RT_BIT(0) + +#define APLIC_SOURCECFG_BASE 0x0004 +#define APLIC_SOURCECFG_D RT_BIT(10) +#define APLIC_SOURCECFG_CHILDIDX_MASK 0x000003ff +#define APLIC_SOURCECFG_SM_MASK 0x00000007 +#define APLIC_SOURCECFG_SM_INACTIVE 0x0 +#define APLIC_SOURCECFG_SM_DETACH 0x1 +#define APLIC_SOURCECFG_SM_EDGE_RISE 0x4 +#define APLIC_SOURCECFG_SM_EDGE_FALL 0x5 +#define APLIC_SOURCECFG_SM_LEVEL_HIGH 0x6 +#define APLIC_SOURCECFG_SM_LEVEL_LOW 0x7 + +#define APLIC_MMSICFGADDR 0x1bc0 +#define APLIC_MMSICFGADDRH 0x1bc4 +#define APLIC_SMSICFGADDR 0x1bc8 +#define APLIC_SMSICFGADDRH 0x1bcc + +#ifdef ARCH_RISCV_M_MODE +#define APLIC_xMSICFGADDR APLIC_MMSICFGADDR +#define APLIC_xMSICFGADDRH APLIC_MMSICFGADDRH +#else +#define APLIC_xMSICFGADDR APLIC_SMSICFGADDR +#define APLIC_xMSICFGADDRH APLIC_SMSICFGADDRH +#endif + +#define APLIC_xMSICFGADDRH_L RT_BIT(31) +#define APLIC_xMSICFGADDRH_HHXS_MASK 0x1f +#define APLIC_xMSICFGADDRH_HHXS_SHIFT 24 +#define APLIC_xMSICFGADDRH_HHXS (APLIC_xMSICFGADDRH_HHXS_MASK << APLIC_xMSICFGADDRH_HHXS_SHIFT) +#define APLIC_xMSICFGADDRH_LHXS_MASK 0x7 +#define APLIC_xMSICFGADDRH_LHXS_SHIFT 20 +#define APLIC_xMSICFGADDRH_LHXS (APLIC_xMSICFGADDRH_LHXS_MASK << APLIC_xMSICFGADDRH_LHXS_SHIFT) +#define APLIC_xMSICFGADDRH_HHXW_MASK 0x7 +#define APLIC_xMSICFGADDRH_HHXW_SHIFT 16 +#define APLIC_xMSICFGADDRH_HHXW (APLIC_xMSICFGADDRH_HHXW_MASK << APLIC_xMSICFGADDRH_HHXW_SHIFT) +#define APLIC_xMSICFGADDRH_LHXW_MASK 0xf +#define APLIC_xMSICFGADDRH_LHXW_SHIFT 12 +#define APLIC_xMSICFGADDRH_LHXW (APLIC_xMSICFGADDRH_LHXW_MASK << APLIC_xMSICFGADDRH_LHXW_SHIFT) +#define APLIC_xMSICFGADDRH_BAPPN_MASK 0xfff +#define APLIC_xMSICFGADDRH_BAPPN_SHIFT 0 +#define APLIC_xMSICFGADDRH_BAPPN (APLIC_xMSICFGADDRH_BAPPN_MASK << APLIC_xMSICFGADDRH_BAPPN_SHIFT) + +#define APLIC_xMSICFGADDR_PPN_SHIFT 12 + +#define APLIC_xMSICFGADDR_PPN_HART(__lhxs) \ + (RT_BIT(__lhxs) - 1) + +#define APLIC_xMSICFGADDR_PPN_LHX_MASK(__lhxw) \ + (RT_BIT(__lhxw) - 1) +#define APLIC_xMSICFGADDR_PPN_LHX_SHIFT(__lhxs) \ + ((__lhxs)) +#define APLIC_xMSICFGADDR_PPN_LHX(__lhxw, __lhxs) \ + (APLIC_xMSICFGADDR_PPN_LHX_MASK(__lhxw) << APLIC_xMSICFGADDR_PPN_LHX_SHIFT(__lhxs)) + +#define APLIC_xMSICFGADDR_PPN_HHX_MASK(__hhxw) \ + (RT_BIT(__hhxw) - 1) +#define APLIC_xMSICFGADDR_PPN_HHX_SHIFT(__hhxs) \ + ((__hhxs) + APLIC_xMSICFGADDR_PPN_SHIFT) +#define APLIC_xMSICFGADDR_PPN_HHX(__hhxw, __hhxs) \ + (APLIC_xMSICFGADDR_PPN_HHX_MASK(__hhxw) << APLIC_xMSICFGADDR_PPN_HHX_SHIFT(__hhxs)) + +#define APLIC_IRQBITS_PER_REG 32 + +#define APLIC_SETIP_BASE 0x1c00 +#define APLIC_SETIPNUM 0x1cdc + +#define APLIC_CLRIP_BASE 0x1d00 +#define APLIC_CLRIPNUM 0x1ddc + +#define APLIC_SETIE_BASE 0x1e00 +#define APLIC_SETIENUM 0x1edc + +#define APLIC_CLRIE_BASE 0x1f00 +#define APLIC_CLRIENUM 0x1fdc + +#define APLIC_SETIPNUM_LE 0x2000 +#define APLIC_SETIPNUM_BE 0x2004 + +#define APLIC_GENMSI 0x3000 + +#define APLIC_TARGET_BASE 0x3004 +#define APLIC_TARGET_HART_IDX_SHIFT 18 +#define APLIC_TARGET_HART_IDX_MASK 0x3fff +#define APLIC_TARGET_HART_IDX (APLIC_TARGET_HART_IDX_MASK << APLIC_TARGET_HART_IDX_SHIFT) +#define APLIC_TARGET_GUEST_IDX_SHIFT 12 +#define APLIC_TARGET_GUEST_IDX_MASK 0x3f +#define APLIC_TARGET_GUEST_IDX (APLIC_TARGET_GUEST_IDX_MASK << APLIC_TARGET_GUEST_IDX_SHIFT) +#define APLIC_TARGET_IPRIO_SHIFT 0 +#define APLIC_TARGET_IPRIO_MASK 0xff +#define APLIC_TARGET_IPRIO (APLIC_TARGET_IPRIO_MASK << APLIC_TARGET_IPRIO_SHIFT) +#define APLIC_TARGET_EIID_SHIFT 0 +#define APLIC_TARGET_EIID_MASK 0x7ff +#define APLIC_TARGET_EIID (APLIC_TARGET_EIID_MASK << APLIC_TARGET_EIID_SHIFT) + +#define APLIC_IDC_BASE 0x4000 +#define APLIC_IDC_SIZE 32 + +#define APLIC_IDC_IDELIVERY 0x00 + +#define APLIC_IDC_IFORCE 0x04 + +#define APLIC_IDC_ITHRESHOLD 0x08 + +#define APLIC_IDC_TOPI 0x18 +#define APLIC_IDC_TOPI_ID_SHIFT 16 +#define APLIC_IDC_TOPI_ID_MASK 0x3ff +#define APLIC_IDC_TOPI_ID (APLIC_IDC_TOPI_ID_MASK << APLIC_IDC_TOPI_ID_SHIFT) +#define APLIC_IDC_TOPI_PRIO_SHIFT 0 +#define APLIC_IDC_TOPI_PRIO_MASK 0xff +#define APLIC_IDC_TOPI_PRIO (APLIC_IDC_TOPI_PRIO_MASK << APLIC_IDC_TOPI_PRIO_SHIFT) + +#define APLIC_IDC_CLAIMI 0x1c + +#define APLIC_ENABLE_IDELIVERY 1 +#define APLIC_ENABLE_ITHRESHOLD 0 + +#define APLIC_DEFAULT_PRIORITY 1 + +struct riscv_aplic_idc +{ + void *regs; + rt_uint32_t hart_index; + rt_bool_t valid; +}; + +struct riscv_aplic_msicfg +{ + rt_ubase_t base_ppn; + rt_uint32_t hhxs; + rt_uint32_t hhxw; + rt_uint32_t lhxs; + rt_uint32_t lhxw; +}; + +struct riscv_aplic +{ + struct rt_pic parent; + struct rt_pic *msi_ic; + const struct imsic_global_config *imsic_global; + + struct rt_ofw_node *np; + + void *regs; + int parent_irq; + int init_cpu_id; + + rt_uint32_t nr_irqs; + rt_uint32_t nr_idcs; + + struct riscv_aplic_msicfg msicfg; + struct riscv_aplic_idc idc[RT_CPUS_NR]; +}; + +rt_always_inline rt_uint32_t irq_cfg_offset(int hwirq) +{ + return (hwirq - 1) * sizeof(rt_uint32_t); +} + +static void aplic_handler(int hwirq, void *data); + +static rt_err_t aplic_irq_init(struct rt_pic *pic) +{ + struct riscv_aplic *aplic = pic->priv_data; + + if (!aplic->msi_ic) + { + if (aplic->parent_irq < 0) + { + struct rt_ofw_cell_args irq_args; + + if (rt_ofw_parse_irq_cells(aplic->np, 0, &irq_args)) + { + LOG_E("%s: Parent IRQ is not available", rt_ofw_node_full_name(aplic->np)); + return RT_EOK; + } + + /* Create mapping */ + irq_args.args[0] = RV_IRQ_EXT; + + aplic->parent_irq = rt_ofw_map_irq(&irq_args); + + rt_hw_interrupt_install(aplic->parent_irq, aplic_handler, aplic, "APLIC"); + } + + rt_pic_irq_unmask(aplic->parent_irq); + } + + return RT_EOK; +} + +static void aplic_irq_unmask(struct rt_pic_irq *pirq) +{ + struct riscv_aplic *aplic = pirq->pic->priv_data; + + HWREG32(aplic->regs + APLIC_SETIENUM) = pirq->hwirq; +} + +static void aplic_irq_mask(struct rt_pic_irq *pirq) +{ + struct riscv_aplic *aplic = pirq->pic->priv_data; + + HWREG32(aplic->regs + APLIC_CLRIENUM) = pirq->hwirq; +} + +static void aplic_msi_irq_eoi(struct rt_pic_irq *pirq) +{ + struct riscv_aplic *aplic = pirq->pic->priv_data; + + switch (pirq->mode) + { + case RT_IRQ_MODE_LEVEL_HIGH: + case RT_IRQ_MODE_LEVEL_LOW: + HWREG32(aplic->regs + APLIC_SETIPNUM_LE) = pirq->hwirq; + break; + + default: + break; + } +} + +static rt_err_t aplic_direct_irq_set_priority(struct rt_pic_irq *pirq, rt_uint32_t priority) +{ + rt_uint32_t value; + struct riscv_aplic *aplic = pirq->pic->priv_data; + + priority = rt_min_t(rt_uint32_t, priority, + APLIC_TARGET_IPRIO_MASK - APLIC_DEFAULT_PRIORITY); + priority += APLIC_DEFAULT_PRIORITY; + + value = HWREG32(aplic->regs + APLIC_TARGET_BASE + irq_cfg_offset(pirq->hwirq)); + value &= ~APLIC_TARGET_IPRIO; + value |= priority << APLIC_TARGET_IPRIO_SHIFT; + + HWREG32(aplic->regs + APLIC_TARGET_BASE + irq_cfg_offset(pirq->hwirq)) = value; + + return RT_EOK; +} + +static rt_err_t aplic_direct_irq_set_affinity(struct rt_pic_irq *pirq, rt_bitmap_t *affinity) +{ + int cpu_id; + rt_uint32_t priority, value; + struct riscv_aplic *aplic = pirq->pic->priv_data; + + cpu_id = rt_bitmap_next_set_bit(affinity, 0, RT_CPUS_NR); + if (cpu_id >= RT_CPUS_NR || !riscv_cpu_id_available(cpu_id) || !aplic->idc[cpu_id].valid) + { + return -RT_EINVAL; + } + + priority = rt_min_t(rt_uint32_t, pirq->priority, + APLIC_TARGET_IPRIO_MASK - APLIC_DEFAULT_PRIORITY); + priority += APLIC_DEFAULT_PRIORITY; + + value = RT_FIELD_PREP(APLIC_TARGET_HART_IDX, aplic->idc[cpu_id].hart_index); + value |= RT_FIELD_PREP(APLIC_TARGET_IPRIO, priority); + + HWREG32(aplic->regs + APLIC_TARGET_BASE + irq_cfg_offset(pirq->hwirq)) = value; + + return RT_EOK; +} + +static rt_err_t aplic_msi_irq_set_affinity(struct rt_pic_irq *pirq, rt_bitmap_t *affinity) +{ + int cpu_id; + rt_uint32_t value; + struct riscv_aplic *aplic = pirq->pic->priv_data; + const struct imsic_local_config *local; + + cpu_id = rt_bitmap_next_set_bit(affinity, 0, RT_CPUS_NR); + if (cpu_id >= RT_CPUS_NR || !riscv_cpu_id_available(cpu_id) || + !aplic->imsic_global || !aplic->imsic_global->local[cpu_id].online) + { + return -RT_EINVAL; + } + + local = &aplic->imsic_global->local[cpu_id]; + + value = RT_FIELD_PREP(APLIC_TARGET_HART_IDX, local->hart_index); + value |= RT_FIELD_PREP(APLIC_TARGET_EIID, pirq->parent->hwirq); + + HWREG32(aplic->regs + APLIC_TARGET_BASE + irq_cfg_offset(pirq->hwirq)) = value; + + return RT_EOK; +} + +static rt_err_t aplic_irq_set_triger_mode(struct rt_pic_irq *pirq, rt_uint32_t mode) +{ + struct riscv_aplic *aplic = pirq->pic->priv_data; + + switch (mode) + { + case RT_IRQ_MODE_NONE: + mode = APLIC_SOURCECFG_SM_INACTIVE; + break; + case RT_IRQ_MODE_LEVEL_LOW: + mode = APLIC_SOURCECFG_SM_LEVEL_LOW; + break; + case RT_IRQ_MODE_LEVEL_HIGH: + mode = APLIC_SOURCECFG_SM_LEVEL_HIGH; + break; + case RT_IRQ_MODE_EDGE_FALLING: + mode = APLIC_SOURCECFG_SM_EDGE_FALL; + break; + case RT_IRQ_MODE_EDGE_RISING: + mode = APLIC_SOURCECFG_SM_EDGE_RISE; + break; + default: + return -RT_EINVAL; + } + + HWREG32(aplic->regs + APLIC_SOURCECFG_BASE + irq_cfg_offset(pirq->hwirq)) = mode; + + return RT_EOK; +} + +#ifdef RT_PCI_MSI +static void aplic_msi_irq_write_msg(struct rt_pic_irq *pirq, struct rt_pci_msi_msg *msg) +{ + rt_ubase_t tppn, tbppn, msg_addr; + rt_uint32_t group_index, hart_index, guest_index, val; + struct riscv_aplic *aplic = pirq->pic->priv_data; + struct riscv_aplic_msicfg *msicfg = &aplic->msicfg; + + /* For zeroed MSI, simply write zero into the target register */ + if (!msg->address_hi && !msg->address_lo && !msg->data) + { + HWREG32(aplic->regs + APLIC_TARGET_BASE + irq_cfg_offset(pirq->hwirq)) = 0; + return; + } + + /* Sanity check on message data */ + if (msg->data > APLIC_TARGET_EIID_MASK) + { + LOG_W("MSI msg out of EIID mask"); + } + + /* Compute target MSI address */ + msg_addr = (((rt_uint64_t)msg->address_hi) << 32) | msg->address_lo; + tppn = msg_addr >> APLIC_xMSICFGADDR_PPN_SHIFT; + + /* Compute target HART Base PPN */ + tbppn = tppn; + tbppn &= ~APLIC_xMSICFGADDR_PPN_HART(msicfg->lhxs); + tbppn &= ~APLIC_xMSICFGADDR_PPN_LHX(msicfg->lhxw, msicfg->lhxs); + tbppn &= ~APLIC_xMSICFGADDR_PPN_HHX(msicfg->hhxw, msicfg->hhxs); + + /* Compute target group and hart indexes */ + group_index = (tppn >> APLIC_xMSICFGADDR_PPN_HHX_SHIFT(msicfg->hhxs)) & + APLIC_xMSICFGADDR_PPN_HHX_MASK(msicfg->hhxw); + hart_index = (tppn >> APLIC_xMSICFGADDR_PPN_LHX_SHIFT(msicfg->lhxs)) & + APLIC_xMSICFGADDR_PPN_LHX_MASK(msicfg->lhxw); + hart_index |= (group_index << msicfg->lhxw); + + if (hart_index > APLIC_TARGET_HART_IDX_MASK) + { + LOG_W("HART index out of IDX mask"); + } + + /* Compute target guest index */ + guest_index = tppn & APLIC_xMSICFGADDR_PPN_HART(msicfg->lhxs); + + if (guest_index > APLIC_TARGET_GUEST_IDX_MASK) + { + LOG_W("Guest index out of IDX mask"); + } + + /* Update IRQ TARGET register */ + val = RT_FIELD_PREP(APLIC_TARGET_HART_IDX, hart_index); + val |= RT_FIELD_PREP(APLIC_TARGET_GUEST_IDX, guest_index); + val |= RT_FIELD_PREP(APLIC_TARGET_EIID, msg->data); + + HWREG32(aplic->regs + APLIC_TARGET_BASE + irq_cfg_offset(pirq->hwirq)) = val; +} +#endif /* RT_PCI_MSI */ + +static int aplic_irq_map(struct rt_pic *pic, int hwirq, rt_uint32_t mode) +{ + int irq; + struct riscv_aplic *aplic = pic->priv_data; + struct rt_pic_irq *pirq; + + if (hwirq <= 0 || hwirq > aplic->nr_irqs) + { + return -RT_EINVAL; + } + + pirq = rt_pic_find_irq(pic, hwirq); + + if (aplic->msi_ic) + { + int msi_irq = aplic->msi_ic->ops->irq_alloc_msi(aplic->msi_ic, RT_NULL); + + if (msi_irq < 0) + { + return msi_irq; + } + + irq = rt_pic_config_irq(pic, hwirq, hwirq); + + rt_pic_cascade(pirq, msi_irq); + } + else + { + irq = rt_pic_config_irq(pic, hwirq, hwirq); + } + + rt_memset(pirq->affinity, 0, sizeof(pirq->affinity)); + + RT_IRQ_AFFINITY_SET(pirq->affinity, aplic->init_cpu_id); + pic->ops->irq_set_affinity(pirq, pirq->affinity); + + pirq->mode = mode; + pic->ops->irq_set_triger_mode(pirq, mode); + + return irq; +} + +static rt_err_t aplic_irq_parse(struct rt_pic *pic, struct rt_ofw_cell_args *args, struct rt_pic_irq *out_pirq) +{ + struct riscv_aplic *aplic = pic->priv_data; + + if (args->args_count < 2 || !args->args[0] || args->args[0] > aplic->nr_irqs) + { + return -RT_EINVAL; + } + + out_pirq->hwirq = args->args[0]; + out_pirq->mode = args->args[1] & RT_IRQ_MODE_MASK; + + return RT_EOK; +} + +const static struct rt_pic_ops aplic_direct_ops = +{ + .name = "APLIC-DIRECT", + .irq_init = aplic_irq_init, + .irq_mask = aplic_irq_mask, + .irq_unmask = aplic_irq_unmask, + .irq_set_priority = aplic_direct_irq_set_priority, + .irq_set_affinity = aplic_direct_irq_set_affinity, + .irq_set_triger_mode = aplic_irq_set_triger_mode, + .irq_map = aplic_irq_map, + .irq_parse = aplic_irq_parse, +}; + +const static struct rt_pic_ops aplic_msi_ops = +{ + .name = "APLIC-MSI", + .irq_init = aplic_irq_init, + .irq_mask = aplic_irq_mask, + .irq_unmask = aplic_irq_unmask, + .irq_eoi = aplic_msi_irq_eoi, + .irq_set_affinity = aplic_msi_irq_set_affinity, + .irq_set_triger_mode = aplic_irq_set_triger_mode, +#ifdef RT_PCI_MSI + .irq_write_msi_msg = aplic_msi_irq_write_msg, +#endif /* RT_PCI_MSI */ + .irq_map = aplic_irq_map, + .irq_parse = aplic_irq_parse, + .flags = RT_PIC_F_IRQ_ROUTING, +}; + +static void aplic_handler(int hwirq, void *data) +{ + struct rt_pic_irq *pirq; + struct riscv_aplic *aplic = data; + struct riscv_aplic_idc *idc = &aplic->idc[rt_hw_cpu_id()]; + + while ((hwirq = HWREG32(idc->regs + APLIC_IDC_CLAIMI))) + { + hwirq >>= APLIC_IDC_TOPI_ID_SHIFT; + + pirq = rt_pic_find_irq(&aplic->parent, hwirq); + + rt_pic_handle_isr(pirq); + } +} + +static rt_err_t aplic_ofw_init(struct rt_ofw_node *np, const struct rt_ofw_node_id *id) +{ + rt_err_t err; + rt_uint32_t val; + struct rt_ofw_node *msi_np = RT_NULL; + struct riscv_aplic *aplic; + + msi_np = rt_ofw_parse_phandle(np, "msi-parent", 0); + + if (msi_np) + { + if (!imsic_ofw_is_current_mode(msi_np)) + { + rt_ofw_node_put(msi_np); + return RT_EOK; + } + } + else + { + struct rt_ofw_cell_args irq_args; + + if (rt_ofw_parse_irq_cells(np, 0, &irq_args) || + irq_args.args_count < 1 || irq_args.args[0] != RV_IRQ_EXT) + { + return RT_EOK; + } + } + + if (!(aplic = rt_calloc(1, sizeof(*aplic)))) + { + err = -RT_ENOMEM; + goto _fail; + } + + aplic->regs = rt_ofw_iomap(np, 0); + + if (!aplic->regs) + { + err = -RT_ENOMEM; + goto _fail; + } + + if ((err = rt_ofw_prop_read_u32(np, "riscv,num-sources", &aplic->nr_irqs))) + { + goto _fail; + } + + if (!aplic->nr_irqs || aplic->nr_irqs > APLIC_MAX_SOURCE) + { + err = -RT_EINVAL; + goto _fail; + } + + /* Disable all interrupts */ + for (int i = 0; i <= aplic->nr_irqs; i += 32) + { + HWREG32(aplic->regs + APLIC_CLRIE_BASE + (i / 32) * sizeof(rt_uint32_t)) = RT_UINT32_MAX; + } + + /* Clear APLIC domaincfg */ + HWREG32(aplic->regs + APLIC_DOMAINCFG) = 0; + + aplic->np = np; + aplic->init_cpu_id = rt_hw_cpu_id(); + rt_ofw_data(np) = &aplic->parent; + + if (msi_np) + { + const struct imsic_global_config *imsic_global; + struct riscv_aplic_msicfg *msicfg = &aplic->msicfg; + + if ((err = imsic_ofw_probe(msi_np, np, id))) + { + goto _fail; + } + + if (!(imsic_global = imsic_global_config_read(msi_np))) + { + err = RT_EOK; + goto _fail; + } + aplic->imsic_global = imsic_global; + + /* Find number of guest index bits (LHXS) */ + msicfg->lhxs = imsic_global->guest_index_bits; + if (APLIC_xMSICFGADDRH_LHXS_MASK < msicfg->lhxs) + { + LOG_E("%s: IMSIC guest index bits big for APLIC LHXS", rt_ofw_node_full_name(np)); + err = -RT_EINVAL; + goto _fail; + } + + /* Find number of HART index bits (LHXW) */ + msicfg->lhxw = imsic_global->hart_index_bits; + if (APLIC_xMSICFGADDRH_LHXW_MASK < msicfg->lhxw) + { + LOG_E("%s: IMSIC hart index bits big for APLIC LHXW", rt_ofw_node_full_name(np)); + err = -RT_EINVAL; + goto _fail; + } + + /* Find number of group index bits (HHXW) */ + msicfg->hhxw = imsic_global->group_index_bits; + if (APLIC_xMSICFGADDRH_HHXW_MASK < msicfg->hhxw) + { + LOG_E("%s: IMSIC group index bits big for APLIC HHXW", rt_ofw_node_full_name(np)); + err = -RT_EINVAL; + goto _fail; + } + + /* Find first bit position of group index (HHXS) */ + msicfg->hhxs = imsic_global->group_index_shift; + if (msicfg->hhxs < (2 * APLIC_xMSICFGADDR_PPN_SHIFT)) + { + LOG_E("%s: IMSIC group index shift should be >= %d", + rt_ofw_node_full_name(np), (2 * APLIC_xMSICFGADDR_PPN_SHIFT)); + err = -RT_EINVAL; + goto _fail; + } + msicfg->hhxs -= (2 * APLIC_xMSICFGADDR_PPN_SHIFT); + if (APLIC_xMSICFGADDRH_HHXS_MASK < msicfg->hhxs) + { + LOG_E("%s: IMSIC group index shift big for APLIC HHXS", rt_ofw_node_full_name(np)); + err = -RT_EINVAL; + goto _fail; + } + + /* Compute PPN base */ + msicfg->base_ppn = imsic_global->base_addr >> APLIC_xMSICFGADDR_PPN_SHIFT; + msicfg->base_ppn &= ~APLIC_xMSICFGADDR_PPN_HART(msicfg->lhxs); + msicfg->base_ppn &= ~APLIC_xMSICFGADDR_PPN_LHX(msicfg->lhxw, msicfg->lhxs); + msicfg->base_ppn &= ~APLIC_xMSICFGADDR_PPN_HHX(msicfg->hhxw, msicfg->hhxs); + } + else + { + int hwirq, cpu_id, idc_count; + rt_ubase_t hartid; + struct riscv_aplic_idc *idc; + struct rt_ofw_cell_args idcs; + struct rt_ofw_node *cpu_np; + + idc_count = rt_min_t(int, rt_ofw_get_irq_count(np), APLIC_MAX_IDC); + + for (int idc_index = 0; idc_index < idc_count; ++idc_index) + { + if (rt_ofw_parse_irq_cells(np, idc_index, &idcs) || idcs.args_count < 1) + { + continue; + } + + cpu_np = rt_ofw_get_parent(idcs.data); + + if (!cpu_np) + { + continue; + } + + hwirq = idcs.args[0]; + hartid = rt_ofw_get_cpu_hwid(cpu_np, 0); + + rt_ofw_node_put(cpu_np); + + if (hwirq != RV_IRQ_EXT) + { + continue; + } + + cpu_id = riscv_hartid_to_cpu_id(hartid); + + if (cpu_id < 0 || cpu_id >= RT_CPUS_NR) + { + continue; + } + + idc = &aplic->idc[cpu_id]; + idc->regs = aplic->regs + APLIC_IDC_BASE + idc_index * APLIC_IDC_SIZE; + idc->hart_index = idc_index; + idc->valid = RT_TRUE; + + /* Priority must be less than threshold for interrupt triggering */ + HWREG32(idc->regs + APLIC_IDC_ITHRESHOLD) = APLIC_ENABLE_ITHRESHOLD; + + /* Delivery must be set to 1 for interrupt triggering */ + HWREG32(idc->regs + APLIC_IDC_IDELIVERY) = APLIC_ENABLE_IDELIVERY; + + ++aplic->nr_idcs; + } + + if (!aplic->nr_idcs) + { + err = RT_EOK; + goto _fail; + } + } + + /* Setup global config and interrupt delivery */ +#ifdef ARCH_RISCV_M_MODE + if (msi_np) + { + rt_uint32_t valh; + + val = rt_lower_32_bits(aplic->msicfg.base_ppn); + valh = RT_FIELD_PREP(APLIC_xMSICFGADDRH_BAPPN, rt_upper_32_bits(aplic->msicfg.base_ppn)); + valh |= RT_FIELD_PREP(APLIC_xMSICFGADDRH_LHXW, aplic->msicfg.lhxw); + valh |= RT_FIELD_PREP(APLIC_xMSICFGADDRH_HHXW, aplic->msicfg.hhxw); + valh |= RT_FIELD_PREP(APLIC_xMSICFGADDRH_LHXS, aplic->msicfg.lhxs); + valh |= RT_FIELD_PREP(APLIC_xMSICFGADDRH_HHXS, aplic->msicfg.hhxs); + + HWREG32(aplic->regs + APLIC_xMSICFGADDR) = val; + HWREG32(aplic->regs + APLIC_xMSICFGADDRH) = valh; + } +#endif /* ARCH_RISCV_M_MODE */ + + /* Setup APLIC domaincfg register */ + val = HWREG32(aplic->regs + APLIC_DOMAINCFG); + val |= APLIC_DOMAINCFG_IE; + + if (msi_np) + { + val |= APLIC_DOMAINCFG_DM; + } + + HWREG32(aplic->regs + APLIC_DOMAINCFG) = val; + + if (HWREG32(aplic->regs + APLIC_DOMAINCFG) != val) + { + LOG_W("Unable to write 0x%x in domaincfg", val); + } + + aplic->parent_irq = -1; + + aplic->parent.priv_data = aplic; + + if (msi_np) + { + aplic->parent.ops = &aplic_msi_ops; + aplic->msi_ic = rt_pic_dynamic_cast(rt_ofw_data(msi_np)); + } + else + { + aplic->parent.ops = &aplic_direct_ops; + } + + /* APLIC source identities are 1-based; keep index 0 reserved. */ + rt_pic_linear_irq(&aplic->parent, aplic->nr_irqs + 1); + + rt_pic_user_extends(&aplic->parent); + + if (msi_np) + { + rt_ofw_node_put(msi_np); + } + + return RT_EOK; + +_fail: + if (aplic && rt_ofw_data(np) == &aplic->parent) + { + rt_ofw_data(np) = RT_NULL; + } + + if (aplic && aplic->regs) + { + rt_iounmap(aplic->regs); + } + + if (msi_np) + { + rt_ofw_node_put(msi_np); + } + + rt_free(aplic); + + return err; +} + +static const struct rt_ofw_node_id aplic_ofw_ids[] = +{ + { .compatible = "riscv,aplic" }, + { /* sentinel */ } +}; +RT_PIC_OFW_DECLARE(aplic, aplic_ofw_ids, aplic_ofw_init); diff --git a/libcpu/risc-v/common/pic/pic-riscv-clic.c b/libcpu/risc-v/common/pic/pic-riscv-clic.c new file mode 100644 index 0000000000..436f0da02b --- /dev/null +++ b/libcpu/risc-v/common/pic/pic-riscv-clic.c @@ -0,0 +1,533 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2025-04-20 GuEe-GUI The first version + */ + +#include <rthw.h> +#include <rtatomic.h> +#include <rtthread.h> +#ifndef PIC_DISABLE_DM +#include <rtdevice.h> +#endif + +#define DBG_TAG "pic.clic" +#define DBG_LVL DBG_INFO +#include <rtdbg.h> + +#include <cpu.h> +#include <csr.h> +#include <sbi.h> +#include <bitmap.h> +#include <riscv_timer.h> + +#include "pic-riscv-clint.h" + +#define CLIC_CFG 0x0000 +#define CLIC_INFO 0x0004 +#define CLIC_VENDOR_MINTTHRESH 0x0008 +#define CLIC_INT_TRIG 0x0040 +#define CLIC_INT_IP 0x1000 +#define CLIC_INT_IE 0x1001 +#define CLIC_INT_ATTR 0x1002 +#define CLIC_INT_CTL 0x1003 +#define CLIC_CFG_LEVEL_SHIFT 1 +#define CLIC_CFG_LEVEL_MASK GENMASK(4, CLIC_CFG_LEVEL_SHIFT) + +#define CLIC_INFO_NUM_INTERRUPT(val) (val & GENMASK(12, 0)) +#define CLIC_INFO_VERSION(val) ((val & GENMASK(20, 13)) >> 13) +#define CLIC_INFO_VERSION_ARCH(val) ((val & GENMASK(20, 17)) >> 17) +#define CLIC_INFO_VERSION_IMPL(val) ((val & GENMASK(16, 13)) >> 13) +#define CLIC_INFO_INT_CTL_BITS(val) ((val & GENMASK(24, 21)) >> 21) +#define CLIC_INFO_NUM_TRIGGER(val) ((val & GENMASK(30, 25)) >> 25) + +#define CLIC_INT_TRIG_ENABLE BIT(31) +#define CLIC_INT_TRIG_IRQ_SHIFT 0 +#define CLIC_INT_TRIG_IRQ_MASK GENMASK(12, 0) + +#define CLIC_INT_IP_PENDING 1 + +#define CLIC_INT_IE_ENABLE 1 + +#define CLIC_INT_ATTR_MODE_SHIFT 6 +#define CLIC_INT_ATTR_MODE_MASK GENMASK(7, CLIC_INT_ATTR_MODE_SHIFT) +#define CLIC_INT_ATTR_MODE_MACHINE (0x3 << CLIC_INT_ATTR_MODE_SHIFT) +#define CLIC_INT_ATTR_MODE_SUPERVISOR (0x01 << CLIC_INT_ATTR_MODE_SHIFT) +#define CLIC_INT_ATTR_MODE_USER (0x00 << CLIC_INT_ATTR_MODE_SHIFT) +#define CLIC_INT_ATTR_TRIGGERED_SHIFT 1 +#define CLIC_INT_ATTR_TRIGGERED_MASK GENMASK(2, CLIC_INT_ATTR_TRIGGERED_SHIFT) +#define CLIC_INT_ATTR_TRIGGERED_LEVEL (0x0 << CLIC_INT_ATTR_TRIGGERED_SHIFT) +#define CLIC_INT_ATTR_TRIGGERED_EDGE (0x1 << CLIC_INT_ATTR_TRIGGERED_SHIFT) +#define CLIC_INT_ATTR_SHV BIT(0) + +#define CLIC_INT_CTL_BITS_MAX 8 + +#define CLIC_INT_ID_USIP 0 /* User software Interrupt */ +#define CLIC_INT_ID_SSIP 1 /* Supervisor software Interrupt */ +#define CLIC_INT_ID_MSIP 3 /* Machine software interrupt */ +#define CLIC_INT_ID_UTIP 4 /* User timer interrupt */ +#define CLIC_INT_ID_STIP 5 /* Supervisor timer interrupt */ +#define CLIC_INT_ID_MTIP 7 /* Machine timer interrupt */ +#define CLIC_INT_ID_UEIP 8 /* User external (PLIC) interrupt */ +#define CLIC_INT_ID_SEIP 9 /* Supervisor external (PLIC) interrupt */ +#define CLIC_INT_ID_MEIP 11 /* Machine external (PLIC) interrupt */ +#define CLIC_INT_ID_CSIP 12 /* CLIC software interrupt */ +#define CLIC_INT_ID_EXT_BASE 16 /* CLIC external base */ +#ifdef ARCH_RISCV_M_MODE +#define CLIC_INT_ID_xSIP CLIC_INT_ID_MSIP +#define CLIC_INT_ID_xTIP CLIC_INT_ID_MTIP +#define CLIC_INT_ID_xEIP CLIC_INT_ID_MEIP +#else +#define CLIC_INT_ID_xSIP CLIC_INT_ID_SSIP +#define CLIC_INT_ID_xTIP CLIC_INT_ID_STIP +#define CLIC_INT_ID_xEIP CLIC_INT_ID_SEIP +#endif + +#define CLIC_QUIRK_MODE_FIXED 0 +#define CLIC_QUIRK_MINTTHRESH 1 +#define CLIC_QUIRK_SHV 2 + +struct riscv_clic +{ +#ifndef PIC_DISABLE_DM + struct rt_pic parent; +#endif + + rt_uint8_t *regs; + + rt_bitmap_t quirks; + rt_bool_t shv; + rt_uint32_t irqs_count:16; + rt_uint32_t int_ctl_bits:8; + +#ifdef RT_USING_SMP + rt_uint32_t ipi_base; + rt_atomic_t ipi_enable[RT_CPUS_NR]; + rt_atomic_t ipi_pending[RT_CPUS_NR]; +#endif +}; + +extern void fast_irq_vectors(void); + +static void riscv_clic_set_mintthresh(struct riscv_clic *clic, rt_uint32_t mintthresh) +{ + if (rt_bitmap_test_bit(&clic->quirks, CLIC_QUIRK_MINTTHRESH)) + { + HWREG32(clic->regs + CLIC_VENDOR_MINTTHRESH) = mintthresh; + } +} + +static void riscv_clic_irq_ack_raw(struct riscv_clic *clic, int hwirq) +{ + if (clic->shv) + { + return; + } + + (void)csr_swap(CSR_NXTI, 0UL); +} + +static void riscv_clic_irq_mask_raw(struct riscv_clic *clic, int hwirq) +{ +#ifdef RT_USING_SMP + if (hwirq >= clic->ipi_base) + { + rt_atomic_t *ipi_enable = &clic->ipi_enable[rt_hw_cpu_id()]; + + rt_hw_atomic_and(ipi_enable, ~BIT(hwirq - clic->ipi_base)); + + if (rt_hw_atomic_load(ipi_enable) != 0) + { + return; + } + + hwirq = CLIC_INT_ID_xSIP; + } +#endif /* RT_USING_SMP */ + + HWREG8(clic->regs + CLIC_INT_IE + 4 * hwirq) = 0; +} + +static void riscv_clic_irq_unmask_raw(struct riscv_clic *clic, int hwirq) +{ +#ifdef RT_USING_SMP + if (hwirq >= clic->ipi_base) + { + rt_atomic_t *ipi_enable = &clic->ipi_enable[rt_hw_cpu_id()]; + + rt_hw_atomic_or(ipi_enable, BIT(hwirq - clic->ipi_base)); + hwirq = CLIC_INT_ID_xSIP; + } +#endif /* RT_USING_SMP */ + + HWREG8(clic->regs + CLIC_INT_IE + 4 * hwirq) = CLIC_INT_IE_ENABLE; +} + +static void riscv_clic_irq_send_ipi_raw(struct riscv_clic *clic, int ipi, rt_bitmap_t *cpumask) +{ +#ifdef RT_USING_SMP + int cpu; + + RT_ASSERT(ipi >= clic->ipi_base && ipi < clic->ipi_base + RT_MAX_IPI); + ipi -= clic->ipi_base; + + rt_bitmap_for_each_set_bit(cpumask, cpu, RT_CPUS_NR) + { + if (!riscv_cpu_id_available(cpu)) + { + continue; + } + + rt_hw_atomic_or(&clic->ipi_pending[cpu], BIT(ipi)); + + #ifdef ARCH_RISCV_M_MODE + clint_send_ipi(cpu); + #endif + } + +#ifndef ARCH_RISCV_M_MODE + sbi_send_ipi((const unsigned long *)cpumask); +#endif +#else + RT_UNUSED(clic); + RT_UNUSED(ipi); + RT_UNUSED(cpumask); +#endif /* RT_USING_SMP */ +} + +static rt_bool_t riscv_clic_handler_isr_raw(struct riscv_clic *clic, int hwirq, + rt_bool_t (*callback)(int hwirq, void *data), void *data) +{ + if (hwirq >= CLIC_INT_ID_EXT_BASE) + { + return callback(hwirq, data); + } + else if (hwirq == CLIC_INT_ID_xTIP) + { + riscv_timer_isr(); + return RT_TRUE; + } +#ifdef RT_USING_SMP + else if (hwirq == CLIC_INT_ID_xSIP) + { + int ipi; + rt_bool_t res = RT_FALSE; + rt_bitmap_t pending; + + #ifdef ARCH_RISCV_M_MODE + clint_clear_ipi(); + #endif + + pending = rt_hw_atomic_exchange(&clic->ipi_pending[rt_hw_cpu_id()], 0); + + rt_bitmap_for_each_set_bit(&pending, ipi, RT_MAX_IPI) + { + res = callback(clic->ipi_base + ipi, data) || res; + } + + return res; + } +#endif /* RT_USING_SMP */ + else if (hwirq != CLIC_INT_ID_xEIP) + { + LOG_E("Unhandled IRQ: %d", hwirq); + } + + return RT_FALSE; +} + +static void riscv_clic_percpu_common_init(struct riscv_clic *clic) +{ + if (clic->shv) + { + csr_write(CSR_TVT, &fast_irq_vectors); + } +} + +static struct riscv_clic *riscv_clic_common_init(rt_ubase_t clic_base, rt_size_t clic_size, + rt_bitmap_t quirks) +{ + rt_uint32_t cfg, value, level_bits; +#ifdef PIC_DISABLE_DM + static struct riscv_clic non_dm_clic; + struct riscv_clic *clic = &non_dm_clic; + + rt_memset(clic, 0, sizeof(*clic)); +#else + struct riscv_clic *clic = rt_calloc(1, sizeof(*clic)); + + if (!clic) + { + rt_set_errno(-RT_ENOMEM); + return RT_NULL; + } +#endif + + clic->regs = rt_ioremap((void *)clic_base, clic_size); + + if (!clic->regs) + { + rt_set_errno(-RT_EIO); + +#ifndef PIC_DISABLE_DM + rt_free(clic); +#endif + return RT_NULL; + } + + clic->quirks = quirks; + + riscv_clic_set_mintthresh(clic, 0); + + value = HWREG32(clic->regs + CLIC_INFO); + clic->int_ctl_bits = CLIC_INFO_INT_CTL_BITS(value); + clic->irqs_count = CLIC_INFO_NUM_INTERRUPT(value); + + cfg = HWREG32(clic->regs + CLIC_CFG); + cfg &= ~CLIC_CFG_LEVEL_MASK; + level_bits = clic->int_ctl_bits; + if (level_bits > CLIC_INT_CTL_BITS_MAX) + { + level_bits = CLIC_INT_CTL_BITS_MAX; + } + cfg |= level_bits << CLIC_CFG_LEVEL_SHIFT; + HWREG32(clic->regs + CLIC_CFG) = cfg; + + /* cliccfg bit 0 advertises SHV capability; it does not enable SHV. */ + clic->shv = rt_bitmap_test_bit(&clic->quirks, CLIC_QUIRK_SHV); + + LOG_D("Version %d.%d, %d IRQs implemented", + CLIC_INFO_VERSION_ARCH(value), CLIC_INFO_VERSION_IMPL(value), + clic->irqs_count); + + value = clic->shv ? CLIC_INT_ATTR_SHV : 0; +#ifdef ARCH_RISCV_M_MODE + value |= CLIC_INT_ATTR_MODE_MACHINE; +#else + value |= CLIC_INT_ATTR_MODE_SUPERVISOR; +#endif + + for (int hwirq = 0; hwirq < clic->irqs_count; ++hwirq) + { + HWREG8(clic->regs + CLIC_INT_ATTR + 4 * hwirq) &= + ~(CLIC_INT_ATTR_MODE_MASK | CLIC_INT_ATTR_TRIGGERED_MASK | CLIC_INT_ATTR_SHV); + HWREG8(clic->regs + CLIC_INT_ATTR + 4 * hwirq) |= value; + } + +#ifdef RT_USING_SMP + clic->irqs_count += RT_MAX_IPI; +#endif + + return clic; +} + +#ifndef PIC_DISABLE_DM +static rt_err_t riscv_clic_irq_init(struct rt_pic *pic) +{ + struct riscv_clic *clic = pic->priv_data; + + riscv_clic_percpu_common_init(clic); + + return RT_EOK; +} + +static void riscv_clic_irq_ack(struct rt_pic_irq *pirq) +{ + struct riscv_clic *clic = pirq->pic->priv_data; + + riscv_clic_irq_ack_raw(clic, pirq->hwirq); +} + +static void riscv_clic_irq_mask(struct rt_pic_irq *pirq) +{ + struct riscv_clic *clic = pirq->pic->priv_data; + + riscv_clic_irq_mask_raw(clic, pirq->hwirq); +} + +static void riscv_clic_irq_unmask(struct rt_pic_irq *pirq) +{ + struct riscv_clic *clic = pirq->pic->priv_data; + + riscv_clic_irq_unmask_raw(clic, pirq->hwirq); +} + +static rt_err_t riscv_clic_irq_set_priority(struct rt_pic_irq *pirq, rt_uint32_t priority) +{ + struct riscv_clic *clic = pirq->pic->priv_data; + rt_uint32_t int_ctl_bits = rt_min_t(rt_uint32_t, clic->int_ctl_bits, + CLIC_INT_CTL_BITS_MAX); + + if (!int_ctl_bits) + { + return -RT_ENOSYS; + } + + priority = rt_min_t(rt_uint32_t, priority, BIT(int_ctl_bits) - 1); + HWREG8(clic->regs + CLIC_INT_CTL + pirq->hwirq * 4) = + priority << (CLIC_INT_CTL_BITS_MAX - int_ctl_bits); + + return RT_EOK; +} + +static rt_err_t riscv_clic_irq_set_triger_mode(struct rt_pic_irq *pirq, rt_uint32_t mode) +{ + int hwirq = pirq->hwirq; + struct riscv_clic *clic = pirq->pic->priv_data; + + if (mode == RT_IRQ_MODE_EDGE_RISING) + { + HWREG8(clic->regs + CLIC_INT_ATTR + 4 * hwirq) &= ~CLIC_INT_ATTR_TRIGGERED_MASK; + HWREG8(clic->regs + CLIC_INT_ATTR + 4 * hwirq) |= CLIC_INT_ATTR_TRIGGERED_EDGE; + + return RT_EOK; + } + else if (mode == RT_IRQ_MODE_LEVEL_HIGH) + { + HWREG8(clic->regs + CLIC_INT_ATTR + 4 * hwirq) &= ~CLIC_INT_ATTR_TRIGGERED_MASK; + HWREG8(clic->regs + CLIC_INT_ATTR + 4 * hwirq) |= CLIC_INT_ATTR_TRIGGERED_LEVEL; + + return RT_EOK; + } + + return -RT_ENOSYS; +} + +static void riscv_clic_irq_send_ipi(struct rt_pic_irq *pirq, rt_bitmap_t *cpumask) +{ + struct riscv_clic *clic = pirq->pic->priv_data; + + riscv_clic_irq_send_ipi_raw(clic, pirq->hwirq, cpumask); +} + +static int riscv_clic_irq_map(struct rt_pic *pic, int hwirq, rt_uint32_t mode) +{ + return rt_pic_config_irq(pic, hwirq, hwirq); +} + +static rt_err_t riscv_clic_irq_parse(struct rt_pic *pic, struct rt_ofw_cell_args *args, struct rt_pic_irq *out_pirq) +{ + struct riscv_clic *clic = pic->priv_data; + + out_pirq->hwirq = args->args[0]; + + if (rt_bitmap_test_bit(&clic->quirks, CLIC_QUIRK_MODE_FIXED)) + { + out_pirq->mode = args->args[1]; + out_pirq->priority = args->args[2]; + } + else + { + out_pirq->mode = RT_IRQ_MODE_LEVEL_HIGH; + out_pirq->priority = args->args[1]; + } + + return RT_EOK; +} + +const static struct rt_pic_ops riscv_clic_ops = +{ + .name = "CLIC", + .irq_init = riscv_clic_irq_init, + .irq_ack = riscv_clic_irq_ack, + .irq_mask = riscv_clic_irq_mask, + .irq_unmask = riscv_clic_irq_unmask, + .irq_set_priority = riscv_clic_irq_set_priority, + .irq_set_triger_mode = riscv_clic_irq_set_triger_mode, + .irq_send_ipi = riscv_clic_irq_send_ipi, + .irq_map = riscv_clic_irq_map, + .irq_parse = riscv_clic_irq_parse, +}; + +static rt_bool_t clic_common_handler_isr(int hwirq, void *data) +{ + struct rt_pic_irq *pirq; + struct riscv_clic *clic = data; + +#ifdef RT_USING_SMP + if (hwirq < clic->ipi_base) + { + pirq = rt_pic_find_irq(&clic->parent, hwirq); + } + else + { + pirq = rt_pic_find_ipi(&clic->parent, hwirq - clic->ipi_base); + } +#else + pirq = rt_pic_find_irq(&clic->parent, hwirq); +#endif /* RT_USING_SMP */ + + riscv_clic_irq_ack(pirq); + + return rt_pic_handle_isr(pirq) == RT_EOK; +} + +static rt_bool_t clic_handler(void *data) +{ + rt_ubase_t cause; + struct riscv_clic *clic = data; + + cause = csr_read(CSR_CAUSE) & GENMASK(11, 0); + return riscv_clic_handler_isr_raw(clic, cause, clic_common_handler_isr, clic); +} + +static rt_err_t riscv_clic_ofw_init(struct rt_ofw_node *np, const struct rt_ofw_node_id *id) +{ + rt_err_t err; + rt_uint64_t clic_base, clic_size; + rt_bitmap_t quirks; + struct riscv_clic *clic; + + if ((err = rt_ofw_get_address(np, 0, &clic_base, &clic_size))) + { + return err; + } + + quirks = id->data ? *(rt_ubase_t *)id->data : 0; + + if (!(clic = riscv_clic_common_init(clic_base, clic_size, quirks))) + { + return rt_get_errno(); + } + +#ifdef ARCH_RISCV_M_MODE + riscv_clint_ofw_init(); +#endif + + clic->parent.priv_data = clic; + clic->parent.ops = &riscv_clic_ops; + rt_ofw_data(np) = &clic->parent; + + rt_pic_linear_irq(&clic->parent, clic->irqs_count); + + rt_pic_add_traps(clic_handler, clic); + +#ifdef RT_USING_SMP + clic->ipi_base = clic->irqs_count - RT_MAX_IPI; + + for (int ipi = 0; ipi < RT_MAX_IPI; ++ipi) + { + rt_pic_config_ipi(&clic->parent, ipi, clic->ipi_base + ipi); + } +#endif /* RT_USING_SMP */ + + rt_pic_user_extends(&clic->parent); + + return RT_EOK; +} + +static rt_bitmap_t xtensa_clic_quirk = RT_BIT(CLIC_QUIRK_MODE_FIXED) | RT_BIT(CLIC_QUIRK_MINTTHRESH); + +static const struct rt_ofw_node_id riscv_clic_ofw_ids[] = +{ + { .compatible = "cdns,xtensa-core-intc", .data = &xtensa_clic_quirk }, + { .compatible = "nuclei,eclic" }, + { .compatible = "nordic,nrf-clic" }, + { /* sentinel */ } +}; +RT_PIC_OFW_DECLARE(riscv_clic, riscv_clic_ofw_ids, riscv_clic_ofw_init); +#endif /* PIC_DISABLE_DM */ diff --git a/libcpu/risc-v/common/pic/pic-riscv-clint.c b/libcpu/risc-v/common/pic/pic-riscv-clint.c new file mode 100644 index 0000000000..ab8c5d4205 --- /dev/null +++ b/libcpu/risc-v/common/pic/pic-riscv-clint.c @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-01-30 GuEe-GUI first version + */ + +#include <rthw.h> +#include <rtthread.h> +#ifndef PIC_DISABLE_DM +#include <rtdevice.h> +#endif + +#include <cpu.h> +#include <csr.h> +#include <riscv_timer.h> + +#include "pic-riscv-clint.h" + +#define CLINT_IPI_OFF 0 +#define CLINT_TIMER_CMP_OFF 0x4000 +#define CLINT_TIMER_VAL_OFF 0xbff8 + +#define CLINT_QUIRK_IPI_FORCE 0 +#define CLINT_QUIRK_IPI_CSR_CLEAR 1 + +static rt_uint32_t *riscv_clint_ipi_base; +rt_uint32_t *riscv_aclint_ipi_base; +static rt_bool_t riscv_clint_ipi_csr_clear; + +void clint_send_ipi(int cpu) +{ + if (!riscv_cpu_id_available(cpu)) + { + return; + } + + if (riscv_clint_ipi_base) + { + HWREG32(riscv_clint_ipi_base + riscv_cpu_id_to_hartid(cpu)) = 1; + } +} + +void clint_clear_ipi(void) +{ + if (riscv_clint_ipi_base) + { + if (riscv_clint_ipi_csr_clear) + { + csr_clear(CSR_IP, IE_SIE); + } + else + { + rt_ubase_t hartid = riscv_cpu_id_to_hartid(rt_hw_cpu_id()); + + HWREG32(riscv_clint_ipi_base + hartid) = 0; + } + } +} + +static rt_err_t riscv_clint_common_init(rt_ubase_t *mmio_ranges, unsigned mmio_count) +{ + void *clint_timer_val, *clint_timer_cmp; + + if (mmio_count == 1) + { + rt_uint8_t *base = rt_ioremap((void *)mmio_ranges[0], mmio_ranges[1]); + + if (!base) + { + return -RT_EIO; + } + + riscv_clint_ipi_base = (rt_uint32_t *)(base + CLINT_IPI_OFF); + clint_timer_cmp = base + CLINT_TIMER_CMP_OFF; + clint_timer_val = base + CLINT_TIMER_VAL_OFF; + } + else + { + riscv_clint_ipi_base = RT_NULL; + + clint_timer_val = rt_ioremap((void *)mmio_ranges[0], mmio_ranges[1]); + + if (!clint_timer_val) + { + return -RT_EIO; + } + + clint_timer_cmp = rt_ioremap((void *)mmio_ranges[2], mmio_ranges[3]); + + if (!clint_timer_cmp) + { + rt_iounmap(clint_timer_val); + return -RT_EIO; + } + } + + riscv_timer_probe(clint_timer_cmp, clint_timer_val); + + return RT_EOK; +} + +#ifndef PIC_DISABLE_DM +#ifdef ARCH_RISCV_M_MODE +static rt_ubase_t aclint_quirks = RT_BIT(CLINT_QUIRK_IPI_FORCE); +#else +static rt_ubase_t aclint_quirks = RT_BIT(CLINT_QUIRK_IPI_FORCE) | + RT_BIT(CLINT_QUIRK_IPI_CSR_CLEAR); +#endif + +static const struct rt_ofw_node_id riscv_clint_ofw_ids[] = +{ + { .compatible = "riscv,clint0", }, + { .compatible = "sifive,clint0", }, + { .compatible = "thead,c900-clint", }, +#ifdef ARCH_RISCV_M_MODE + { .compatible = "riscv,aclint-mtimer", }, + { .compatible = "riscv,aclint-mswi", .data = &aclint_quirks }, +#else + { .compatible = "riscv,aclint-sswi", .data = &aclint_quirks }, +#endif + { /* sentinel */ } +}; + +rt_err_t riscv_clint_ofw_init(void) +{ + rt_err_t err; + rt_ubase_t quirks; + unsigned mmio_count; + rt_uint64_t addr, size; + rt_ubase_t mmio_ranges[4]; + struct rt_ofw_node *np; + const struct rt_ofw_node_id *id; + + if (!(np = rt_ofw_find_node_by_ids_r(RT_NULL, riscv_clint_ofw_ids, &id))) + { + return -RT_ENOSYS; + } + quirks = id->data ? *(rt_ubase_t *)id->data : 0; + + if ((err = rt_ofw_get_address(np, 0, &addr, &size))) + { + goto _out; + } + + mmio_ranges[0] = addr; + mmio_ranges[1] = size; + mmio_count = 1; + + if (!rt_ofw_get_address(np, 1, &addr, &size)) + { + mmio_ranges[2] = addr; + mmio_ranges[3] = size; + mmio_count = 2; + } + + rt_ofw_node_set_flag(np, RT_OFW_F_READLY); + + if (quirks & RT_BIT(CLINT_QUIRK_IPI_FORCE)) + { + riscv_clint_ipi_base = rt_ioremap((void *)mmio_ranges[0], mmio_ranges[1]); + + if (!riscv_clint_ipi_base) + { + err = -RT_EIO; + goto _out; + } + + riscv_aclint_ipi_base = riscv_clint_ipi_base; + riscv_clint_ipi_csr_clear = !!(quirks & RT_BIT(CLINT_QUIRK_IPI_CSR_CLEAR)); + err = RT_EOK; + } + else + { + err = riscv_clint_common_init(mmio_ranges, mmio_count); + } + +_out: + rt_ofw_node_put(np); + + return err; +} +#endif /* !PIC_DISABLE_DM */ diff --git a/libcpu/risc-v/common/pic/pic-riscv-clint.h b/libcpu/risc-v/common/pic/pic-riscv-clint.h new file mode 100644 index 0000000000..fbf090fcd0 --- /dev/null +++ b/libcpu/risc-v/common/pic/pic-riscv-clint.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __PIC_RISCV_CLINT_H__ +#define __PIC_RISCV_CLINT_H__ + +#include <rtthread.h> + +extern rt_uint32_t *riscv_aclint_ipi_base; + +void clint_send_ipi(int cpu); +void clint_clear_ipi(void); + +#ifndef PIC_DISABLE_DM +rt_err_t riscv_clint_ofw_init(void); +#endif + +#endif /* __PIC_RISCV_CLINT_H__ */ diff --git a/libcpu/risc-v/common/pic/pic-riscv-imsic.c b/libcpu/risc-v/common/pic/pic-riscv-imsic.c new file mode 100644 index 0000000000..23bb7ad239 --- /dev/null +++ b/libcpu/risc-v/common/pic/pic-riscv-imsic.c @@ -0,0 +1,944 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-01-30 GuEe-GUI first version + */ + +#include <rthw.h> +#include <rtthread.h> +#include <rtdevice.h> + +#define DBG_TAG "pic.imsic" +#define DBG_LVL DBG_INFO +#include <rtdbg.h> + +#include <cpu.h> +#include <csr.h> +#include <drivers/pic.h> +#include "pic-riscv-imsic.h" + +#define IMSIC_ID_START 1 +#define IMSIC_IPI_ID IMSIC_ID_START + +#define IMSIC_ENABLE_EIDELIVERY 1 +#define IMSIC_DISABLE_EIDELIVERY 0 +#define IMSIC_ENABLE_EITHRESHOLD 0 +#define IMSIC_DISABLE_EITHRESHOLD 1 + +struct riscv_imsic_mmio +{ + rt_uint64_t addr; + rt_uint64_t size; + + void *mapped; +}; + +struct riscv_imsic +{ + struct rt_pic parent; + struct rt_pic *parent_ic; + + struct rt_ofw_node *np; + + int parent_irq; + int msi_id_start; + int init_cpu_id; + int sync_id; + + struct imsic_global_config global; + + rt_bitmap_t *vectors; + rt_bitmap_t *enabled; + struct rt_spinlock lock; +}; + +extern rt_uint32_t *riscv_aclint_ipi_base; + +rt_inline unsigned int __fls(unsigned long word) +{ + unsigned int num = RT_BITS_PER_LONG - 1; + +#if RT_BITS_PER_LONG == 64 + if (!(word & (~0UL << 32))) + { + num -= 32; + word <<= 32; + } +#endif + if (!(word & (~0UL << (RT_BITS_PER_LONG - 16)))) + { + num -= 16; + word <<= 16; + } + if (!(word & (~0UL << (RT_BITS_PER_LONG - 8)))) + { + num -= 8; + word <<= 8; + } + if (!(word & (~0UL << (RT_BITS_PER_LONG - 4)))) + { + num -= 4; + word <<= 4; + } + if (!(word & (~0UL << (RT_BITS_PER_LONG - 2)))) + { + num -= 2; + word <<= 2; + } + if (!(word & (~0UL << (RT_BITS_PER_LONG - 1)))) + { + num -= 1; + } + return num; +} + +rt_inline void imsic_csr_write(rt_ubase_t reg, rt_ubase_t val) +{ + csr_write(CSR_ISELECT, reg); + csr_write(CSR_IREG, val); +} + +rt_inline rt_ubase_t imsic_csr_read(rt_ubase_t reg) +{ + csr_write(CSR_ISELECT, reg); + return csr_read(CSR_IREG); +} + +rt_inline rt_ubase_t imsic_csr_read_clear(rt_ubase_t reg, rt_ubase_t val) +{ + csr_write(CSR_ISELECT, reg); + return csr_read_clear(CSR_IREG, val); +} + +rt_inline void imsic_csr_set(rt_ubase_t reg, rt_ubase_t val) +{ + csr_write(CSR_ISELECT, reg); + csr_set(CSR_IREG, val); +} + +rt_inline void imsic_csr_clear(rt_ubase_t reg, rt_ubase_t val) +{ + csr_write(CSR_ISELECT, reg); + csr_clear(CSR_IREG, val); +} + +static void imsic_eix_update(rt_ubase_t base_id, rt_ubase_t num_id, rt_bool_t pend, rt_bool_t val) +{ + rt_ubase_t id = base_id, last_id = base_id + num_id, isel, ireg; + + while (id < last_id) + { + isel = id / RT_BITS_PER_LONG; + isel *= RT_BITS_PER_LONG / IMSIC_EIPx_BITS; + isel += pend ? IMSIC_EIP0 : IMSIC_EIE0; + + /* + * Prepare the ID mask to be programmed in the + * IMSIC EIEx and EIPx registers. These registers + * are XLEN-wide and we must not touch IDs which + * are < base_id and >= (base_id + num_id). + */ + ireg = 0; + for (rt_ubase_t i = id & (__riscv_xlen - 1); id < last_id && i < __riscv_xlen; ++i) + { + ireg |= RT_BIT(i); + id++; + } + + /* + * The IMSIC EIEx and EIPx registers are indirectly + * accessed via using ISELECT and IREG CSRs so we + * need to access these CSRs without getting preempted. + * + * All existing users of this function call this + * function with local IRQs disabled so we don't + * need to do anything special here. + */ + if (val) + { + imsic_csr_set(isel, ireg); + } + else + { + imsic_csr_clear(isel, ireg); + } + } +} + +rt_inline void imsic_id_clear_enable(rt_ubase_t id) +{ + imsic_eix_update(id, 1, RT_FALSE, RT_FALSE); +} + +rt_inline void imsic_id_clear_pending(rt_ubase_t id) +{ + imsic_eix_update(id, 1, RT_TRUE, RT_FALSE); +} + +rt_inline void imsic_local_delivery(rt_bool_t enable) +{ + if (enable) + { + imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ENABLE_EITHRESHOLD); + imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_ENABLE_EIDELIVERY); + } + else + { + imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_DISABLE_EIDELIVERY); + imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_DISABLE_EITHRESHOLD); + } +} + +static void imsic_handler(int hwirq, void *data); + +/* Caller holds imsic->lock with local interrupts disabled. */ +static void imsic_sync_local(struct riscv_imsic *imsic) +{ + for (rt_ubase_t id = 0; id <= imsic->global.nr_ids; id += RT_BITS_PER_LONG) + { + imsic_csr_write(IMSIC_EIE0 + id / IMSIC_EIPx_BITS, + imsic->enabled[id / RT_BITS_PER_LONG]); + } +} + +static void imsic_update_enable(struct riscv_imsic *imsic, int hwirq, rt_bool_t enable) +{ + rt_ubase_t level = rt_spin_lock_irqsave(&imsic->lock); + + if (enable) + { + rt_bitmap_set_bit(imsic->enabled, hwirq); + } + else + { + rt_bitmap_clear_bit(imsic->enabled, hwirq); + } + + /* EIE is hart-local. MSI affinity does not follow the calling thread. */ + for (int cpu = 0; cpu < RT_CPUS_NR; ++cpu) + { + struct imsic_local_config *local = &imsic->global.local[cpu]; + + if (!local->online) + { + continue; + } + if (cpu == rt_hw_cpu_id()) + { + imsic_sync_local(imsic); + } + else + { + HWREG32(local->msi) = imsic->sync_id; + } + } + + rt_spin_unlock_irqrestore(&imsic->lock, level); +} + +static rt_bool_t imsic_id_allocated(struct riscv_imsic *imsic, int hwirq) +{ + if (hwirq < IMSIC_ID_START || hwirq > imsic->global.nr_ids) + { + return RT_FALSE; + } + + return rt_bitmap_test_bit(imsic->vectors, hwirq - IMSIC_ID_START); +} + +static rt_err_t riscv_imsic_irq_init(struct rt_pic *pic) +{ + rt_ubase_t level; + int cpu_id = rt_hw_cpu_id(); + struct riscv_imsic *imsic = pic->priv_data; + struct imsic_local_config *local = &imsic->global.local[cpu_id]; + + if (!local->valid) + { + return -RT_EINVAL; + } + + imsic_local_delivery(RT_FALSE); + + for (int hwirq = IMSIC_ID_START; hwirq <= imsic->global.nr_ids; ++hwirq) + { + imsic_id_clear_enable(hwirq); + imsic_id_clear_pending(hwirq); + } + + if (imsic->parent_irq < 0) + { + struct rt_ofw_cell_args irq_args; + + if (rt_ofw_parse_irq_cells(imsic->np, 0, &irq_args)) + { + LOG_E("%s: Parent IRQ is not available", rt_ofw_node_full_name(imsic->np)); + return RT_EOK; + } + + /* Create mapping */ + irq_args.args[0] = RV_IRQ_EXT; + + imsic->parent_irq = rt_ofw_map_irq(&irq_args); + + rt_hw_interrupt_install(imsic->parent_irq, imsic_handler, imsic, "IMSIC"); + } + + rt_pic_irq_unmask(imsic->parent_irq); + level = rt_spin_lock_irqsave(&imsic->lock); + imsic_sync_local(imsic); + imsic_local_delivery(RT_TRUE); + local->online = RT_TRUE; + rt_spin_unlock_irqrestore(&imsic->lock, level); + + return RT_EOK; +} + +static void riscv_imsic_irq_mask(struct rt_pic_irq *pirq) +{ +#ifdef RT_PCI_MSI + if (pirq->msi_desc) + { + rt_pci_msi_mask_irq(pirq); + } +#endif /* RT_PCI_MSI */ + + if (pirq->hwirq >= IMSIC_ID_START) + { + imsic_update_enable(pirq->pic->priv_data, pirq->hwirq, RT_FALSE); + } +} + +static void riscv_imsic_irq_unmask(struct rt_pic_irq *pirq) +{ +#ifdef RT_PCI_MSI + if (pirq->msi_desc) + { + rt_pci_msi_unmask_irq(pirq); + } +#endif /* RT_PCI_MSI */ + + if (pirq->hwirq >= IMSIC_ID_START) + { + imsic_update_enable(pirq->pic->priv_data, pirq->hwirq, RT_TRUE); + } +} + +static rt_err_t riscv_imsic_irq_set_affinity(struct rt_pic_irq *pirq, rt_bitmap_t *affinity) +{ +#ifdef RT_PCI_MSI + struct rt_pci_msi_desc *desc = pirq->msi_desc; + + if (desc) + { + int old_cpu, cpu; + + cpu = rt_bitmap_next_set_bit(affinity, 0, RT_CPUS_NR); + if (cpu >= RT_CPUS_NR) + { + return -RT_EINVAL; + } + + /* IMSIC supports only one HART affinity */ + if (rt_bitmap_next_set_bit(affinity, cpu + 1, RT_CPUS_NR) != RT_CPUS_NR) + { + return -RT_ENOSYS; + } + + if (!riscv_cpu_id_available(cpu) || !((struct riscv_imsic *)pirq->pic->priv_data)->global.local[cpu].online) + { + return -RT_EINVAL; + } + + old_cpu = rt_bitmap_next_set_bit(pirq->affinity, 0, RT_CPUS_NR); + + if (cpu == old_cpu) + { + return RT_EOK; + } + + RT_IRQ_AFFINITY_CLEAR(pirq->affinity, old_cpu); + RT_IRQ_AFFINITY_SET(pirq->affinity, cpu); + + /* Update MSI info */ + pirq->pic->ops->irq_compose_msi_msg(pirq, &desc->msg); + + /* Update for PCI device */ + rt_pci_msi_write_msg(desc, &desc->msg); + + return RT_EOK; + } +#endif /* RT_PCI_MSI */ + + return -RT_EINVAL; +} + +#ifdef ARCH_RISCV_M_MODE +static void riscv_imsic_irq_send_ipi(struct rt_pic_irq *pirq, rt_bitmap_t *cpumask) +{ + int cpu_id; + struct imsic_local_config *local; + struct riscv_imsic *imsic = pirq->pic->priv_data; + + rt_bitmap_for_each_set_bit(cpumask, cpu_id, RT_CPUS_NR) + { + local = &imsic->global.local[cpu_id]; + + if (!riscv_cpu_id_available(cpu_id) || !local->online) + { + continue; + } + + HWREG32(local->msi) = pirq->hwirq; + } +} +#endif /* ARCH_RISCV_M_MODE */ + +#ifdef RT_PCI_MSI +static void riscv_imsic_irq_compose_msi_msg(struct rt_pic_irq *pirq, struct rt_pci_msi_msg *msg) +{ + int cpu_id; + struct imsic_local_config *local; + struct imsic_global_config *global; + struct riscv_imsic *imsic = pirq->pic->priv_data; + + global = &imsic->global; + msg->address_hi = 0; + msg->address_lo = 0; + msg->data = 0; + + if (RT_BIT(global->guest_index_bits) <= 0) + { + LOG_W("Global guest disabled"); + return; + } + + cpu_id = rt_bitmap_next_set_bit(pirq->affinity, 0, RT_CPUS_NR); + if (cpu_id >= RT_CPUS_NR || !riscv_cpu_id_available(cpu_id) || !global->local[cpu_id].online) + { + LOG_W("CPU%d MSI target is not available", cpu_id); + return; + } + + local = &global->local[cpu_id]; + + msg->address_hi = rt_upper_32_bits(local->msi_phy); + msg->address_lo = rt_lower_32_bits(local->msi_phy); + msg->data = pirq->hwirq; +} + +static void riscv_imsic_irq_write_msi_msg(struct rt_pic_irq *pirq, struct rt_pci_msi_msg *msg) +{ + struct rt_pic_irq virtual_pirq = {}; + struct riscv_imsic *imsic = pirq->pic->priv_data; + + if (!imsic->parent_ic || !imsic->parent_ic->ops->irq_write_msi_msg) + { + return; + } + + virtual_pirq.pic = imsic->parent_ic; + virtual_pirq.hwirq = pirq->hwirq; + + imsic->parent_ic->ops->irq_write_msi_msg(&virtual_pirq, msg); +} +#endif /* RT_PCI_MSI */ + +static int riscv_imsic_irq_alloc_msi(struct rt_pic *pic, struct rt_pci_msi_desc *msi_desc) +{ + rt_ubase_t level; + int irq, hwirq_index; + struct rt_pic_irq *pirq; + struct imsic_global_config *global; + struct riscv_imsic *imsic = pic->priv_data; + + global = &imsic->global; + + level = rt_spin_lock_irqsave(&imsic->lock); + hwirq_index = rt_bitmap_next_clear_bit(imsic->vectors, 0, global->nr_ids); + + if (hwirq_index >= global->nr_ids) + { + irq = -RT_EEMPTY; + goto _out_lock; + } + + irq = rt_pic_config_irq(pic, hwirq_index, hwirq_index + IMSIC_ID_START); + if (irq < 0) + { + goto _out_lock; + } + pirq = rt_pic_find_irq(pic, hwirq_index); + pirq->mode = RT_IRQ_MODE_EDGE_RISING; + rt_memset(pirq->affinity, 0, sizeof(pirq->affinity)); + RT_IRQ_AFFINITY_SET(pirq->affinity, imsic->init_cpu_id); + + rt_bitmap_set_bit(imsic->vectors, hwirq_index); + +_out_lock: + rt_spin_unlock_irqrestore(&imsic->lock, level); + + if (irq >= 0) + { + imsic_update_enable(imsic, hwirq_index + IMSIC_ID_START, RT_TRUE); + } + + return irq; +} + +static void riscv_imsic_irq_free_msi(struct rt_pic *pic, int irq) +{ + rt_ubase_t level; + struct rt_pic_irq *pirq; + struct riscv_imsic *imsic; + + pirq = rt_pic_find_pirq(pic, irq); + + if (!pirq) + { + return; + } + + imsic = pirq->pic->priv_data; + + imsic_update_enable(imsic, pirq->hwirq, RT_FALSE); + + level = rt_spin_lock_irqsave(&imsic->lock); + rt_bitmap_clear_bit(imsic->vectors, pirq->hwirq - IMSIC_ID_START); + rt_spin_unlock_irqrestore(&imsic->lock, level); +} + +const static struct rt_pic_ops riscv_imsic_ops = +{ + .name = "IMSIC", + .irq_init = riscv_imsic_irq_init, + .irq_mask = riscv_imsic_irq_mask, + .irq_unmask = riscv_imsic_irq_unmask, + .irq_set_affinity = riscv_imsic_irq_set_affinity, +#ifdef ARCH_RISCV_M_MODE + .irq_send_ipi = riscv_imsic_irq_send_ipi, +#endif +#ifdef RT_PCI_MSI + .irq_compose_msi_msg = riscv_imsic_irq_compose_msi_msg, + .irq_write_msi_msg = riscv_imsic_irq_write_msi_msg, +#endif /* RT_PCI_MSI */ + .irq_alloc_msi = riscv_imsic_irq_alloc_msi, + .irq_free_msi = riscv_imsic_irq_free_msi, +}; + +static void imsic_handler(int hwirq, void *data) +{ + rt_ubase_t local_id; + struct rt_pic_irq *pirq; + struct riscv_imsic *imsic = data; + + while ((local_id = csr_swap(CSR_TOPEI, 0))) + { + local_id >>= TOPEI_ID_SHIFT; + + if (local_id == imsic->sync_id) + { + rt_ubase_t level = rt_spin_lock_irqsave(&imsic->lock); + + imsic_sync_local(imsic); + rt_spin_unlock_irqrestore(&imsic->lock, level); + continue; + } + + if (local_id >= imsic->msi_id_start) + { + if (!imsic_id_allocated(imsic, local_id)) + { + imsic_id_clear_enable(local_id); + LOG_W("Ignore unallocated IMSIC ID %d", local_id); + continue; + } + + pirq = rt_pic_find_irq(&imsic->parent, local_id - IMSIC_ID_START); + } + else + { + #if defined(RT_USING_SMP) && defined(ARCH_RISCV_M_MODE) + pirq = rt_pic_find_ipi(&imsic->parent, local_id - IMSIC_IPI_ID); + #else + LOG_E("It's is impossible"); + RT_ASSERT(0); + #endif + } + + if (!pirq || pirq->pic != &imsic->parent) + { + imsic_id_clear_enable(local_id); + LOG_W("Ignore unmapped IMSIC ID %d", local_id); + continue; + } + + rt_pic_handle_isr(pirq); + } +} + +const struct imsic_global_config *imsic_global_config_read(struct rt_ofw_node *np) +{ + struct rt_pic *pic = rt_pic_dynamic_cast(rt_ofw_data(np)); + struct riscv_imsic *imsic = RT_NULL; + + if (pic && pic->ops == &riscv_imsic_ops) + { + imsic = rt_container_of(pic, struct riscv_imsic, parent); + } + + return imsic ? &imsic->global : RT_NULL; +} + +static const struct rt_ofw_node_id riscv_imsic_ofw_ids[] = +{ + { .compatible = "riscv,imsics" }, + { /* sentinel */ } +}; + +rt_bool_t imsic_ofw_is_available(void) +{ + struct rt_ofw_node *np; + + rt_ofw_foreach_node_by_ids(np, riscv_imsic_ofw_ids) + { + if (imsic_ofw_is_current_mode(np)) + { + rt_ofw_node_put(np); + return RT_TRUE; + } + } + + return RT_FALSE; +} + +rt_bool_t imsic_ofw_is_current_mode(struct rt_ofw_node *np) +{ + struct rt_ofw_cell_args irq_args; + + if (rt_ofw_parse_irq_cells(np, 0, &irq_args)) + { + return RT_FALSE; + } + + return irq_args.args[0] == RV_IRQ_EXT; +} + +rt_err_t imsic_ofw_probe(struct rt_ofw_node *np, + struct rt_ofw_node *ic_np, const struct rt_ofw_node_id *ic_id) +{ + rt_err_t err; + int irq_count, mmio_count, i; + struct rt_ofw_node_id *id; + struct riscv_imsic *imsic; + struct imsic_global_config *global; + struct riscv_imsic_mmio *mmio = RT_NULL; + struct rt_pic *pic = rt_pic_dynamic_cast(rt_ofw_data(np)); + + if (!(id = rt_ofw_node_match(np, riscv_imsic_ofw_ids))) + { + return -RT_EINVAL; + } + + if (!imsic_ofw_is_current_mode(np)) + { + return RT_EOK; + } + + if (pic && pic->ops == &riscv_imsic_ops) + { + return RT_EOK; + } + + if (!(imsic = rt_calloc(1, sizeof(*imsic)))) + { + return -RT_ENOMEM; + } + global = &imsic->global; + + irq_count = rt_ofw_get_irq_count(np); + + rt_ofw_prop_read_u32(np, "riscv,guest-index-bits", &global->guest_index_bits); + + if (rt_ofw_prop_read_u32(np, "riscv,hart-index-bits", &global->hart_index_bits)) + { + global->hart_index_bits = __fls(irq_count); + + if (RT_BIT(global->hart_index_bits) < irq_count) + { + ++global->hart_index_bits; + } + } + + rt_ofw_prop_read_u32(np, "riscv,group-index-bits", &global->group_index_bits); + + if (rt_ofw_prop_read_u32(np, "riscv,group-index-shift", &global->group_index_shift)) + { + global->group_index_shift = IMSIC_MMIO_PAGE_SHIFT * 2; + } + + if ((err = rt_ofw_prop_read_u32(np, "riscv,num-ids", &global->nr_ids))) + { + LOG_E("%s: Number of IRQ is unknown", rt_ofw_node_full_name(np)); + goto _fail; + } + + if (rt_ofw_prop_read_u32(np, "riscv,num-guest-ids", &global->nr_guest_ids)) + { + global->nr_guest_ids = global->nr_ids; + } + + /* Sanity check guest index bits */ + i = RT_BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT; + if (i < global->guest_index_bits) + { + LOG_E("%s: %s index %s too big", "Guest", "bits", rt_ofw_node_full_name(np)); + + err = -RT_EINVAL; + goto _fail; + } + + /* Sanity check HART index bits */ + i = RT_BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT - global->guest_index_bits; + if (i < global->hart_index_bits) + { + LOG_E("%s: %s index %s too big", "HART", "bits", rt_ofw_node_full_name(np)); + + err = -RT_EINVAL; + goto _fail; + } + + /* Sanity check group index bits */ + i = RT_BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT - global->guest_index_bits - global->hart_index_bits; + if (i < global->group_index_bits) + { + LOG_E("%s: %s index %s too big", "Group", "bits", rt_ofw_node_full_name(np)); + + err = -RT_EINVAL; + goto _fail; + } + + /* Sanity check group index shift */ + i = global->group_index_bits + global->group_index_shift - 1; + if (i >= RT_BITS_PER_LONG) + { + LOG_E("%s: %s index %s too big", "Group", "shift", rt_ofw_node_full_name(np)); + + err = -RT_EINVAL; + goto _fail; + } + + /* Sanity check number of interrupt identities */ + if (global->nr_ids < IMSIC_MIN_ID || + global->nr_ids >= IMSIC_MAX_ID || + (global->nr_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) + { + LOG_E("%s: Invalid number of %sinterrupt identities", "", rt_ofw_node_full_name(np)); + + err = -RT_EINVAL; + goto _fail; + } + + /* Sanity check number of guest interrupt identities */ + if (global->nr_guest_ids < IMSIC_MIN_ID || + global->nr_guest_ids >= IMSIC_MAX_ID || + (global->nr_guest_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) + { + LOG_E("%s: Invalid number of %sinterrupt identities", "guest ", rt_ofw_node_full_name(np)); + + err = -RT_EINVAL; + goto _fail; + } + + mmio_count = rt_ofw_get_address_count(np); + mmio = rt_calloc(mmio_count, sizeof(*mmio)); + + if (!mmio) + { + err = -RT_ENOMEM; + goto _fail; + } + + for (int i = 0; i < mmio_count; ++i) + { + rt_ubase_t base_addr; + + if ((err = rt_ofw_get_address(np, i, &mmio[i].addr, &mmio[i].size))) + { + goto _fail; + } + + if (i == 0) + { + global->base_addr = mmio[i].addr; + global->base_addr &= ~(RT_BIT(global->guest_index_bits + + global->hart_index_bits + IMSIC_MMIO_PAGE_SHIFT) - 1); + global->base_addr &= ~((RT_BIT(global->group_index_bits) - 1) << + global->group_index_shift); + } + + base_addr = mmio[i].addr; + base_addr &= ~(RT_BIT(global->guest_index_bits + + global->hart_index_bits + IMSIC_MMIO_PAGE_SHIFT) - 1); + base_addr &= ~((RT_BIT(global->group_index_bits) - 1) << + global->group_index_shift); + + if (base_addr != global->base_addr) + { + err = -RT_EINVAL; + goto _fail; + } + + mmio[i].mapped = rt_ofw_iomap(np, i); + + if (!mmio[i].mapped) + { + err = -RT_EIO; + goto _fail; + } + } + + for (int i = 0; i < irq_count; ++i) + { + int cpu_id; + int index, reloff; + rt_ubase_t hartid; + struct rt_ofw_node *cpu_np; + struct rt_ofw_cell_args arg; + struct imsic_local_config *local; + + if (rt_ofw_parse_irq_cells(np, i, &arg)) + { + continue; + } + + cpu_np = rt_ofw_get_parent(arg.data); + hartid = rt_ofw_get_cpu_hwid(cpu_np, 0); + rt_ofw_node_put(cpu_np); + + cpu_id = riscv_hartid_to_cpu_id(hartid); + + if (cpu_id < 0 || cpu_id >= RT_CPUS_NR) + { + continue; + } + + /* Find MMIO location of MSI page */ + index = mmio_count; + reloff = i * RT_BIT(global->guest_index_bits) * IMSIC_MMIO_PAGE_SZ; + for (int j = 0; j < mmio_count; ++j) + { + if (reloff < mmio[j].size) + { + index = j; + break; + } + + /* + * MMIO region size may not be aligned to + * RT_BIT(global->guest_index_bits) * IMSIC_MMIO_PAGE_SZ + * if holes are present. + */ + reloff -= RT_ALIGN(mmio[j].size, RT_BIT(global->guest_index_bits) * IMSIC_MMIO_PAGE_SZ); + } + + if (index >= mmio_count) + { + LOG_E("%s: MMIO not found for parent IRQ[%d]", rt_ofw_node_full_name(np), i); + continue; + } + + local = &global->local[cpu_id]; + local->msi_phy = mmio[index].addr + reloff; + local->msi = mmio[index].mapped + reloff; + local->hart_index = i; + local->valid = RT_TRUE; + } + rt_free(mmio); + mmio = RT_NULL; + + if (!(imsic->vectors = rt_calloc(1, RT_BITMAP_LEN(global->nr_ids) * sizeof(rt_bitmap_t)))) + { + err = -RT_ENOMEM; + goto _fail; + } + + imsic->enabled = rt_calloc(RT_BITMAP_LEN(global->nr_ids + 1), sizeof(rt_bitmap_t)); + if (!imsic->enabled) + { + err = -RT_ENOMEM; + goto _fail; + } + + imsic->sync_id = global->nr_ids; + rt_bitmap_set_bit(imsic->vectors, imsic->sync_id - IMSIC_ID_START); + rt_bitmap_set_bit(imsic->enabled, imsic->sync_id); + + rt_spin_lock_init(&imsic->lock); + + imsic->np = np; + imsic->parent_irq = -1; + imsic->init_cpu_id = rt_hw_cpu_id(); + + imsic->parent.priv_data = imsic; + imsic->parent.ops = &riscv_imsic_ops; + + if (ic_np) + { + imsic->parent_ic = rt_pic_dynamic_cast(rt_ofw_data(ic_np)); + } + + rt_ofw_data(np) = &imsic->parent; + rt_ofw_node_set_flag(np, RT_OFW_F_READLY); + + rt_pic_linear_irq(&imsic->parent, global->nr_ids); + + imsic->msi_id_start = IMSIC_ID_START; + +#if defined(RT_USING_SMP) && defined(ARCH_RISCV_M_MODE) + if (!riscv_aclint_ipi_base) + { + for (int ipi = 0; ipi < RT_MAX_IPI; ++ipi) + { + int hwirq = IMSIC_IPI_ID + ipi; + + rt_bitmap_set_bit(imsic->vectors, hwirq - IMSIC_ID_START); + rt_bitmap_set_bit(imsic->enabled, hwirq); + rt_pic_config_ipi(&imsic->parent, ipi, hwirq); + } + + imsic->msi_id_start += RT_MAX_IPI; + } +#endif /* RT_USING_SMP && ARCH_RISCV_M_MODE */ + + rt_pic_user_extends(&imsic->parent); + + return RT_EOK; + +_fail: + if (mmio) + { + for (int i = 0; i < mmio_count && mmio[i].mapped; ++i) + { + rt_iounmap(mmio[i].mapped); + } + + rt_free(mmio); + } + + rt_free(imsic->enabled); + rt_free(imsic->vectors); + rt_free(imsic); + + return err; +} + +static rt_err_t imsic_ofw_init(struct rt_ofw_node *np, const struct rt_ofw_node_id *id) +{ + return imsic_ofw_probe(np, RT_NULL, id); +} +RT_PIC_OFW_DECLARE(imsic, riscv_imsic_ofw_ids, imsic_ofw_init); diff --git a/libcpu/risc-v/common/pic/pic-riscv-imsic.h b/libcpu/risc-v/common/pic/pic-riscv-imsic.h new file mode 100644 index 0000000000..e83343ed3f --- /dev/null +++ b/libcpu/risc-v/common/pic/pic-riscv-imsic.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-01-30 GuEe-GUI first version + */ + +#ifndef __PIC_RISCV_IMSIC_H__ +#define __PIC_RISCV_IMSIC_H__ + +#include <drivers/ofw.h> +#include <drivers/misc.h> +#include <drivers/core/dm.h> + +#define IMSIC_MMIO_PAGE_SHIFT 12 +#define IMSIC_MMIO_PAGE_SZ RT_BIT(IMSIC_MMIO_PAGE_SHIFT) +#define IMSIC_MMIO_PAGE_LE 0x00 +#define IMSIC_MMIO_PAGE_BE 0x04 + +#define IMSIC_MIN_ID 63 +#define IMSIC_MAX_ID 2048 + +#define IMSIC_EIDELIVERY 0x70 + +#define IMSIC_EITHRESHOLD 0x72 + +#define IMSIC_EIP0 0x80 +#define IMSIC_EIP63 0xbf +#define IMSIC_EIPx_BITS 32 + +#define IMSIC_EIE0 0xc0 +#define IMSIC_EIE63 0xff +#define IMSIC_EIEx_BITS 32 + +#define IMSIC_FIRST IMSIC_EIDELIVERY +#define IMSIC_LAST IMSIC_EIE63 + +#define IMSIC_MMIO_SETIPNUM_LE 0x00 +#define IMSIC_MMIO_SETIPNUM_BE 0x04 + +struct imsic_local_config +{ + void *msi; + rt_uint64_t msi_phy; + rt_uint32_t hart_index; + rt_bool_t valid; + rt_bool_t online; +}; + +struct imsic_global_config +{ + /* + * MSI Target Address Scheme + * + * XLEN-1 12 0 + * | | | + * ------------------------------------------------------------- + * |xxxxxx|Group Index|xxxxxxxxxxx|HART Index|Guest Index| 0 | + * ------------------------------------------------------------- + */ + + /* Bits representing Guest index, HART index, and Group index */ + rt_uint32_t guest_index_bits; + rt_uint32_t hart_index_bits; + rt_uint32_t group_index_bits; + rt_uint32_t group_index_shift; + + /* Global base address matching all target MSI addresses */ + rt_ubase_t base_addr; + + /* Number of interrupt identities */ + rt_uint32_t nr_ids; + + /* Number of guest interrupt identities */ + rt_uint32_t nr_guest_ids; + + /* IMSIC addresses */ + struct imsic_local_config local[RT_CPUS_NR]; +}; + +const struct imsic_global_config *imsic_global_config_read(struct rt_ofw_node *np); +rt_bool_t imsic_ofw_is_available(void); +rt_bool_t imsic_ofw_is_current_mode(struct rt_ofw_node *np); +rt_err_t imsic_ofw_probe(struct rt_ofw_node *np, + struct rt_ofw_node *ic_np, const struct rt_ofw_node_id *ic_id); + +#endif /* __PIC_RISCV_IMSIC_H__ */ diff --git a/libcpu/risc-v/common/pic/pic-riscv-intc-andes.h b/libcpu/risc-v/common/pic/pic-riscv-intc-andes.h new file mode 100644 index 0000000000..e9358510db --- /dev/null +++ b/libcpu/risc-v/common/pic/pic-riscv-intc-andes.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-01-30 GuEe-GUI first version + */ + +#ifndef __PIC_RISCV_INTC_ANDES_H__ +#define __PIC_RISCV_INTC_ANDES_H__ + +/* Andes PMU irq number */ +#define ANDES_RV_IRQ_PMOVI 18 +#define ANDES_RV_IRQ_LAST ANDES_RV_IRQ_PMOVI +#define ANDES_SLI_CAUSE_BASE 256 + +/* Andes PMU related registers */ +#define ANDES_CSR_SLIE 0x9c4 +#define ANDES_CSR_SLIP 0x9c5 +#define ANDES_CSR_SCOUNTEROF 0x9d4 + +#endif /* __PIC_RISCV_INTC_ANDES_H__ */ diff --git a/libcpu/risc-v/common/pic/pic-riscv-intc.c b/libcpu/risc-v/common/pic/pic-riscv-intc.c new file mode 100644 index 0000000000..238634d85c --- /dev/null +++ b/libcpu/risc-v/common/pic/pic-riscv-intc.c @@ -0,0 +1,521 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-01-30 GuEe-GUI first version + */ + +#include <rthw.h> +#include <rtatomic.h> +#include <rtthread.h> +#ifndef PIC_DISABLE_DM +#include <rtdevice.h> +#endif + +#define DBG_TAG "pic.riscv" +#define DBG_LVL DBG_INFO +#include <rtdbg.h> + +#include <csr.h> +#include <sbi.h> +#include <cpu.h> +#include <isa_ext.h> +#include <riscv_timer.h> + +#include "pic-riscv-intc-andes.h" +#include "pic-riscv-clint.h" +#ifdef RT_PIC_RISCV_AIA +#include "pic-riscv-imsic.h" +#endif + +struct riscv_intc +{ +#ifndef PIC_DISABLE_DM + struct rt_pic parent; +#endif /* !PIC_DISABLE_DM */ + + rt_uint32_t irqs_base; + rt_uint32_t irqs_count; + rt_uint32_t irqs_def_count; + +#ifdef RT_USING_SMP + rt_uint32_t ipi_base; + rt_atomic_t ipi_enable[RT_CPUS_NR]; + rt_atomic_t ipi_pending[RT_CPUS_NR]; +#endif +}; + +static struct riscv_intc system_rv_intc; + +static rt_bool_t riscv_intc_use_aia(void) +{ +#if defined(RT_PIC_RISCV_AIA) && !defined(PIC_DISABLE_DM) + return riscv_isa_ext_test(SxAIA) && imsic_ofw_is_available(); +#else + return RT_FALSE; +#endif +} + +rt_inline void riscv_intc_irq_mask_raw(struct riscv_intc *rv_intc, int hwirq) +{ +#ifdef RT_USING_SMP + if (hwirq >= rv_intc->ipi_base) + { + rt_atomic_t *ipi_enable = &rv_intc->ipi_enable[rt_hw_cpu_id()]; + + rt_hw_atomic_and(ipi_enable, ~BIT(hwirq - rv_intc->ipi_base)); + + if (rt_hw_atomic_load(ipi_enable) != 0) + { + return; + } + + hwirq = RV_IRQ_SOFT; + } +#endif /* RT_USING_SMP */ + +#if BITS_PER_LONG == 32 + if (hwirq >= BITS_PER_LONG) + { + csr_clear(CSR_IEH, BIT(hwirq - BITS_PER_LONG)); + } + else +#endif /* BITS_PER_LONG == 32 */ + { + csr_clear(CSR_IE, BIT(hwirq)); + } +} + +rt_inline void riscv_intc_irq_unmask_raw(struct riscv_intc *rv_intc, int hwirq) +{ +#ifdef RT_USING_SMP + if (hwirq >= rv_intc->ipi_base) + { + rt_atomic_t *ipi_enable = &rv_intc->ipi_enable[rt_hw_cpu_id()]; + + rt_hw_atomic_or(ipi_enable, BIT(hwirq - rv_intc->ipi_base)); + hwirq = RV_IRQ_SOFT; + } +#endif /* RT_USING_SMP */ + +#if BITS_PER_LONG == 32 + if (hwirq >= BITS_PER_LONG) + { + csr_set(CSR_IEH, BIT(hwirq - BITS_PER_LONG)); + } + else +#endif /* BITS_PER_LONG == 32 */ + { + csr_set(CSR_IE, BIT(hwirq)); + } +} + +rt_inline void andes_intc_irq_mask_raw(struct riscv_intc *rv_intc, int hwirq) +{ + /* + * Andes specific S-mode local interrupt causes (hwirq) + * are defined as (256 + n) and controlled by n-th bit of SLIE. + */ + rt_uint32_t mask; + +#ifdef RT_USING_SMP + if (hwirq >= rv_intc->ipi_base) + { + rt_atomic_t *ipi_enable = &rv_intc->ipi_enable[rt_hw_cpu_id()]; + + rt_hw_atomic_and(ipi_enable, ~BIT(hwirq - rv_intc->ipi_base)); + + if (rt_hw_atomic_load(ipi_enable) != 0) + { + return; + } + + hwirq = RV_IRQ_SOFT; + } +#endif /* RT_USING_SMP */ + + mask = BIT(hwirq % BITS_PER_LONG); + + if (hwirq < ANDES_SLI_CAUSE_BASE) + { + csr_clear(CSR_IE, mask); + } + else + { + csr_clear(ANDES_CSR_SLIE, mask); + } +} + +rt_inline void andes_intc_irq_unmask_raw(struct riscv_intc *rv_intc, int hwirq) +{ + rt_uint32_t mask; + +#ifdef RT_USING_SMP + if (hwirq >= rv_intc->ipi_base) + { + rt_atomic_t *ipi_enable = &rv_intc->ipi_enable[rt_hw_cpu_id()]; + + rt_hw_atomic_or(ipi_enable, BIT(hwirq - rv_intc->ipi_base)); + hwirq = RV_IRQ_SOFT; + } +#endif /* RT_USING_SMP */ + + mask = BIT(hwirq % BITS_PER_LONG); + + if (hwirq < ANDES_SLI_CAUSE_BASE) + { + csr_set(CSR_IE, mask); + } + else + { + csr_set(ANDES_CSR_SLIE, mask); + } +} + +rt_inline void riscv_intc_irq_send_ipi_raw(struct riscv_intc *rv_intc, int ipi, rt_bitmap_t *cpumask) +{ +#if defined(RT_USING_SMP) || defined(ARCH_RISCV_M_MODE) + int cpu; + +#ifdef RT_USING_SMP + RT_ASSERT(ipi >= rv_intc->ipi_base && ipi < rv_intc->ipi_base + RT_MAX_IPI); + ipi -= rv_intc->ipi_base; +#endif + + rt_bitmap_for_each_set_bit(cpumask, cpu, RT_CPUS_NR) + { + if (!riscv_cpu_id_available(cpu)) + { + continue; + } + + #ifdef RT_USING_SMP + rt_hw_atomic_or(&rv_intc->ipi_pending[cpu], BIT(ipi)); + #endif + + #ifndef ARCH_RISCV_M_MODE + if (riscv_aclint_ipi_base) + #endif + { + clint_send_ipi(cpu); + } + } +#endif /* RT_USING_SMP || ARCH_RISCV_M_MODE */ + +#ifndef ARCH_RISCV_M_MODE + if (!riscv_aclint_ipi_base) + { + sbi_send_ipi((const unsigned long *)cpumask); + } +#endif /* !ARCH_RISCV_M_MODE */ +} + +rt_inline rt_bool_t riscv_intc_common_handler_raw(struct riscv_intc *rv_intc, int hwirq, + rt_bool_t (*callback)(int hwirq, void *data), void *data) +{ +#ifdef RT_USING_SMP + if (hwirq != RV_IRQ_SOFT) +#endif + { + if (hwirq != RV_IRQ_TIMER) + { + return callback(hwirq, data); + } + else + { + riscv_timer_isr(); + return RT_TRUE; + } + } +#ifdef RT_USING_SMP + else + { + int ipi; + rt_bool_t res = RT_TRUE; + rt_bitmap_t pending; + + #ifndef ARCH_RISCV_M_MODE + if (riscv_aclint_ipi_base) + #endif + { + clint_clear_ipi(); + } + #ifndef ARCH_RISCV_M_MODE + else + { + csr_clear(CSR_IP, IE_SIE); + } + #endif + + pending = rt_hw_atomic_exchange(&rv_intc->ipi_pending[rt_hw_cpu_id()], 0); + + rt_bitmap_for_each_set_bit(&pending, ipi, RT_MAX_IPI) + { + res = callback(rv_intc->ipi_base + ipi, data) || res; + } + + return res; + } +#endif /* RT_USING_SMP */ +} + +rt_inline rt_bool_t riscv_intc_handler_raw(struct riscv_intc *rv_intc, rt_ubase_t scause, + rt_bool_t (*callback)(int hwirq, void *data), void *data) +{ + return riscv_intc_common_handler_raw(rv_intc, scause & ~CAUSE_IRQ_FLAG, callback, data); +} + +rt_inline rt_bool_t riscv_intc_aia_handler_raw(struct riscv_intc *rv_intc, + rt_bool_t (*callback)(int hwirq, void *data), void *data) +{ + rt_ubase_t topi; + rt_bool_t res = RT_FALSE; + + while ((topi = csr_read(CSR_TOPI) >> TOPI_IID_SHIFT)) + { + res = riscv_intc_common_handler_raw(rv_intc, topi, callback, data) || res; + } + + return res; +} + +static void riscv_intc_common_init(struct riscv_intc *rv_intc, rt_bool_t is_andes) +{ + rt_memset(rv_intc, 0, sizeof(*rv_intc)); + rv_intc->irqs_def_count = RT_BITS_PER_LONG; + + if (is_andes) + { + rv_intc->irqs_base = ANDES_SLI_CAUSE_BASE; + rv_intc->irqs_count = ANDES_RV_IRQ_LAST; + } + else + { + rv_intc->irqs_base = BITS_PER_LONG; + rv_intc->irqs_count = 0; + } + + if (riscv_intc_use_aia()) + { + rv_intc->irqs_def_count = 64; + } + else + { + rv_intc->irqs_def_count = IRQ_LOCAL_MAX; + } + +#ifdef RT_USING_SMP + rv_intc->irqs_def_count += RT_MAX_IPI; +#endif +} + +#ifndef PIC_DISABLE_DM +static void riscv_intc_irq_mask(struct rt_pic_irq *pirq) +{ + RT_IRQ_AFFINITY_CLEAR(pirq->affinity, rt_hw_cpu_id()); + riscv_intc_irq_mask_raw(pirq->pic->priv_data, pirq->hwirq); +} + +static void riscv_intc_irq_unmask(struct rt_pic_irq *pirq) +{ + RT_IRQ_AFFINITY_SET(pirq->affinity, rt_hw_cpu_id()); + riscv_intc_irq_unmask_raw(pirq->pic->priv_data, pirq->hwirq); +} + +static void andes_intc_irq_mask(struct rt_pic_irq *pirq) +{ + andes_intc_irq_mask_raw(pirq->pic->priv_data, pirq->hwirq); + RT_IRQ_AFFINITY_CLEAR(pirq->affinity, rt_hw_cpu_id()); +} + +static void andes_intc_irq_unmask(struct rt_pic_irq *pirq) +{ + RT_IRQ_AFFINITY_SET(pirq->affinity, rt_hw_cpu_id()); + andes_intc_irq_unmask_raw(pirq->pic->priv_data, pirq->hwirq); +} + +static void riscv_intc_irq_send_ipi(struct rt_pic_irq *pirq, rt_bitmap_t *cpumask) +{ + struct riscv_intc *rv_intc = pirq->pic->priv_data; + + riscv_intc_irq_send_ipi_raw(rv_intc, pirq->hwirq, cpumask); +} + +static int riscv_intc_irq_map(struct rt_pic *pic, int hwirq, rt_uint32_t mode) +{ + int irq, irq_index = -1; + struct riscv_intc *rv_intc = rt_container_of(pic, struct riscv_intc, parent); + + if (hwirq >= rv_intc->irqs_def_count && + (hwirq < rv_intc->irqs_base || hwirq >= rv_intc->irqs_base + rv_intc->irqs_count)) + { + return -RT_EINVAL; + } + + if (hwirq < rv_intc->irqs_def_count) + { + irq_index = hwirq; + } + else if (hwirq >= rv_intc->irqs_base && + hwirq < rv_intc->irqs_base + rv_intc->irqs_count) + { + irq_index = rv_intc->irqs_def_count + hwirq - rv_intc->irqs_base; + } + + if (irq_index >= 0) + { + irq = rt_pic_config_irq(pic, hwirq, irq_index); + } + else + { + irq = -1; + } + + return irq; +} + +static rt_err_t riscv_intc_irq_parse(struct rt_pic *pic, + struct rt_ofw_cell_args *args, struct rt_pic_irq *out_pirq) +{ + rt_err_t err = RT_EOK; + + if (args->args_count == 1) + { + out_pirq->hwirq = args->args[0]; + out_pirq->mode = RT_IRQ_MODE_EDGE_RISING; + } + else + { + err = -RT_EINVAL; + } + + return err; +} + +const static struct rt_pic_ops riscv_intc_ops = +{ + .name = "RISC-V-INTC", + .irq_mask = riscv_intc_irq_mask, + .irq_unmask = riscv_intc_irq_unmask, + .irq_send_ipi = riscv_intc_irq_send_ipi, + .irq_map = riscv_intc_irq_map, + .irq_parse = riscv_intc_irq_parse, +}; + +const static struct rt_pic_ops andes_intc_ops = +{ + .name = "ANDES-INTC", + .irq_mask = andes_intc_irq_mask, + .irq_unmask = andes_intc_irq_unmask, + .irq_send_ipi = riscv_intc_irq_send_ipi, + .irq_map = riscv_intc_irq_map, + .irq_parse = riscv_intc_irq_parse, +}; + +static rt_bool_t riscv_intc_common_handler_isr(int hwirq, void *data) +{ + struct rt_pic_irq *pirq; + struct riscv_intc *rv_intc = data; + +#ifdef RT_USING_SMP + if (hwirq < rv_intc->ipi_base) +#endif + { + pirq = rt_pic_find_irq(&rv_intc->parent, hwirq); + } +#ifdef RT_USING_SMP + else + { + pirq = rt_pic_find_ipi(&rv_intc->parent, hwirq - rv_intc->ipi_base); + } +#endif + + return rt_pic_handle_isr(pirq) == RT_EOK; +} + +static rt_bool_t riscv_intc_handler(void *data) +{ + rt_ubase_t cause; + struct riscv_intc *rv_intc = data; + + cause = csr_read(CSR_CAUSE); + + return riscv_intc_handler_raw(rv_intc, cause, riscv_intc_common_handler_isr, data); +} + +static rt_bool_t riscv_intc_aia_handler(void *data) +{ + struct riscv_intc *rv_intc = data; + rt_ubase_t cause; + + cause = csr_read(CSR_CAUSE) & ~CAUSE_IRQ_FLAG; + if (cause == RV_IRQ_SOFT) + { + return riscv_intc_common_handler_raw(rv_intc, cause, + riscv_intc_common_handler_isr, data); + } + + return riscv_intc_aia_handler_raw(rv_intc, riscv_intc_common_handler_isr, data); +} + +static rt_err_t riscv_intc_ofw_init(struct rt_ofw_node *np, const struct rt_ofw_node_id *id) +{ + const struct rt_pic_ops *ops; + struct riscv_intc *rv_intc = &system_rv_intc; + struct rt_ofw_node *pnp = rt_ofw_get_parent(np); + + /* Only boot CPU to init */ + if (rt_ofw_get_cpu_hwid(pnp, 0) != boot_cpu_hartid) + { + rt_ofw_node_put(pnp); + rt_ofw_data(np) = &rv_intc->parent; + + return RT_EOK; + } + rt_ofw_node_put(pnp); + + ops = id->data; + riscv_intc_common_init(rv_intc, ops == &andes_intc_ops); + + riscv_clint_ofw_init(); + + if (riscv_intc_use_aia()) + { + rt_pic_add_traps(riscv_intc_aia_handler, rv_intc); + } + else + { + rt_pic_add_traps(riscv_intc_handler, rv_intc); + } + + rv_intc->parent.priv_data = rv_intc; + rv_intc->parent.ops = ops; + rt_ofw_data(np) = &rv_intc->parent; + + rt_pic_linear_irq(&rv_intc->parent, rv_intc->irqs_def_count + rv_intc->irqs_count); + +#ifdef RT_USING_SMP + rv_intc->ipi_base = rv_intc->parent.irq_nr - RT_MAX_IPI; + + for (int ipi = 0; ipi < RT_MAX_IPI; ++ipi) + { + rt_pic_config_ipi(&rv_intc->parent, ipi, rv_intc->ipi_base + ipi); + } +#endif /* RT_USING_SMP */ + + rt_pic_user_extends(&rv_intc->parent); + + return RT_EOK; +} + +static const struct rt_ofw_node_id riscv_intc_ofw_ids[] = +{ + { .compatible = "riscv,cpu-intc", (void *)&riscv_intc_ops }, + { .compatible = "andestech,cpu-intc", (void *)&andes_intc_ops }, + { /* sentinel */ } +}; +RT_PIC_OFW_DECLARE(riscv_intc, riscv_intc_ofw_ids, riscv_intc_ofw_init); +#endif /* !PIC_DISABLE_DM */ diff --git a/libcpu/risc-v/common/pic/pic-sifive-plic.c b/libcpu/risc-v/common/pic/pic-sifive-plic.c new file mode 100644 index 0000000000..393b64a8a0 --- /dev/null +++ b/libcpu/risc-v/common/pic/pic-sifive-plic.c @@ -0,0 +1,528 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-01-30 GuEe-GUI first version + */ + +#include <rthw.h> +#include <rtthread.h> +#ifndef PIC_DISABLE_DM +#include <rtdevice.h> +#endif + +#define DBG_TAG "pic.plic" +#define DBG_LVL DBG_INFO +#include <rtdbg.h> + +#include <csr.h> +#include <cpu.h> +#include <cache.h> +#include <bitmap.h> + +/* + * This driver implements a version of the RISC-V PLIC with the actual layout + * specified in chapter 8 of the SiFive U5 Coreplex Series Manual: + * + * https://static.dev.sifive.com/U54-MC-RVCoreIP.pdf + */ + +#define MAX_DEVICES 1024 +#define MAX_CONTEXTS 15872 + +/* + * Each interrupt source has a priority register associated with it. + * We always hardwire it to one in Linux. + */ +#define PRIORITY_BASE 0 +#define PRIORITY_PER_ID 4 + +#ifdef ARCH_RISCV_M_MODE +#define CONTEXT_ENABLE_MODE_OFFSET 0 +#define CONTEXT_BASE_MODE_OFFSET 0 +#else +#define CONTEXT_ENABLE_MODE_OFFSET 0x80 +#define CONTEXT_BASE_MODE_OFFSET 0x1000 +#endif + +/* + * Each hart context has a vector of interrupt enable bits associated with it. + * There's one bit for each interrupt source. + */ +#define CONTEXT_ENABLE_BASE (0x2000 + CONTEXT_ENABLE_MODE_OFFSET) +#define CONTEXT_ENABLE_SIZE(hartid) ((2 * hartid) * 0x80) + +/* + * Each hart context has a set of control registers associated with it. Right + * now there's only two: a source priority threshold over which the hart will + * take an interrupt, and a register to claim interrupts. + */ +#define CONTEXT_BASE (0x200000 + CONTEXT_BASE_MODE_OFFSET) +#define CONTEXT_SIZE(hartid) ((2 * hartid) * 0x1000) +#define CONTEXT_THRESHOLD 0x00 +#define CONTEXT_CLAIM 0x04 + +#define PLIC_DISABLE_THRESHOLD 0x7 +#define PLIC_ENABLE_THRESHOLD 0 + +#define PLIC_HWIRQ_START 1 + +#define PLIC_QUIRK_EDGE_INTERRUPT 0 + +struct plic +{ +#ifndef PIC_DISABLE_DM + struct rt_pic parent; + struct rt_ofw_node *np; + + int parent_irq; +#endif + + void *regs; + + rt_bitmap_t quirks; + + rt_uint32_t irqs_count; +}; + +struct plic_percpu +{ + void *hart_base; + void *enable_base; + rt_bool_t valid; + + struct rt_spinlock enable_lock; +}; + +static struct plic_percpu plic_percpu[RT_CPUS_NR] rt_align(L1_CACHE_BYTES); + +static void plic_set_threshold_raw(struct plic_percpu *percpu, rt_uint32_t threshold) +{ + if (!percpu->valid) + { + return; + } + + /* Priority must be > threshold to trigger an interrupt */ + HWREG32(percpu->hart_base + CONTEXT_THRESHOLD) = threshold; +} + +static void plic_toggle(struct plic_percpu *percpu, int hwirq, rt_bool_t enable) +{ + rt_ubase_t level; + rt_uint32_t *reg, hwirq_mask; + + if (!percpu->valid) + { + return; + } + + level = rt_spin_lock_irqsave(&percpu->enable_lock); + +#ifdef SOC_SPACEMIT_K1 + if (!enable) + { + HWREG32(percpu->hart_base + CONTEXT_CLAIM) = hwirq; + } +#endif + + reg = percpu->enable_base + (hwirq / 32) * sizeof(rt_uint32_t); + hwirq_mask = 1 << (hwirq % 32); + + if (enable) + { + HWREG32(reg) |= hwirq_mask; + } + else + { + HWREG32(reg) &= ~hwirq_mask; + } + +#ifdef SOC_SPACEMIT_K1 + if (enable) + { + HWREG32(percpu->hart_base + CONTEXT_CLAIM) = hwirq; + } +#endif + + rt_spin_unlock_irqrestore(&percpu->enable_lock, level); +} + +static void plic_irq_unmask_raw(int cpu, int hwirq) +{ + struct plic_percpu *percpu = &plic_percpu[cpu]; + + plic_toggle(percpu, hwirq, RT_TRUE); +} + +#if !defined(SOC_SPACEMIT_K1) || defined(PIC_DISABLE_DM) +static void plic_irq_mask_raw(int cpu, int hwirq) +{ + struct plic_percpu *percpu = &plic_percpu[cpu]; + + plic_toggle(percpu, hwirq, RT_FALSE); +} +#endif + +static void plic_irq_ack_or_eoi(int cpu, int hwirq) +{ + struct plic_percpu *percpu = &plic_percpu[cpu]; + + if (percpu->valid) + { + HWREG32(percpu->hart_base + CONTEXT_CLAIM) = hwirq; + } +} + +static rt_bool_t plic_handler_isr_raw(struct plic *plic, + rt_bool_t (*callback)(int hwirq, void *data), void *data) +{ + int hwirq; + void *claim; + rt_bool_t res = RT_FALSE; + struct plic_percpu *percpu = &plic_percpu[rt_hw_cpu_id()]; + + claim = percpu->hart_base + CONTEXT_CLAIM; + + while ((hwirq = HWREG32(claim))) + { + res = callback(hwirq, data) || res; + } + + return res; +} + +static void plic_percpu_common_init(struct plic *plic) +{ + struct plic_percpu *percpu = &plic_percpu[rt_hw_cpu_id()]; + + plic_set_threshold_raw(percpu, PLIC_ENABLE_THRESHOLD); +} + +static struct plic *plic_common_init(rt_ubase_t plic_base, rt_size_t plic_size, + rt_size_t ndev, rt_bitmap_t quirks) +{ +#ifdef PIC_DISABLE_DM + static struct plic non_dm_plic; + struct plic *plic = &non_dm_plic; + + rt_memset(plic, 0, sizeof(*plic)); +#else + struct plic *plic = rt_calloc(1, sizeof(*plic)); + + if (!plic) + { + rt_set_errno(-RT_ENOMEM); + return RT_NULL; + } +#endif + + plic->regs = rt_ioremap((void *)plic_base, plic_size); + + if (!plic->regs) + { + rt_set_errno(-RT_EIO); + +#ifndef PIC_DISABLE_DM + rt_free(plic); +#endif + return RT_NULL; + } + + plic->quirks = quirks; + plic->irqs_count = ndev; + + for (int cpu = 0, hartid; cpu < RT_CPUS_NR; ++cpu) + { + struct plic_percpu *percpu = &plic_percpu[cpu]; + + #ifndef RT_USING_SMP + hartid = boot_cpu_hartid; + #else + if (!riscv_cpu_id_available(cpu)) + { + continue; + } + + hartid = riscv_cpu_id_to_hartid(cpu); + #endif + + percpu->hart_base = plic->regs + CONTEXT_BASE + CONTEXT_SIZE(hartid); + percpu->enable_base = plic->regs + CONTEXT_ENABLE_BASE + CONTEXT_ENABLE_SIZE(hartid); + percpu->valid = RT_TRUE; + + rt_spin_lock_init(&percpu->enable_lock); + } + + /* Set default priority */ + for (int hwirq = PLIC_HWIRQ_START; hwirq <= plic->irqs_count; ++hwirq) + { + HWREG32(plic->regs + PRIORITY_BASE + hwirq * PRIORITY_PER_ID) = PLIC_ENABLE_THRESHOLD + 1; + } + + return plic; +} + +#ifndef PIC_DISABLE_DM +static void plic_handler(int hwirq, void *data); + +static rt_err_t plic_irq_init(struct rt_pic *pic) +{ + struct plic *plic = pic->priv_data; + + if (plic->parent_irq < 0) + { + struct rt_ofw_cell_args irq_args; + + if (rt_ofw_parse_irq_cells(plic->np, 0, &irq_args)) + { + LOG_E("%s: Parent IRQ is not available", rt_ofw_node_full_name(plic->np)); + goto _percpu_init; + } + + /* Create mapping */ + irq_args.args[0] = RV_IRQ_EXT; + + plic->parent_irq = rt_ofw_map_irq(&irq_args); + rt_hw_interrupt_install(plic->parent_irq, plic_handler, plic, "PLIC"); + } + + rt_hw_interrupt_umask(plic->parent_irq); + +_percpu_init: + plic_percpu_common_init(plic); + + return RT_EOK; +} + +static void plic_irq_ack(struct rt_pic_irq *pirq) +{ + if (pirq->mode == RT_IRQ_MODE_EDGE_RISING) + { + plic_irq_ack_or_eoi(rt_hw_cpu_id(), pirq->hwirq); + } +} + +static void plic_irq_unmask(struct rt_pic_irq *pirq) +{ + int cpu, count = 0; +#ifdef SOC_SPACEMIT_K1 + struct plic *plic = pirq->pic->priv_data; +#endif + + rt_bitmap_for_each_set_bit(pirq->affinity, cpu, RT_CPUS_NR) + { + plic_irq_unmask_raw(cpu, pirq->hwirq); + ++count; + } + + if (!count) + { + int boot_cpu_id = riscv_hartid_to_cpu_id(boot_cpu_hartid); + + RT_IRQ_AFFINITY_SET(pirq->affinity, boot_cpu_id); + plic_irq_unmask_raw(boot_cpu_id, pirq->hwirq); + } + +#ifdef SOC_SPACEMIT_K1 + HWREG32(plic->regs + PRIORITY_BASE + pirq->hwirq * PRIORITY_PER_ID) = + PLIC_ENABLE_THRESHOLD + 1; +#endif +} + +static void plic_irq_mask(struct rt_pic_irq *pirq) +{ +#ifdef SOC_SPACEMIT_K1 + struct plic *plic = pirq->pic->priv_data; + + /* K1 masks a live source by priority; keep its context enable bit set. */ + HWREG32(plic->regs + PRIORITY_BASE + pirq->hwirq * PRIORITY_PER_ID) = + PLIC_ENABLE_THRESHOLD; +#else + int cpu; + + rt_bitmap_for_each_set_bit(pirq->affinity, cpu, RT_CPUS_NR) + { + plic_irq_mask_raw(cpu, pirq->hwirq); + } +#endif +} + +static void plic_irq_eoi(struct rt_pic_irq *pirq) +{ + if (pirq->mode == RT_IRQ_MODE_LEVEL_HIGH) + { + plic_irq_ack_or_eoi(rt_hw_cpu_id(), pirq->hwirq); + } +} + +static rt_err_t plic_irq_set_priority(struct rt_pic_irq *pirq, rt_uint32_t priority) +{ + struct plic *plic = pirq->pic->priv_data; + + priority = rt_min_t(rt_uint32_t, priority, PLIC_DISABLE_THRESHOLD); + + HWREG32(plic->regs + PRIORITY_BASE + pirq->hwirq * PRIORITY_PER_ID) = priority; + + return RT_EOK; +} + +static rt_err_t plic_irq_set_affinity(struct rt_pic_irq *pirq, rt_bitmap_t *affinity) +{ + int cpu, count = 0; + + rt_bitmap_for_each_clear_bit(affinity, cpu, RT_CPUS_NR) + { + plic_toggle(&plic_percpu[cpu], pirq->hwirq, RT_FALSE); + } + + rt_bitmap_for_each_set_bit(affinity, cpu, RT_CPUS_NR) + { + if (!plic_percpu[cpu].valid) + { + continue; + } + + plic_toggle(&plic_percpu[cpu], pirq->hwirq, RT_TRUE); + ++count; + } + + return count ? RT_EOK : -RT_EINVAL; +} + +static rt_err_t plic_irq_set_triger_mode(struct rt_pic_irq *pirq, rt_uint32_t mode) +{ + struct plic *plic = pirq->pic->priv_data; + + if (rt_bitmap_test_bit(&plic->quirks, PLIC_QUIRK_EDGE_INTERRUPT)) + { + return -RT_ENOSYS; + } + + if (mode == RT_IRQ_MODE_EDGE_RISING || mode == RT_IRQ_MODE_LEVEL_HIGH) + { + return RT_EOK; + } + + return -RT_ENOSYS; +} + +static int plic_irq_map(struct rt_pic *pic, int hwirq, rt_uint32_t mode) +{ + struct rt_pic_irq *pirq = rt_pic_find_irq(pic, hwirq - PLIC_HWIRQ_START); + + pirq->mode = mode; + + return rt_pic_config_irq(pic, hwirq - PLIC_HWIRQ_START, hwirq); +} + +static rt_err_t plic_irq_parse(struct rt_pic *pic, struct rt_ofw_cell_args *args, struct rt_pic_irq *out_pirq) +{ + struct plic *plic = pic->priv_data; + + out_pirq->hwirq = args->args[0]; + + if (rt_bitmap_test_bit(&plic->quirks, PLIC_QUIRK_EDGE_INTERRUPT)) + { + out_pirq->mode = args->args[1]; + } + else + { + out_pirq->mode = RT_IRQ_MODE_LEVEL_HIGH; + } + + return RT_EOK; +} + +const static struct rt_pic_ops plic_ops = +{ + .name = "SiFive-PLIC", + .irq_init = plic_irq_init, + .irq_ack = plic_irq_ack, + .irq_mask = plic_irq_mask, + .irq_unmask = plic_irq_unmask, + .irq_eoi = plic_irq_eoi, + .irq_set_priority = plic_irq_set_priority, + .irq_set_affinity = plic_irq_set_affinity, + .irq_set_triger_mode = plic_irq_set_triger_mode, + .irq_map = plic_irq_map, + .irq_parse = plic_irq_parse, +}; + +static rt_bool_t plic_common_handler_isr(int hwirq, void *data) +{ + rt_bool_t res; + struct plic *plic = data; + struct rt_pic_irq *pirq = rt_pic_find_irq(&plic->parent, hwirq - PLIC_HWIRQ_START); + + plic_irq_ack(pirq); + + res = rt_pic_handle_isr(pirq) == RT_EOK; + + plic_irq_eoi(pirq); + + return res; +} + +static void plic_handler(int hwirq, void *data) +{ + struct plic *plic = data; + + plic_handler_isr_raw(plic, plic_common_handler_isr, plic); +} + +static rt_err_t plic_ofw_init(struct rt_ofw_node *np, const struct rt_ofw_node_id *id) +{ + rt_err_t err; + rt_uint32_t irqs_count; + rt_uint64_t plic_base, plic_size; + rt_bitmap_t quirks; + struct plic *plic; + + if ((err = rt_ofw_get_address(np, 0, &plic_base, &plic_size))) + { + return err; + } + + if ((err = rt_ofw_prop_read_u32(np, "riscv,ndev", &irqs_count))) + { + return err; + } + + quirks = id->data ? *(rt_ubase_t *)id->data : 0; + + if (!(plic = plic_common_init(plic_base, plic_size, irqs_count, quirks))) + { + return rt_get_errno(); + } + + plic->np = np; + plic->parent_irq = -1; + + plic->parent.priv_data = plic; + plic->parent.ops = &plic_ops; + rt_ofw_data(np) = &plic->parent; + + rt_pic_linear_irq(&plic->parent, plic->irqs_count); + + rt_pic_user_extends(&plic->parent); + + return RT_EOK; +} + +static rt_bitmap_t andestech_nceplic100_quirk_edge = RT_BIT(PLIC_QUIRK_EDGE_INTERRUPT); +static rt_bitmap_t thead_c900_quirk_edge = RT_BIT(PLIC_QUIRK_EDGE_INTERRUPT); + +static const struct rt_ofw_node_id plic_ofw_ids[] = +{ + { .compatible = "sifive,plic-1.0.0" }, + { .compatible = "riscv,plic0" }, + { .compatible = "andestech,nceplic100", .data = &andestech_nceplic100_quirk_edge }, + { .compatible = "thead,c900-plic", .data = &thead_c900_quirk_edge }, + { /* sentinel */ } +}; +RT_PIC_OFW_DECLARE(plic, plic_ofw_ids, plic_ofw_init); +#endif /* PIC_DISABLE_DM */ diff --git a/libcpu/risc-v/common/riscv_timer.c b/libcpu/risc-v/common/riscv_timer.c new file mode 100644 index 0000000000..8ff871c2d1 --- /dev/null +++ b/libcpu/risc-v/common/riscv_timer.c @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2018-10-28 Bernard The unify RISC-V porting code. + * 2024-07-08 Shell Using CPUTIME as tick + * 2025-04-20 GuEe-GUI Port to the DM + */ + +#include <rtthread.h> + +#include <sbi.h> +#include <cpu.h> +#include <csr.h> +#include <cpuport.h> +#include <isa_ext.h> +#ifdef RT_USING_CLOCK_TIME +#include <drivers/clock_time.h> +#endif + +static rt_uint32_t timebase_frequency; +static volatile rt_ubase_t tick_cycles; + +#ifndef ARCH_RISCV_M_MODE +static rt_bool_t riscv_sstc_available; +#endif + +#ifdef ARCH_RISCV_M_MODE +static void *riscv_clint_timer_val, *riscv_clint_timer_cmp; +#endif + +void riscv_timer_set_frequency(rt_uint32_t freq) +{ + HWREG32(&timebase_frequency) = freq; + rt_hw_wmb(); +} + +rt_uint32_t riscv_timer_get_frequency(void) +{ + rt_hw_rmb(); + return HWREG32(&timebase_frequency); +} + +rt_weak rt_uint64_t rt_hw_get_clock_timer_freq(void) +{ + rt_uint32_t freq = riscv_timer_get_frequency(); + +#ifdef CLOCK_TIMER_FREQ + if (!freq) + { + freq = CLOCK_TIMER_FREQ; + } +#endif + + return freq; +} + +rt_weak rt_uint64_t riscv_timer_rdtime(void) +{ +#ifdef ARCH_RISCV_M_MODE + RT_ASSERT(riscv_clint_timer_val != RT_NULL); + if (!riscv_clint_timer_val) + { + return 0; + } + +#if __riscv_xlen == 64 + return HWREG64(riscv_clint_timer_val); +#else + volatile rt_uint32_t *mtime = riscv_clint_timer_val; + rt_uint32_t hi, lo; + + do + { + hi = HWREG32(mtime + 1); + lo = HWREG32(mtime); + } while (hi != HWREG32(mtime + 1)); + + return ((rt_uint64_t)hi << 32) | lo; +#endif /* __riscv_xlen == 64 */ +#else +#if __riscv_xlen == 64 + uint64_t cycles64; + + __asm__ volatile ("rdtime %0":"=r"(cycles64)); + + return cycles64; +#else + rt_uint32_t hi, lo, hi_check; + + do + { + __asm__ volatile ("rdtimeh %0" : "=r"(hi)); + __asm__ volatile ("rdtime %0" : "=r"(lo)); + __asm__ volatile ("rdtimeh %0" : "=r"(hi_check)); + } while (hi != hi_check); + + return ((rt_uint64_t)hi << 32) | lo; +#endif /* __riscv_xlen == 64 */ +#endif /* ARCH_RISCV_M_MODE */ +} + +rt_weak void riscv_timer_set_compare(rt_uint64_t value) +{ +#ifdef ARCH_RISCV_M_MODE + rt_ubase_t hartid = csr_read(CSR_MHARTID); + volatile rt_uint32_t *mtimecmp; + + RT_ASSERT(riscv_clint_timer_cmp != RT_NULL); + if (!riscv_clint_timer_cmp) + { + return; + } + + mtimecmp = (void *)((rt_ubase_t)riscv_clint_timer_cmp + + hartid * sizeof(rt_uint64_t)); + +#if __riscv_xlen == 64 + HWREG64(mtimecmp) = value; +#else + HWREG32(mtimecmp + 1) = RT_UINT32_MAX; + HWREG32(mtimecmp) = value & RT_UINT32_MAX; + HWREG32(mtimecmp + 1) = value >> 32; +#endif +#else + if (riscv_sstc_available) + { +#if __riscv_xlen == 64 + csr_write(CSR_STIMECMP, value); +#else + csr_write(CSR_STIMECMPH, RT_UINT32_MAX); + csr_write(CSR_STIMECMP, value & RT_UINT32_MAX); + csr_write(CSR_STIMECMPH, value >> 32); +#endif + } + else + { + sbi_set_timer(value); + } +#endif /* !ARCH_RISCV_M_MODE */ +} + +static void riscv_timer_next_event(void) +{ + riscv_timer_set_compare(riscv_timer_rdtime() + tick_cycles); +} + +void riscv_timer_isr(void) +{ + rt_tick_increase(); + + riscv_timer_next_event(); +} + +int tick_isr(void) +{ + riscv_timer_isr(); + + return 0; +} + +/* Sets and enable the timer interrupt */ +void riscv_timer_init(void) +{ + rt_uint32_t freq = riscv_timer_get_frequency(); + +#ifdef RT_USING_SMP + if (rt_hw_cpu_id() == riscv_hartid_to_cpu_id(boot_cpu_hartid)) +#endif + { + if (!freq && (freq = (rt_uint32_t)rt_hw_get_clock_timer_freq())) + { + riscv_timer_set_frequency(freq); + } + + RT_ASSERT(freq >= RT_TICK_PER_SECOND); + if (freq < RT_TICK_PER_SECOND) + { + return; + } + + #ifdef RT_USING_CLOCK_TIME + rt_clock_time_source_init(); + #endif + + /* Calculate the tick cycles */ + tick_cycles = freq / RT_TICK_PER_SECOND; + + #ifndef ARCH_RISCV_M_MODE + /* Check if timer interrupt in S-mode is available via sstc extension. */ + riscv_sstc_available = riscv_isa_ext_test(SSTC); + #endif + + rt_hw_wmb(); + } + +#ifdef RT_USING_SMP + rt_hw_rmb(); +#endif + + RT_ASSERT(tick_cycles > 0); + if (!tick_cycles) + { + return; + } + + /* Keep the timer masked while programming its first event. */ + csr_clear(CSR_IE, BIT(RV_IRQ_TIMER)); + + /* Set timer */ + riscv_timer_next_event(); + + csr_set(CSR_IE, BIT(RV_IRQ_TIMER)); +} + +rt_weak int rt_hw_tick_init(void) +{ + riscv_timer_init(); + + return 0; +} + +void riscv_timer_probe(void *clint_timer_cmp, void *clint_timer_val) +{ +#ifdef ARCH_RISCV_M_MODE + riscv_clint_timer_cmp = clint_timer_cmp; + riscv_clint_timer_val = clint_timer_val; +#endif +} + +void riscv_timer_us_delay(rt_uint32_t us) +{ + rt_uint64_t delay_time; + + delay_time = riscv_timer_rdtime(); + delay_time += (rt_uint64_t)us * (riscv_timer_get_frequency() / 1000000); + + while (riscv_timer_rdtime() < delay_time) + { + rt_hw_cpu_relax(); + } +} + +#if !defined(RT_USING_DM) || !defined(RT_USING_HWTIMER) +/** + * This function will delay for some us. + * + * @param us the delay time of us + */ +void rt_hw_us_delay(rt_uint32_t us) +{ + riscv_timer_us_delay(us); +} +#endif /* !RT_USING_DM || !RT_USING_HWTIMER */ diff --git a/libcpu/risc-v/common/sbi.c b/libcpu/risc-v/common/sbi.c new file mode 100644 index 0000000000..70fe88c76f --- /dev/null +++ b/libcpu/risc-v/common/sbi.c @@ -0,0 +1,384 @@ +/* + * Copyright (c) 2006-2021, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2021-05-18 Bernard port from FreeBSD + */ + +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 Mitchell Horne <[email protected]> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <rtthread.h> + +#include <cpu.h> +#include <sbi.h> + + +/* SBI Implementation-Specific Definitions */ +#define OPENSBI_VERSION_MAJOR_OFFSET 16 +#define OPENSBI_VERSION_MINOR_MASK 0xFFFF + +unsigned long sbi_spec_version; +unsigned long sbi_impl_id; +unsigned long sbi_impl_version; + +bool has_time_extension = false; +bool has_ipi_extension = false; +bool has_rfnc_extension = false; +bool has_srst_extension = false; + +static rt_bool_t sbi_cpu_mask_test(const unsigned long *cpu_mask, int cpu) +{ + const int bits_per_word = sizeof(*cpu_mask) * 8; + + return !!(cpu_mask[cpu / bits_per_word] & + (1UL << (cpu % bits_per_word))); +} + +static struct sbi_ret sbi_get_spec_version(void) +{ + return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_SPEC_VERSION)); +} + +static struct sbi_ret sbi_get_impl_id(void) +{ + return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_IMPL_ID)); +} + +static struct sbi_ret sbi_get_impl_version(void) +{ + return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_IMPL_VERSION)); +} + +void sbi_print_version(void) +{ + uint32_t major; + uint32_t minor; + + /* For legacy SBI implementations. */ + if (sbi_spec_version == 0) + { + rt_kprintf("SBI: Unknown (Legacy) Implementation\n"); + rt_kprintf("SBI Specification Version: 0.1\n"); + return; + } + + switch (sbi_impl_id) + { + case (SBI_IMPL_ID_BBL): + rt_kprintf("SBI: Berkely Boot Loader %lu\n", sbi_impl_version); + break; + case (SBI_IMPL_ID_XVISOR): + rt_kprintf("SBI: eXtensible Versatile hypervISOR %lu\n", + sbi_impl_version); + break; + case (SBI_IMPL_ID_KVM): + rt_kprintf("SBI: Kernel-based Virtual Machine %lu\n", sbi_impl_version); + break; + case (SBI_IMPL_ID_RUSTSBI): + rt_kprintf("SBI: RustSBI %lu\n", sbi_impl_version); + break; + case (SBI_IMPL_ID_DIOSIX): + rt_kprintf("SBI: Diosix %lu\n", sbi_impl_version); + break; + case (SBI_IMPL_ID_OPENSBI): + major = sbi_impl_version >> OPENSBI_VERSION_MAJOR_OFFSET; + minor = sbi_impl_version & OPENSBI_VERSION_MINOR_MASK; + rt_kprintf("SBI: OpenSBI v%u.%u\n", major, minor); + break; + default: + rt_kprintf("SBI: Unrecognized Implementation: %lu\n", sbi_impl_id); + break; + } + + major = (sbi_spec_version & SBI_SPEC_VERS_MAJOR_MASK) >> + SBI_SPEC_VERS_MAJOR_OFFSET; + minor = (sbi_spec_version & SBI_SPEC_VERS_MINOR_MASK); + rt_kprintf("SBI Specification Version: %u.%u\n", major, minor); +} + +void sbi_set_timer(uint64_t val) +{ + struct sbi_ret ret; + + /* Use the TIME legacy replacement extension, if available. */ + if (has_time_extension) + { + #if __riscv_xlen == 32 + ret = SBI_CALL2(SBI_EXT_ID_TIME, SBI_TIME_SET_TIMER, + (rt_uint32_t)val, (rt_uint32_t)(val >> 32)); + #else + ret = SBI_CALL1(SBI_EXT_ID_TIME, SBI_TIME_SET_TIMER, val); + #endif + RT_ASSERT(ret.error == SBI_SUCCESS); + } + else + { + #if __riscv_xlen == 32 + (void)SBI_CALL2(SBI_SET_TIMER, 0, + (rt_uint32_t)val, (rt_uint32_t)(val >> 32)); + #else + (void)SBI_CALL1(SBI_SET_TIMER, 0, val); + #endif + } +} + +static rt_bool_t sbi_cpu_mask_to_hart_mask(const unsigned long *cpu_mask, + unsigned long *hart_mask) +{ + *hart_mask = 0; + + for (int cpu = 0; cpu < RT_CPUS_NR; ++cpu) + { + rt_ubase_t hartid; + + if (cpu_mask && !sbi_cpu_mask_test(cpu_mask, cpu)) + { + continue; + } + if (!riscv_cpu_id_available(cpu)) + { + continue; + } + + hartid = riscv_cpu_id_to_hartid(cpu); + if (hartid >= sizeof(*hart_mask) * 8) + { + return RT_FALSE; + } + + *hart_mask |= 1UL << hartid; + } + + return RT_TRUE; +} + +void sbi_send_ipi(const unsigned long *cpu_mask) +{ + struct sbi_ret ret; + + /* Use the IPI legacy replacement extension, if available. */ + if (has_ipi_extension) + { + if (!cpu_mask) + { + ret = SBI_CALL2(SBI_EXT_ID_IPI, SBI_IPI_SEND_IPI, + 0, SBI_HART_MASK_BASE_ALL); + RT_ASSERT(ret.error == SBI_SUCCESS); + } + else + { + for (int cpu = 0; cpu < RT_CPUS_NR; ++cpu) + { + rt_ubase_t hartid; + + if (!sbi_cpu_mask_test(cpu_mask, cpu) || + !riscv_cpu_id_available(cpu)) + { + continue; + } + + hartid = riscv_cpu_id_to_hartid(cpu); + ret = SBI_CALL2(SBI_EXT_ID_IPI, SBI_IPI_SEND_IPI, 1, hartid); + RT_ASSERT(ret.error == SBI_SUCCESS); + } + } + } + else + { + unsigned long hart_mask; + + if (!sbi_cpu_mask_to_hart_mask(cpu_mask, &hart_mask)) + { + RT_ASSERT(0); + return; + } + (void)SBI_CALL1(SBI_SEND_IPI, 0, (rt_ubase_t)&hart_mask); + } +} + +void sbi_remote_fence_i(const unsigned long *cpu_mask) +{ + struct sbi_ret ret; + + /* Use the RFENCE legacy replacement extension, if available. */ + if (has_rfnc_extension) + { + if (!cpu_mask) + { + ret = SBI_CALL2(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_FENCE_I, + 0, SBI_HART_MASK_BASE_ALL); + RT_ASSERT(ret.error == SBI_SUCCESS); + } + else + { + for (int cpu = 0; cpu < RT_CPUS_NR; ++cpu) + { + rt_ubase_t hartid; + + if (!sbi_cpu_mask_test(cpu_mask, cpu) || + !riscv_cpu_id_available(cpu)) + { + continue; + } + + hartid = riscv_cpu_id_to_hartid(cpu); + ret = SBI_CALL2(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_FENCE_I, 1, hartid); + RT_ASSERT(ret.error == SBI_SUCCESS); + } + } + } + else + { + unsigned long hart_mask; + + if (!sbi_cpu_mask_to_hart_mask(cpu_mask, &hart_mask)) + { + RT_ASSERT(0); + return; + } + (void)SBI_CALL1(SBI_REMOTE_FENCE_I, 0, (rt_ubase_t)&hart_mask); + } +} + +int sbi_remote_sfence_vma(const unsigned long *hart_mask, + const unsigned long hart_mask_base, + unsigned long start, unsigned long size) +{ + struct sbi_ret ret = {.error = SBI_SUCCESS}; + + /* Use the RFENCE legacy replacement extension, if available. */ + if (has_rfnc_extension) + { + ret = SBI_CALL4(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_SFENCE_VMA, *hart_mask, + hart_mask_base, start, size); + } + else + { + /* Legacy SBI uses a NULL mask to address all harts. */ + if (hart_mask_base == SBI_HART_MASK_BASE_ALL) + { + hart_mask = RT_NULL; + } + (void)SBI_CALL3(SBI_REMOTE_SFENCE_VMA, 0, (rt_ubase_t)hart_mask, start, + size); + } + return ret.error; +} + +void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, + unsigned long start, unsigned long size, + unsigned long asid) +{ + struct sbi_ret ret; + + /* Use the RFENCE legacy replacement extension, if available. */ + if (has_rfnc_extension) + { + ret = SBI_CALL5(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_SFENCE_VMA_ASID, + *hart_mask, 0, start, size, asid); + RT_ASSERT(ret.error == SBI_SUCCESS); + } + else + { + (void)SBI_CALL4(SBI_REMOTE_SFENCE_VMA_ASID, 0, (rt_ubase_t)hart_mask, + start, size, asid); + } +} + +int sbi_hsm_hart_start(unsigned long hart, unsigned long start_addr, + unsigned long priv) +{ + struct sbi_ret ret; + + ret = SBI_CALL3(SBI_EXT_ID_HSM, SBI_HSM_HART_START, hart, start_addr, priv); + return (ret.error != 0 ? (int)ret.error : 0); +} + +void sbi_hsm_hart_stop(void) +{ + (void)SBI_CALL0(SBI_EXT_ID_HSM, SBI_HSM_HART_STOP); +} + +int sbi_hsm_hart_status(unsigned long hart) +{ + struct sbi_ret ret; + + ret = SBI_CALL1(SBI_EXT_ID_HSM, SBI_HSM_HART_STATUS, hart); + + return (ret.error != 0 ? (int)ret.error : (int)ret.value); +} + +static void sbi_srst_reset(unsigned long type, unsigned long reason) +{ + SBI_CALL2(SBI_EXT_ID_SRST, SBI_SRST_FID_RESET, type, reason); +} + +void sbi_srst_reboot(unsigned long reset_type) +{ + sbi_srst_reset(reset_type, SBI_SRST_RESET_REASON_NONE); +} + +void sbi_srst_power_off(void) +{ + sbi_srst_reset(SBI_SRST_RESET_TYPE_SHUTDOWN, SBI_SRST_RESET_REASON_NONE); +} + +void sbi_init(void) +{ + struct sbi_ret sret; + + /* + * Get the spec version. For legacy SBI implementations this will + * return an error, otherwise it is guaranteed to succeed. + */ + sret = sbi_get_spec_version(); + if (sret.error != 0) + { + /* We are running a legacy SBI implementation. */ + sbi_spec_version = 0; + return; + } + + /* Set the SBI implementation info. */ + sbi_spec_version = sret.value; + sbi_impl_id = sbi_get_impl_id().value; + sbi_impl_version = sbi_get_impl_version().value; + + /* Probe for legacy replacement extensions. */ + if (sbi_probe_extension(SBI_EXT_ID_TIME) != 0) + has_time_extension = true; + if (sbi_probe_extension(SBI_EXT_ID_IPI) != 0) + has_ipi_extension = true; + if (sbi_probe_extension(SBI_EXT_ID_RFNC) != 0) + has_rfnc_extension = true; + if (sbi_probe_extension(SBI_EXT_ID_SRST) != 0) + has_srst_extension = true; +} diff --git a/libcpu/risc-v/common/serial/Kconfig b/libcpu/risc-v/common/serial/Kconfig new file mode 100644 index 0000000000..1ef9725983 --- /dev/null +++ b/libcpu/risc-v/common/serial/Kconfig @@ -0,0 +1,5 @@ +config RT_SERIAL_EARLY_SBI + bool "Early SBI" + depends on RT_USING_SERIAL + depends on !ARCH_RISCV_M_MODE + default y diff --git a/libcpu/risc-v/common/serial/SConscript b/libcpu/risc-v/common/serial/SConscript new file mode 100644 index 0000000000..675b16a71f --- /dev/null +++ b/libcpu/risc-v/common/serial/SConscript @@ -0,0 +1,17 @@ +from building import * + +group = [] + +if not GetDepend(['RT_USING_SERIAL']): + Return('group') + +src = [] +cwd = GetCurrentDir() +CPPPATH = [cwd] + +if GetDepend(['RT_SERIAL_EARLY_SBI']): + src += ['serial-early-sbi.c'] + +group = DefineGroup('DeviceDrivers', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/libcpu/risc-v/common/serial/serial-early-sbi.c b/libcpu/risc-v/common/serial/serial-early-sbi.c new file mode 100644 index 0000000000..afbe8d39ab --- /dev/null +++ b/libcpu/risc-v/common/serial/serial-early-sbi.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-01-30 GuEe-GUI first version + */ + +#include <rtdevice.h> + +#include <sbi.h> + +static void sbi_early_console_putchar(void *data, char c) +{ + sbi_console_putchar(c); +} + +static rt_err_t sbi_early_setup(struct rt_fdt_earlycon *con, const char *options) +{ + con->console_putc = sbi_early_console_putchar; + + return RT_EOK; +} +RT_FDT_EARLYCON_EXPORT(sbi, "sbi", "riscv,sbi-console", sbi_early_setup); diff --git a/libcpu/risc-v/common/setup.c b/libcpu/risc-v/common/setup.c new file mode 100644 index 0000000000..140a8ed12d --- /dev/null +++ b/libcpu/risc-v/common/setup.c @@ -0,0 +1,860 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2025-04-20 GuEe-GUI the first version + */ + +#include <rtthread.h> +#include <rtdevice.h> + +#if __riscv_xlen == 128 +#define DBG_TAG "cpu.rv128" +#elif __riscv_xlen == 64 +#define DBG_TAG "cpu.setup" +#elif __riscv_xlen == 32 +#define DBG_TAG "cpu.rv32" +#else +#error "Unexpected __riscv_xlen" +#endif +#define DBG_LVL DBG_INFO +#include <rtdbg.h> + +#include <cpu.h> +#include <sbi.h> +#include <board.h> +#ifdef RT_USING_CACHE +#include <cache.h> +#endif +#include <setup.h> +#include <isa_ext.h> +#include <cpuport.h> +#ifdef RT_USING_SMP +#include <smp_call.h> +#endif +#include <interrupt.h> +#include <riscv_timer.h> +#ifdef ARCH_MM_MMU +#include <mmu.h> +#include <stdlib.h> +#include <ioremap.h> +#include <mm_memblock.h> +#include <dt-bindings/size.h> +#endif +#ifdef RT_USING_LWP +#include <lwp.h> +#include <lwp_arch.h> +#endif + +#ifdef ARCH_MM_MMU +#ifndef IOREMAP_SIZE +#define IOREMAP_SIZE 0x20000000 +#endif +#endif + +#ifdef RT_USING_BUILTIN_FDT +extern const rt_uint8_t rt_hw_builtin_fdt[]; +#endif +extern rt_ubase_t _start, _end; +#ifdef ARCH_MM_MMU +extern rt_ubase_t MMUTable[]; +#if __riscv_xlen == 32 && !defined(RT_USING_SMART) +extern char __text_start; +#endif +#endif + +static void *fdt_ptr = RT_NULL; +static rt_size_t fdt_size = 0; + +#ifdef RT_USING_SMP +static rt_uint32_t cpu_boot_delay_us = 0; + +#ifndef ARCH_RISCV_M_MODE +extern struct cpu_ops_t cpu_sbi_ops; +#endif +extern struct cpu_ops_t cpu_spinwait_ops; + +static struct cpu_ops_t *cpu_ops = &cpu_spinwait_ops; +static struct rt_ofw_node *cpu_np[RT_CPUS_NR] = { }; +#endif /* RT_USING_SMP */ + +#ifdef ARCH_MM_MMU +static const struct { rt_uint8_t size; const char name[11]; } mmu_type[] = +{ + { 0, "riscv,none" }, +#if __riscv_xlen == 32 + { 32, "riscv,sv32" }, +#else + { 39, "riscv,sv39" }, + { 48, "riscv,sv48" }, + { 57, "riscv,sv57" }, + { 64, "riscv,sv64" }, +#endif +}; +#endif /* ARCH_MM_MMU */ + +rt_weak void rt_hw_fdt_vendor_install_early(void *fdt) +{ + RT_UNUSED(fdt); +} + +void rt_hw_fdt_install_early(void *fdt) +{ + int registered_cpus = 1, cpus, cpu; + +#ifdef RT_USING_BUILTIN_FDT + fdt = (void *)rt_hw_builtin_fdt; +#endif + + if (!fdt || fdt_check_header(fdt)) + { + fdt_ptr = RT_NULL; + fdt_size = 0; + return; + } + + fdt_ptr = fdt; + fdt_size = fdt_totalsize(fdt); + + rt_hw_fdt_vendor_install_early(fdt); + +#ifdef ARCH_MM_MMU + rt_hw_arch_vaddr_width = ARCH_ADDRESS_WIDTH_BITS; +#endif + + cpus = fdt_path_offset(fdt, "/cpus"); + if (cpus < 0) + { + return; + } + + fdt_for_each_subnode(cpu, fdt, cpus) + { + uint32_t hartid; +#ifdef ARCH_MM_MMU + int size; + const char *name; +#endif + + /* Skip status=disabled (and other non-okay) CPU nodes. */ + if (!rt_fdt_device_is_available(fdt, cpu)) + { + continue; + } + +#ifdef ARCH_MM_MMU + name = fdt_getprop(fdt, cpu, "mmu-type", RT_NULL); +#endif + + if (fdt_getprop_u32(fdt, cpu, "reg", &hartid, RT_NULL)) + { + continue; + } + + if (hartid != boot_cpu_hartid && riscv_hartid_to_cpu_id(hartid) < 0) + { + if (registered_cpus >= RT_CPUS_NR) + { + continue; + } + ++registered_cpus; + } + + riscv_hartid_register(hartid); + + if (hartid == boot_cpu_hartid) + { + uint32_t cbom_size; + + if (!fdt_getprop_u32(fdt, cpu, "riscv,cbom-block-size", &cbom_size, RT_NULL) && + cbom_size > 0) + { + riscv_isa_ext_set(ZICBOM); +#ifdef RT_USING_CACHE + riscv_cache_set_cbom_block_size(cbom_size); +#endif + } + } + +#ifdef ARCH_MM_MMU + size = name ? rt_hw_arch_vaddr_width : 0; + + for (int i = 0; name && i < RT_ARRAY_SIZE(mmu_type); ++i) + { + rt_typeof(mmu_type[0]) *type = &mmu_type[i]; + + if (!rt_strcmp(name, type->name)) + { + size = type->size; + break; + } + } + + if (size < rt_hw_arch_vaddr_width) + { + rt_hw_arch_vaddr_width = size; + } +#endif /* ARCH_MM_MMU */ + } +} + +rt_weak void rt_hw_idle_wfi(void) +{ + rt_hw_wfi(); +} + +static struct { const char *name; rt_uint8_t size; } isa_ext[] = +{ +#define DECLARE_ISA(isa, name) [RISCV_ISA_EXT_##isa] = { name, sizeof(name) - 1 }, + DECLARE_ISA(I, "i") + DECLARE_ISA(M, "m") + DECLARE_ISA(A, "a") + DECLARE_ISA(F, "f") + DECLARE_ISA(D, "d") + DECLARE_ISA(Q, "q") + DECLARE_ISA(C, "c") + DECLARE_ISA(V, "v") + DECLARE_ISA(H, "h") + DECLARE_ISA(SSCOFPMF, "sscofpmf") + DECLARE_ISA(SSTC, "sstc") + DECLARE_ISA(SVINVAL, "svinval") + DECLARE_ISA(SVPBMT, "svpbmt") + DECLARE_ISA(ZBB, "zbb") + DECLARE_ISA(ZICBOM, "zicbom") + DECLARE_ISA(ZIHINTPAUSE, "zihintpause") + DECLARE_ISA(SVNAPOT, "svnapot") + DECLARE_ISA(ZICBOZ, "zicboz") + DECLARE_ISA(SMAIA, "smaia") + DECLARE_ISA(SSAIA, "ssaia") + DECLARE_ISA(ZBA, "zba") + DECLARE_ISA(ZBS, "zbs") + DECLARE_ISA(ZICNTR, "zicntr") + DECLARE_ISA(ZICSR, "zicsr") + DECLARE_ISA(ZIFENCEI, "zifencei") + DECLARE_ISA(ZIHPM, "zihpm") + DECLARE_ISA(SMSTATEEN, "smstateen") + DECLARE_ISA(ZICOND, "zicond") + DECLARE_ISA(ZBC, "zbc") + DECLARE_ISA(ZBKB, "zbkb") + DECLARE_ISA(ZBKC, "zbkc") + DECLARE_ISA(ZBKX, "zbkx") + DECLARE_ISA(ZKND, "zknd") + DECLARE_ISA(ZKNE, "zkne") + DECLARE_ISA(ZKNH, "zknh") + DECLARE_ISA(ZKR, "zkr") + DECLARE_ISA(ZKSED, "zksed") + DECLARE_ISA(ZKSH, "zksh") + DECLARE_ISA(ZKT, "zkt") + DECLARE_ISA(ZVBB, "zvbb") + DECLARE_ISA(ZVBC, "zvbc") + DECLARE_ISA(ZVKB, "zvkb") + DECLARE_ISA(ZVKG, "zvkg") + DECLARE_ISA(ZVKNED, "zvkned") + DECLARE_ISA(ZVKNHA, "zvknha") + DECLARE_ISA(ZVKNHB, "zvknhb") + DECLARE_ISA(ZVKSED, "zvksed") + DECLARE_ISA(ZVKSH, "zvksh") + DECLARE_ISA(ZVKT, "zvkt") + DECLARE_ISA(ZFH, "zfh") + DECLARE_ISA(ZFHMIN, "zfhmin") + DECLARE_ISA(ZIHINTNTL, "zihintntl") + DECLARE_ISA(ZVFH, "zvfh") + DECLARE_ISA(ZVFHMIN, "zvfhmin") + DECLARE_ISA(ZFA, "zfa") + DECLARE_ISA(ZTSO, "ztso") + DECLARE_ISA(ZACAS, "zacas") + DECLARE_ISA(XANDESPMU, "xandespmu") +#undef DECLARE_ISA +}; + +rt_inline void cpu_info_init(void) +{ + int i = 0; + rt_uint32_t value; + const char *string; + struct rt_ofw_prop *prop; + struct rt_ofw_node *np, *boot_cpu_np = RT_NULL; + + if ((np = rt_ofw_find_node_by_path("/cpus")) && + !rt_ofw_prop_read_u32(np, "timebase-frequency", &value)) + { + riscv_timer_set_frequency(value); + + #ifdef RT_USING_HWTIMER + if (!rt_device_hwtimer_us_delay) + { + rt_device_hwtimer_us_delay = &riscv_timer_us_delay; + } + #endif /* RT_USING_HWTIMER */ + } + else + { + LOG_E("No '%s' in DTS", "timebase-frequency"); + } + +#ifdef RT_USING_SMP + rt_ofw_prop_read_u32(np, "cpu-boot-delay-us", &cpu_boot_delay_us); +#endif + + rt_ofw_node_put(np); + +#if defined(RT_USING_SMP) && !defined(ARCH_RISCV_M_MODE) + if (sbi_probe_extension(SBI_EXT_ID_HSM)) + { + rt_bool_t use_hsm = RT_TRUE; + + for (int cpu_id = 0; cpu_id < RT_CPUS_NR; ++cpu_id) + { + rt_ubase_t hartid; + int status; + + if (!riscv_cpu_id_available(cpu_id)) + { + continue; + } + + hartid = riscv_cpu_id_to_hartid(cpu_id); + if (hartid == boot_cpu_hartid) + { + continue; + } + + status = sbi_hsm_hart_status(hartid); + if (status == SBI_HSM_STATUS_STARTED) + { + use_hsm = RT_FALSE; + break; + } + } + + if (use_hsm) + { + LOG_I("SBI HSM extension detected"); + cpu_ops = &cpu_sbi_ops; + } + else + { + LOG_I("SBI HSM extension detected, use spinwait for started harts"); + } + } +#endif /* RT_USING_SMP && !ARCH_RISCV_M_MODE */ + + rt_ofw_foreach_cpu_node(np) + { + rt_ubase_t hartid = (rt_ubase_t)rt_ofw_get_cpu_hwid(np, 0); + + if (hartid == boot_cpu_hartid) + { + boot_cpu_np = np; + rt_ofw_node_get(boot_cpu_np); + } + + #ifdef RT_USING_SMP + int cpu_id = riscv_hartid_to_cpu_id(hartid); + + if (cpu_id >= 0 && cpu_id < RT_CPUS_NR) + { + cpu_np[cpu_id] = np; + + if (cpu_ops->cpu_init) + { + cpu_ops->cpu_init(cpu_id, np); + } + } + #endif /* RT_USING_SMP */ + } + + if (!(np = boot_cpu_np)) + { + LOG_E("No '%s' in DTS", "Boot CPU"); + return; + } + + rt_ofw_foreach_prop_string(np, "riscv,isa-extensions", prop, string) + { + for (i = 0; i < RT_ARRAY_SIZE(isa_ext); ++i) + { + rt_typeof(isa_ext[0]) *ext = &isa_ext[i]; + + if (ext->name && !rt_strcmp(ext->name, string)) + { + riscv_isa_ext_set_raw(i); + break; + } + } + } + + if (!rt_ofw_prop_read_bool(np, "riscv,isa-base")) + { + if (rt_ofw_prop_read_string(np, "riscv,isa", &string)) + { + LOG_E("%s ISA no found", rt_ofw_node_full_name(np)); + goto _exit; + } + + if ((!rt_strncmp(string, "rv32", 4) && __riscv_xlen != 32) || + (!rt_strncmp(string, "rv64", 4) && __riscv_xlen != 64)) + { + LOG_E("%s ISA %.4s is not supported", rt_ofw_node_full_name(np), string); + goto _exit; + } + + string += 4; + + /* ISA: 'i', 'm', 'a', 'f', 'd', 'q', 'c', 'v', 'h'... */ + while (*string && *string != '_') + { + for (i = 0; i < RISCV_ISA_EXT_BASE; ++i) + { + rt_typeof(isa_ext[0]) *ext = &isa_ext[i]; + + if (ext->name && ext->name[0] == *string) + { + riscv_isa_ext_set_raw(i); + break; + } + } + + ++string; + } + + /* Skip '_' */ + while (*string && *(++string)) + { + int size = 0; + + for (i = RISCV_ISA_EXT_BASE; i < RT_ARRAY_SIZE(isa_ext); ++i) + { + rt_typeof(isa_ext[0]) *ext = &isa_ext[i]; + + if (ext->name && !rt_strncmp(ext->name, string, ext->size)) + { + size = ext->size; + riscv_isa_ext_set_raw(i); + break; + } + } + + if (!size) + { + const char *skip = rt_strstr(string, "_"); + + size = skip ? skip - string : rt_strlen(string); + + LOG_E("Unknow ISA %.*s", size, string); + } + + string += size; + } + } + +_exit: + rt_ofw_node_put(np); +} + +#if defined(ARCH_MM_MMU) && defined(RT_USING_DMA) +static rt_size_t string_to_size(const char *string, const char *who) +{ + char unit; + rt_size_t size; + const char *cp = string; + + size = atoi(cp); + + while (*cp >= '0' && *cp <= '9') + { + ++cp; + } + + unit = *cp; + if (unit >= 'a' && unit <= 'z') + { + unit -= 'a' - 'A'; + } + + if (unit == 'M') + { + size *= SIZE_MB; + } + else if (unit == 'K') + { + size *= SIZE_KB; + } + else if (unit == 'G') + { + size *= SIZE_GB; + } + else + { + LOG_W("Unknown unit of '%c' in `%s`", unit, who); + } + + return size; +} +#endif /* ARCH_MM_MMU && RT_USING_DMA */ + +void rt_hw_common_setup(void) +{ +#ifdef RT_USING_BUILTIN_FDT + fdt_ptr = (void *)rt_hw_builtin_fdt; + fdt_size = fdt_totalsize(fdt_ptr); +#endif + + if (!fdt_ptr) + { + LOG_E("No valid FDT installed"); + RT_ASSERT(0); + return; + } + +#ifdef ARCH_MM_MMU + rt_uint64_t initrd_ranges[3]; + rt_size_t kernel_start, kernel_end; + rt_size_t heap_start, heap_end; + rt_size_t init_page_start, init_page_end; + rt_size_t fdt_start, fdt_end; + rt_region_t init_page_region = { 0 }; + rt_region_t platform_mem_region = { 0 }; + static struct mem_desc platform_mem_desc; + const rt_ubase_t pv_off = PV_OFFSET; + +#ifdef RT_USING_SMART +#if KERNEL_VADDR_START > USER_VADDR_START + rt_hw_mmu_map_init(&rt_kernel_space, (void *)(0UL - IOREMAP_SIZE), IOREMAP_SIZE, MMUTable, pv_off); +#else + rt_hw_mmu_map_init(&rt_kernel_space, (void *)(USER_VADDR_START - IOREMAP_SIZE), IOREMAP_SIZE, MMUTable, pv_off); +#endif +#else +#if __riscv_xlen == 32 + rt_hw_mmu_map_init(&rt_kernel_space, (void *)((rt_ubase_t)&__text_start - IOREMAP_SIZE), IOREMAP_SIZE, MMUTable, 0); +#else + /* Keep dynamic mappings within the lower canonical address range. */ + rt_hw_mmu_map_init(&rt_kernel_space, (void *)((1UL << (rt_hw_arch_vaddr_width - 1)) - IOREMAP_SIZE), IOREMAP_SIZE, MMUTable, 0); +#endif +#endif /* RT_USING_SMART */ + + kernel_start = RT_ALIGN_DOWN((rt_size_t)&_start + pv_off - 64, ARCH_PAGE_SIZE); + kernel_end = RT_ALIGN((rt_size_t)&_end + pv_off, ARCH_PAGE_SIZE); + heap_start = kernel_end; + heap_end = RT_ALIGN(heap_start + ARCH_HEAP_SIZE, ARCH_PAGE_SIZE); + init_page_start = heap_end; + init_page_end = RT_ALIGN(init_page_start + ARCH_INIT_PAGE_SIZE, ARCH_PAGE_SIZE); + fdt_start = init_page_end; + fdt_end = RT_ALIGN(fdt_start + fdt_size, ARCH_PAGE_SIZE); + + platform_mem_region.start = kernel_start; +#ifndef RT_USING_BUILTIN_FDT + platform_mem_region.end = fdt_end; +#else + platform_mem_region.end = init_page_end; + (void)fdt_start; + (void)fdt_end; +#endif + + rt_memblock_reserve_memory("kernel", kernel_start, kernel_end, MEMBLOCK_NONE); + rt_memblock_reserve_memory("memheap", heap_start, heap_end, MEMBLOCK_NONE); + rt_memblock_reserve_memory("init-page", init_page_start, init_page_end, MEMBLOCK_NONE); +#ifndef RT_USING_BUILTIN_FDT + rt_memblock_reserve_memory("fdt", fdt_start, fdt_end, MEMBLOCK_NONE); + + /* To virtual address */ + fdt_ptr = (void *)(fdt_ptr - pv_off); +#ifdef KERNEL_VADDR_START + if ((rt_ubase_t)fdt_ptr + fdt_size - KERNEL_VADDR_START > ARCH_EARLY_MAP_SIZE) + { + fdt_ptr = rt_ioremap_early(fdt_ptr + pv_off, fdt_size); + + RT_ASSERT(fdt_ptr != RT_NULL); + } +#endif /* KERNEL_VADDR_START */ + rt_memmove((void *)(fdt_start - pv_off), fdt_ptr, fdt_size); + fdt_ptr = (void *)fdt_start - pv_off; +#endif /* RT_USING_BUILTIN_FDT */ + + rt_system_heap_init((void *)(heap_start - pv_off), (void *)(heap_end - pv_off)); + + init_page_region.start = init_page_start - pv_off; + init_page_region.end = init_page_end - pv_off; + rt_page_init(init_page_region); + + /* Create MMU mapping of kernel memory */ + platform_mem_region.start = RT_ALIGN_DOWN(platform_mem_region.start, ARCH_PAGE_SIZE); + platform_mem_region.end = RT_ALIGN(platform_mem_region.end, ARCH_PAGE_SIZE); + + platform_mem_desc.paddr_start = platform_mem_region.start; + platform_mem_desc.vaddr_start = platform_mem_region.start - pv_off; + platform_mem_desc.vaddr_end = platform_mem_region.end - pv_off - 1; + platform_mem_desc.attr = NORMAL_MEM; + + rt_hw_mmu_setup(&rt_kernel_space, &platform_mem_desc, 1); +#else +#ifdef RT_USING_HEAP + rt_system_heap_init(&_end, (void *)((rt_ubase_t)&_end + ARCH_HEAP_SIZE)); +#endif /* RT_USING_HEAP */ +#endif /* ARCH_MM_MMU */ + + if (rt_fdt_prefetch(fdt_ptr)) + { + LOG_E("Failed to prefetch FDT"); + RT_ASSERT(0); + return; + } + +#ifdef RT_USING_HWCACHE + rt_hwcache_init(); + + rt_hwcache_icache_enable(); + rt_hwcache_dcache_enable(); +#endif /* RT_USING_HWCACHE */ + + rt_fdt_scan_chosen_stdout(); + +#ifdef ARCH_MM_MMU + rt_fdt_scan_initrd(initrd_ranges); + + rt_fdt_scan_memory(); + +#ifdef RT_USING_DMA + do { + const char *bootargs; + rt_ubase_t dma_pool_base; + rt_size_t cma_size = 0, coherent_pool_size = 0; + + if (!rt_fdt_bootargs_select("cma=", 0, &bootargs)) + { + cma_size = string_to_size(bootargs, "cma"); + } + + if (!rt_fdt_bootargs_select("coherent_pool=", 0, &bootargs)) + { + coherent_pool_size = string_to_size(bootargs, "coherent-pool"); + } + + if (cma_size <= coherent_pool_size) + { + if (cma_size || coherent_pool_size) + { + LOG_W("DMA pool %s=%u > %s=%u", + "CMA", cma_size, "coherent-pool", coherent_pool_size); + } + + cma_size = 8 * SIZE_MB; + coherent_pool_size = 2 * SIZE_MB; + } + + dma_pool_base = platform_mem_region.end; + rt_memblock_reserve_memory("dma-pool", + dma_pool_base, dma_pool_base + cma_size + coherent_pool_size, MEMBLOCK_NONE); + + if (rt_dma_pool_extract(cma_size, coherent_pool_size)) + { + LOG_E("Alloc DMA pool %s=%u, %s=%u fail", + "CMA", cma_size, "coherent-pool", coherent_pool_size); + } + } while (0); +#endif /* RT_USING_DMA */ + + rt_memblock_setup_memory_environment(); +#endif /* ARCH_MM_MMU */ + + rt_fdt_earlycon_kick(FDT_EARLYCON_KICK_UPDATE); + + if (rt_fdt_unflatten()) + { + LOG_E("Failed to unflatten FDT"); + RT_ASSERT(0); + return; + } + +#if !defined(ARCH_MM_MMU) && defined(RT_USING_DMA) + { + struct rt_ofw_node *reserved_memory, *np; + + if ((reserved_memory = rt_ofw_find_node_by_path("/reserved-memory"))) + { + rt_uint64_t address, size, max_address = (rt_uint64_t)~(rt_ubase_t)0; + + rt_ofw_foreach_available_child_node(reserved_memory, np) + { + if (!rt_ofw_node_is_compatible(np, "rt-thread,dma-pool") && + !rt_ofw_node_is_compatible(np, "shared-dma-pool")) + { + continue; + } + + for (int index = 0; !rt_ofw_get_address(np, index, &address, &size); ++index) + { + rt_region_t region; + + if (!size || address > max_address || size > max_address - address) + { + LOG_W("DMA pool %s has invalid range", rt_ofw_node_full_name(np)); + continue; + } + + region.name = rt_ofw_node_full_name(np); + region.start = (rt_ubase_t)address; + region.end = region.start + (rt_size_t)size; + + if (!rt_dma_pool_install(®ion)) + { + LOG_E("Install DMA pool %s failed", region.name); + } + } + } + + rt_ofw_node_put(reserved_memory); + } + } +#endif /* !ARCH_MM_MMU && RT_USING_DMA */ + + cpu_info_init(); + +#ifdef RT_USING_PIC + rt_pic_init(); + rt_pic_irq_init(); +#endif + + riscv_timer_init(); + +#ifdef RT_USING_COMPONENTS_INIT + rt_components_board_init(); +#endif + +#if defined(RT_USING_CONSOLE) && defined(RT_USING_DEVICE) + rt_ofw_console_setup(); +#endif + +#if defined(RT_USING_HOOK) || defined(RT_USING_IDLE_HOOK) + rt_thread_idle_sethook(rt_hw_idle_wfi); +#endif + +#ifdef RT_USING_SMP + rt_smp_call_init(); +#ifdef RT_USING_PIC + /* Install the IPI handle */ + rt_hw_ipi_handler_install(RT_SCHEDULE_IPI, rt_scheduler_ipi_handler); + rt_hw_ipi_handler_install(RT_STOP_IPI, rt_scheduler_ipi_handler); + rt_hw_ipi_handler_install(RT_SMP_CALL_IPI, rt_smp_call_ipi_handler); + rt_hw_interrupt_umask(RT_SCHEDULE_IPI); + rt_hw_interrupt_umask(RT_STOP_IPI); + rt_hw_interrupt_umask(RT_SMP_CALL_IPI); +#endif /* RT_USING_PIC */ +#endif /* RT_USING_SMP */ +} + +#ifndef ARCH_RISCV_M_MODE +static rt_err_t sbi_srst_reboot_mode(struct rt_device *dev, char *cmd) +{ + if (!rt_strcmp(cmd, "software") || !rt_strcmp(cmd, "warm")) + { + sbi_srst_reboot(SBI_SRST_RESET_TYPE_WARM_REBOOT); + } + else + { + sbi_srst_reboot(SBI_SRST_RESET_TYPE_COLD_REBOOT); + } + + return RT_EOK; +} + +static int system_power_init(void) +{ + rt_dm_machine_shutdown = rt_dm_machine_shutdown ? : sbi_shutdown; + + if (has_srst_extension) + { + static struct rt_device pm; + + rt_dm_machine_shutdown = sbi_srst_power_off; + + rt_dm_dev_set_name(&pm, "SBI-SRST"); + rt_dm_reboot_mode_register(&pm, sbi_srst_reboot_mode); + } + + return 0; +} +INIT_PREV_EXPORT(system_power_init); +#endif /* ARCH_RISCV_M_MODE */ + +#ifdef RT_USING_SMP +rt_weak void rt_hw_secondary_cpu_up(void) +{ + int cpu_id = rt_hw_cpu_id(); +#ifdef ARCH_MM_MMU + rt_ubase_t entry = (rt_ubase_t)rt_kmem_v2p(_secondary_cpu_entry); +#else + rt_ubase_t entry = (rt_ubase_t)_secondary_cpu_entry; +#endif + + if (!entry) + { + LOG_E("Failed to translate '_secondary_cpu_entry' to physical address"); + RT_ASSERT(0); + } + + /* Maybe we are no in the first cpu */ + for (int i = 0; i < RT_ARRAY_SIZE(cpu_np); ++i) + { + int err; + + if (!cpu_np[i] || i == cpu_id) + { + continue; + } + + if (cpu_boot_delay_us) + { + rt_hw_us_delay(cpu_boot_delay_us); + } + + if ((err = cpu_ops->cpu_boot(i, entry))) + { + LOG_W("Call cpu %d on %s", i, "failed"); + } + } +} + +rt_weak void rt_hw_secondary_cpu_bsp_start(void) +{ + int cpu_id; + + cpu_id = rt_hw_cpu_id(); + rt_hw_spin_lock(&_cpus_lock); + +#ifdef ARCH_MM_MMU + rt_hw_mmu_tbl_set((unsigned long)rt_kernel_space.page_table); +#endif + +#ifdef RT_USING_PIC + rt_pic_irq_init(); +#endif + + riscv_timer_init(); + + rt_dm_secondary_cpu_init(); + +#ifdef RT_USING_PIC + rt_hw_interrupt_umask(RT_SCHEDULE_IPI); + rt_hw_interrupt_umask(RT_STOP_IPI); + rt_hw_interrupt_umask(RT_SMP_CALL_IPI); +#endif + + LOG_I("Call cpu %d on %s", cpu_id, "success"); + + rt_system_scheduler_start(); +} + +rt_weak void rt_hw_secondary_cpu_idle_exec(void) +{ + rt_hw_idle_wfi(); +} +#endif /* RT_USING_SMP */ + +void rt_hw_console_output(const char *str) +{ + rt_fdt_earlycon_output(str); +} diff --git a/libcpu/risc-v/common/stack.c b/libcpu/risc-v/common/stack.c new file mode 100644 index 0000000000..d7d428b745 --- /dev/null +++ b/libcpu/risc-v/common/stack.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2018-10-28 Bernard The unify RISC-V porting code. + * 2021-02-11 lizhirui add gp support + * 2021-11-19 JasonHu add fpu support + * 2025-04-20 GuEe-GUI Port for CPU ID + */ + +#include <rtthread.h> + +#include <cpu.h> +#include <csr.h> +#include <stack.h> +#include <cpuport.h> +#ifdef ARCH_MM_MMU +#include <mmu.h> +#endif + +/** + * This function will initialize thread stack, we assuming + * when scheduler restore this new thread, context will restore + * an entry to user first application + * + * s0-s11, ra, sstatus, a0 + * @param tentry the entry of thread + * @param parameter the parameter of entry + * @param stack_addr the beginning stack address + * @param texit the function will be called when thread exit + * + * @return stack address + */ +rt_uint8_t *rt_hw_stack_init(void *tentry, + void *parameter, + rt_uint8_t *stack_addr, + void *texit) +{ + rt_ubase_t *sp; + rt_hw_switch_frame_t frame; + + /* We use a strict alignment requirement for Q extension */ + sp = (rt_ubase_t *)RT_ALIGN_DOWN((rt_ubase_t)stack_addr, 16); + + *(--sp) = (rt_ubase_t)tentry; + *(--sp) = (rt_ubase_t)parameter; + *(--sp) = (rt_ubase_t)texit; + /* Alignment */ + --sp; + + frame = (rt_hw_switch_frame_t)((rt_ubase_t)sp - sizeof(*frame)); + rt_memset(frame, 0, sizeof(*frame)); + + frame->regs[RT_HW_SWITCH_CONTEXT_RA] = (rt_ubase_t)_thread_start; + frame->regs[RT_HW_SWITCH_CONTEXT_XSTATUS] = K_XSTATUS_DEFAULT; + /* Bare SMP keeps the current hart's tp across context switches. */ + + return (void *)frame; +} diff --git a/libcpu/risc-v/common/stack_gcc.S b/libcpu/risc-v/common/stack_gcc.S new file mode 100644 index 0000000000..c6eaaba754 --- /dev/null +++ b/libcpu/risc-v/common/stack_gcc.S @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2006-2022, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-10-19 RT-Thread the first version + */ + +#include "asm-generic.h" + + .section ".text","ax" + +START_POINT(_thread_start) + REG_L ra, SZREG(sp) /* thread exit */ + addi sp, sp, 2 * SZREG + REG_L a0, (sp) /* parameter */ + REG_L t0, SZREG(sp) /* tentry */ + addi sp, sp, 2 * SZREG + mv s1, ra + jalr t0 + jalr s1 + j . /* never here */ +START_POINT_END(_thread_start) diff --git a/libcpu/risc-v/common/startup.c b/libcpu/risc-v/common/startup.c new file mode 100644 index 0000000000..65b3038146 --- /dev/null +++ b/libcpu/risc-v/common/startup.c @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2006-2026, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include <rtthread.h> + +/* + * Generic startup preserves the firmware-provided FDT before entering the + * kernel. The generic DM setup provides the strong implementation; a BSP with + * custom setup may override this hook when it still uses the common startup. + */ +rt_weak void rt_hw_fdt_install_early(void *fdt) +{ + RT_UNUSED(fdt); +} + +/* + * This runs after gp and a temporary stack are ready, before normal C + * runtime state is available. + */ +rt_weak void riscv_early_setup(void) +{ +} + +rt_weak void riscv_bss_init(void) +{ + extern rt_uint8_t __bss_start[]; + extern rt_uint8_t __bss_end[]; + volatile rt_uint8_t *ptr = __bss_start; + rt_ubase_t *word, *word_end, word_start; + + word_start = RT_ALIGN((rt_ubase_t)__bss_start, sizeof(rt_ubase_t)); + + while (ptr < __bss_end && (rt_ubase_t)ptr < word_start) + { + *ptr++ = 0; + } + + word = (rt_ubase_t *)ptr; + word_end = (rt_ubase_t *)RT_ALIGN_DOWN((rt_ubase_t)__bss_end, sizeof(rt_ubase_t)); + + while (word < word_end) + { + *word++ = 0; + } + + ptr = (rt_uint8_t *)word; + while (ptr < __bss_end) + { + *ptr++ = 0; + } +} diff --git a/libcpu/risc-v/common/startup_gcc.S b/libcpu/risc-v/common/startup_gcc.S new file mode 100644 index 0000000000..d973e2b96d --- /dev/null +++ b/libcpu/risc-v/common/startup_gcc.S @@ -0,0 +1,339 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2018-10-01 Bernard The first version + * 2018-12-27 Jesven Add SMP support + * 2020-6-12 Xim Port to QEMU and remove SMP support + * 2024-06-30 Shell Support of kernel remapping + * 2025-04-20 GuEe-GUI Support DM and merge 64I and 32I/E mode with SMP + */ + +#include <asm.h> +#include <csr.h> +#include <page_def.h> +#ifdef ARCH_MM_MMU +#include <mmu.h> +#endif + +#ifndef ARCH_SECONDARY_CPU_STACK_SIZE +#define ARCH_SECONDARY_CPU_STACK_SIZE ARCH_PAGE_SIZE +#endif + +#define RISCV_IMAGE_FLAG_BE_SHIFT 0 +#define RISCV_IMAGE_FLAG_BE_MASK 0x1 + +#define RISCV_IMAGE_FLAG_LE 0 +#define RISCV_IMAGE_FLAG_BE 1 + +#define RISCV_HEADER_VERSION_MAJOR 0 +#define RISCV_HEADER_VERSION_MINOR 2 + +#ifdef ARCH_CPU_BIG_ENDIAN +#error conversion of header fields to LE not yet implemented +#else +#define __HEAD_FLAG_BE RISCV_IMAGE_FLAG_LE +#endif + +#define __HEAD_FLAG(field) (__HEAD_FLAG_##field << RISCV_IMAGE_FLAG_##field##_SHIFT) + +#define __HEAD_FLAGS (__HEAD_FLAG(BE)) + +#define __HEAD_VERSION (RISCV_HEADER_VERSION_MAJOR << 16 | RISCV_HEADER_VERSION_MINOR) + + .section ".start","ax" + +#ifdef RT_USING_OFW +/* + * We follow the kernel's boot in RISC-V: + * https://www.kernel.org/doc/html/latest/arch/riscv/boot-image-header.html + */ +_head: + j _start /* Executable code */ + .word 0 /* Executable code */ + .balign 8 + .dword _text_offset /* Image load offset from start of RAM */ + .dword _end - _head /* Effective Image size, little endian (_end defined in link.lds) */ + .dword __HEAD_FLAGS /* kernel flags, little endian */ + .word __HEAD_VERSION /* Version of this header */ + .word 0 /* Reserved */ + .dword 0 /* Reserved */ + .ascii "RISCV\0\0\0" /* Magic number, little endian */ + .balign 4 + .ascii "RSC\x05" /* Magic number 2, little endian */ + .word 0 /* Reserved for PE COFF offset */ +#endif /* RT_USING_OFW */ + +#ifndef RT_USING_OFW +_head: +#endif + .global _start +_start: +/* + * Boot CPU general-purpose register settings: + * https://www.kernel.org/doc/html/latest/arch/riscv/boot.html#pre-kernel-requirements-and-constraints + * a0 = hartid of the current core. + * a1 = physical address of device tree blob (dtb) in system RAM. + */ +#ifdef ARCH_RISCV_M_MODE + csrr s0, CSR_MHARTID +#else + mv s0, a0 +#endif + mv s1, a1 + call init_cpu_common + +#ifdef RT_USING_SMP + /* Pick one hart to run the main boot sequence */ + la t0, .hart_lottery + li t1, 1 + amoadd.w t0, t1, (t0) + bnez t0, secondary_cpu_spin +#endif + + /* Just for temporary use, sp will reset later */ + la sp, .percpu_stack_top + + call riscv_early_setup + call riscv_bss_init + + mv a0, s0 + call riscv_boot_hart_init + +#ifdef RT_USING_OFW + /* Restore the devicetree address saved before calling C code. */ + mv a0, s1 + call rt_hw_fdt_install_early +#endif + +#ifndef ARCH_RISCV_M_MODE + call sbi_init +#endif + + /* Now we are in the end of boot cpu process */ + la s1, rtthread_startup + j init_kernel_early + +kernel_start: +#ifdef ARCH_RISCV_USING_IRQ_STACK + call riscv_irq_stack_init +#endif + + /* Set traps vectors (Mode = 0) */ + la t0, system_vectors + csrw CSR_TVEC, t0 + + /* Jump to the PE's system entry */ + mv ra, s1 + mv fp, zero + jalr x0, s1, 0 + +cpu_idle: + wfi + j cpu_idle + +#ifdef RT_USING_SMP +secondary_cpu_spin: + la t0, riscv_spinwait_table + +.wait_for_cpu_up: + lbu t1, (t0) + beqz t1, .wait_for_cpu_up + fence + j _secondary_cpu_start + + .global _secondary_cpu_entry +_secondary_cpu_entry: + /* a0 = hartid of the current core. */ + mv s0, a0 + + call init_cpu_common + +_secondary_cpu_start: + /* Secondary cpu start to startup */ + la s1, rt_hw_secondary_cpu_bsp_start + j start_kernel_early +#endif /* RT_USING_SMP */ + +init_cpu_common: + /* Mask all interrupts */ + csrw CSR_IE, zero + csrw CSR_IP, zero + + /* Load the global pointer */ + LOAD_GLOBAL_POINTER + + /* + * Disable FPU & VECTOR to detect illegal usage of + * floating point or vector in kernel space + */ + li t0, SR_FS_VS + csrc CSR_STATUS, t0 + + /* + * Set sup0 scratch register to 0, indicating to exception vector that + * we are presently executing in kernel. + */ + csrw CSR_SCRATCH, zero + + ret + +init_kernel_early: +#ifdef ARCH_MM_MMU + la a0, .early_page_array + call set_free_page + + /* Get linker address to a0 */ + la a0, .kernel_vaddr_start + REG_L a0, (a0) + /* Get runtime address to a1 */ + la a1, _head + /* Get early page for map to a2 */ + la a2, .early_tbl_page + + /* Try to setup */ + call rt_hw_mem_setup_early +#endif /* ARCH_MM_MMU */ + +start_kernel_early: +#ifdef RT_USING_SMP + /* Only reset sp in SMP mode */ + la t0, cpu_id_to_hartid_map + mv t1, zero + li t2, RT_CPUS_NR + j .cpu_id_confirm + +.cpu_next_id_confirm: + add t1, t1, 1 + /* Out of range? */ + beq t1, t2, cpu_idle + add t0, t0, RISCV_SZPTR + +.cpu_id_confirm: + REG_L a0, (t0) + bne s0, a0, .cpu_next_id_confirm + + /* Set stack top by CPU ID without requiring the M extension. */ + la t2, .percpu_stack_top + mv a0, t1 + beqz a0, .cpu_stack_ready + li t0, ARCH_SECONDARY_CPU_STACK_SIZE + +.cpu_stack_next: + add t2, t2, t0 + addi a0, a0, -1 + bnez a0, .cpu_stack_next + +.cpu_stack_ready: + mv sp, t2 + + /* Save CPU ID, if MMU is available, tp will not use */ + mv tp, t1 +#endif /* RT_USING_SMP */ + +#ifdef ARCH_MM_MMU + /* Support MMU? */ + la a0, rt_hw_arch_vaddr_width + REG_L a0, (a0) + beqz a0, .prepare_start_kernel + +#ifdef RT_USING_SMP + la a0, .percpu_hartid + mv a1, s0 + call percpu_hartid_init +#endif /* RT_USING_SMP */ + + la a0, .early_tbl_page + mv a1, s0 + call mmu_satp_load + mv s2, a0 + + /* Read the PV offset to a0 */ + call rt_kmem_pvoff + + /* Set the gp to virtual address */ + sub gp, gp, a0 + +#ifdef RT_USING_SMP + /* The sp is unity in per-CPU */ + la sp, .percpu_stack_top +#endif + /* Set the sp to virtual address */ + sub sp, sp, a0 + + /* Enable MMU */ + csrw satp, s2 + sfence.vma zero, zero + +.prepare_start_kernel: + /* Set kernel entry to the virtual address */ + sub s1, s1, a0 +#else + mv a0, zero +#endif /* ARCH_MM_MMU */ + + /* Set RA to kernel_start function, to return to the virtual address */ + la t0, kernel_start + sub t0, t0, a0 + mv ra, t0 + + ret + + .section ".data" + .balign RISCV_SZPTR +#ifdef RT_USING_SMP +.hart_lottery: + .word 0 +#endif + +.kernel_vaddr_start: + RISCV_PTR __text_start + +/* + * CPU stack builtin + */ + .section ".percpu" +.percpu_stack_bottom: +#if defined(RT_USING_SMP) && defined(ARCH_MM_MMU) +#if ARCH_SECONDARY_CPU_STACK_SIZE < ARCH_PAGE_SIZE +#error ARCH_SECONDARY_CPU_STACK_SIZE MUST >= ARCH_PAGE_SIZE +#endif + .space ARCH_SECONDARY_CPU_STACK_SIZE - 16 +.percpu_stack_top: +.percpu_hartid: + .global __percpu_hartid +__percpu_hartid: + .space 16 +#else + .space ARCH_SECONDARY_CPU_STACK_SIZE +.percpu_stack_top: +#endif /* RT_USING_SMP && ARCH_MM_MMU */ + +#ifdef ARCH_MM_MMU +/* + * Early page builtin, map 4G by 4MB/2MB superpage: + * Sv32 => 1 TBL -> 4 pages + * Sv39 => 1 TBL -> 5 pages + * Sv48 => 1 TBL -> 6 pages + * Sv57 => 1 TBL -> 7 pages + * Sv64 => 1 TBL -> 8 pages + * ioremap early -> 8 pages + * .percpu early -> 5 pages (Sv64) + */ + .section ".bss.noclean.early_page" + .balign ARCH_PAGE_SIZE +.early_tbl_page: + .space 1 * ARCH_PAGE_SIZE +#if defined(RT_USING_SMP) && RT_CPUS_NR > 1 + .space (RT_CPUS_NR - 1) * ARCH_PAGE_SIZE +#endif + +.early_page_array: + .space (8 + 8) * ARCH_PAGE_SIZE +#ifdef RT_USING_SMP + .space RT_CPUS_NR * 5 * ARCH_PAGE_SIZE +#endif +#endif /* ARCH_MM_MMU */ diff --git a/libcpu/risc-v/common/syscall.c b/libcpu/risc-v/common/syscall.c new file mode 100644 index 0000000000..c8df566c4c --- /dev/null +++ b/libcpu/risc-v/common/syscall.c @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2006-2021, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2021-02-03 lizhirui first version + * 2022-11-10 WangXiaoyao Add readable syscall tracing + */ + +#include <rthw.h> +#include <rtthread.h> + +#define DBG_TAG "syscall" +#define DBG_LVL DBG_WARNING +#include <rtdbg.h> + +#include <mmu.h> +#include <page.h> +#include <stack.h> +#include <lwp_user_mm.h> + +typedef rt_ubase_t (*syscallfunc_t)(rt_ubase_t, rt_ubase_t, rt_ubase_t, rt_ubase_t, rt_ubase_t, rt_ubase_t, rt_ubase_t); + +void syscall_handler(struct rt_hw_stack_frame *regs) +{ + const char *syscall_name; + int syscallid = regs->a7; + + if (syscallid == 0) + { + LOG_E("syscall id = 0!\n"); + + while (RT_TRUE) + { + rt_hw_cpu_relax(); + } + } + + syscallfunc_t syscallfunc = (syscallfunc_t)lwp_get_sys_api(syscallid); + + if (syscallfunc == RT_NULL) + { + LOG_E("unsupported syscall!\n"); + sys_exit_group(-1); + } + +#if DBG_LVL >= DBG_INFO + syscall_name = lwp_get_syscall_name(syscallid); +#else + RT_UNUSED(syscall_name); +#endif + + LOG_I("[0x%lx] %s(0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx)", rt_thread_self(), syscall_name, + regs->a0, regs->a1, regs->a2, regs->a3, regs->a4, regs->a5, regs->a6); + regs->a0 = syscallfunc(regs->a0, regs->a1, regs->a2, regs->a3, regs->a4, regs->a5, regs->a6); + regs->a7 = 0; + regs->epc += 4; // skip ecall instruction + LOG_I("[0x%lx] %s ret: 0x%lx", rt_thread_self(), syscall_name, regs->a0); +} diff --git a/libcpu/risc-v/common/trap.c b/libcpu/risc-v/common/trap.c new file mode 100644 index 0000000000..64074399fc --- /dev/null +++ b/libcpu/risc-v/common/trap.c @@ -0,0 +1,404 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-12-08 RT-Thread first version + * 2025-04-20 GuEe-GUI Port to the DM + */ + +#include <rthw.h> +#include <rtthread.h> + +#include <csr.h> +#include <stack.h> +#include <riscv_timer.h> +#ifdef RT_USING_DM +#include <drivers/pic.h> +#else +#include <board.h> +#include <interrupt.h> +#endif +#ifdef RT_USING_SMART +#include <lwp_arch.h> +#endif + +#define DBG_TAG "libcpu.trap" +#define DBG_LVL DBG_INFO +#include <rtdbg.h> + +#ifdef RT_USING_FINSH +extern long list_thread(void); +#endif + +#ifdef RT_USING_LWP +#include <mm_aspace.h> +#include <lwp.h> +#include <lwp_arch.h> + +#ifdef LWP_USING_CORE_DUMP +#include <lwp_core_dump.h> +#endif + +static void _check_fault(struct rt_hw_stack_frame *regs, uint32_t pc_adj, char *info) +{ + rt_thread_t th; + rt_bool_t is_user_fault = !(regs->xstatus & SR_PP); + + if (is_user_fault) + { + rt_kprintf("%s! pc = %p\n", info, regs->epc - pc_adj); + } + + /* User stack backtrace */ + th = rt_thread_self(); + if (th && th->lwp) + { + arch_backtrace_uthread(th); + } + + if (is_user_fault) + { + #ifdef LWP_USING_CORE_DUMP + lwp_core_dump(regs, pc_adj); + #endif + sys_exit_group(-1); + } +} + +rt_inline rt_bool_t user_fault_fixable(struct rt_hw_stack_frame *regs, rt_ubase_t xcause, rt_ubase_t tval) +{ + struct rt_lwp *lwp; + enum rt_mm_fault_op fault_op; + enum rt_mm_fault_type fault_type; + + switch (xcause) + { + case EXC_LOAD_PAGE_FAULT: + fault_op = MM_FAULT_OP_READ; + fault_type = MM_FAULT_TYPE_GENERIC_MMU; + break; + case EXC_LOAD_ACCESS: + fault_op = MM_FAULT_OP_READ; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + case EXC_LOAD_MISALIGNED: + fault_op = MM_FAULT_OP_READ; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + case EXC_STORE_PAGE_FAULT: + fault_op = MM_FAULT_OP_WRITE; + fault_type = MM_FAULT_TYPE_GENERIC_MMU; + break; + case EXC_STORE_ACCESS: + fault_op = MM_FAULT_OP_WRITE; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + case EXC_STORE_MISALIGNED: + fault_op = MM_FAULT_OP_WRITE; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + case EXC_INST_PAGE_FAULT: + fault_op = MM_FAULT_OP_EXECUTE; + fault_type = MM_FAULT_TYPE_GENERIC_MMU; + break; + case EXC_INST_ACCESS: + fault_op = MM_FAULT_OP_EXECUTE; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + case EXC_INST_MISALIGNED: + fault_op = MM_FAULT_OP_EXECUTE; + fault_type = MM_FAULT_TYPE_BUS_ERROR; + break; + default: + fault_op = 0; + break; + } + + if (fault_op) + { + rt_base_t saved_stat; + struct rt_aspace_fault_msg msg = + { + .fault_op = fault_op, + .fault_type = fault_type, + .fault_vaddr = (void *)tval, + }; + + lwp = lwp_self(); + + saved_stat = csr_read_set(CSR_STATUS, SR_IE); + + if (lwp && rt_aspace_fault_try_fix(lwp->aspace, &msg)) + { + csr_write(CSR_STATUS, saved_stat); + return RT_TRUE; + } + + csr_write(CSR_STATUS, saved_stat); + } + + return RT_FALSE; +} +#endif /* RT_USING_LWP */ + +#ifdef ARCH_RISCV_VECTOR +rt_inline void vector_enable(struct rt_hw_stack_frame *regs) +{ + regs->xstatus |= SR_VS_INITIAL; +} + +/** + * detect V/D support, and do not distinguish V/D instruction + */ +rt_inline rt_bool_t illegal_inst_recoverable(struct rt_hw_stack_frame *regs, rt_ubase_t tval) +{ + /* first 7 bits is opcode */ + int opcode = tval & 0x7f; + int csr = (tval & 0xfff00000) >> 20; + /* ref riscv-v-spec-1.0, [Vector Instruction Formats] */ + int width = ((tval & 0x7000) >> 12) - 1; + + switch (opcode) + { + case 0x57: /* V */ + case 0x27: /* scalar FLOAT */ + case 0x07: + case 0x73: /* CSR */ + vector_enable(regs); + return RT_TRUE; + + default: + break; + } + + return RT_FALSE; +} +#endif /* ARCH_RISCV_VECTOR */ + +/** + * this function will show registers of CPU + * + * @param regs the registers point + */ +void rt_hw_show_register(struct rt_hw_stack_frame *regs) +{ +#ifndef ARCH_RISCV_M_MODE + const char *sv_mode; + rt_ubase_t satp = csr_read(CSR_SATP); + + switch (satp & SATP_MODE_MASK) + { + case SATP_MODE_BARE: + sv_mode = "Bare"; + break; +#if __riscv_xlen == 32 + case SATP_MODE_32: + sv_mode = "Sv32"; + break; +#else + case SATP_MODE_39: + sv_mode = "Sv39"; + break; + case SATP_MODE_48: + sv_mode = "Sv48"; + break; + case SATP_MODE_57: + sv_mode = "Sv57"; + break; + case SATP_MODE_64: + sv_mode = "Sv64"; + break; +#endif /* __riscv_xlen == 32 */ + default: + sv_mode = "Reserved"; + break; + } +#endif /* !ARCH_RISCV_M_MODE */ + +#if __riscv_xlen == 64 +#define F "0x%016lx" +#elif __riscv_xlen == 32 +#define F "0x%08lx" +#else +#error "Unexpected __riscv_xlen" +#endif + rt_kprintf("Execption:\n"); + rt_kprintf("x0/zero: " F " x1/ra : " F " x2/sp : " F " x3/gp : " F "\n", 0UL, regs->ra, (rt_ubase_t)(regs + 1), regs->gp); + rt_kprintf("x4/tp : " F " x5/t0 : " F " x6/t1 : " F " x7/t2 : " F "\n", regs->tp, regs->t0, regs->t1, regs->t2); + rt_kprintf("x8/s0 : " F " x9/s1 : " F " x10/a0 : " F " x11/a1 : " F "\n", regs->s0_fp, regs->s1, regs->a0, regs->a1); + rt_kprintf("x12/a2 : " F " x13/a3: " F " x14/a4 : " F " x15/a5 : " F "\n", regs->a2, regs->a3, regs->a4, regs->a5); +#ifndef ARCH_RISCV_32E + rt_kprintf("x16/a6 : " F " x17/a7: " F " x18/s2 : " F " x19/s3 : " F "\n", regs->a6, regs->a7, regs->s2, regs->s3); + rt_kprintf("x20/s4 : " F " x21/s5: " F " x22/s6 : " F " x23/s7 : " F "\n", regs->s4, regs->s5, regs->s6, regs->s7); + rt_kprintf("x24/s8 : " F " x25/s9: " F " x26/s10: " F " x27/s11: " F "\n", regs->s8, regs->s9, regs->s10, regs->s11); + rt_kprintf("x28/t3 : " F " x29/t4: " F " x30/t5 : " F " x31/t6 : " F "\n", regs->t3, regs->t4, regs->t5, regs->t6); +#endif /* !ARCH_RISCV_32E */ +#ifdef ARCH_RISCV_M_MODE + rt_kprintf("status : " F " (%s %s %s)\n", regs->xstatus, + (regs->xstatus & SR_IE) ? "IE" : "--", + (regs->xstatus & SR_PIE) ? "PIE" : "---", + (regs->xstatus & SR_PP) ? "PP" : "--"); +#else + rt_kprintf("status : " F " (%s %s %s %s %s)\n", regs->xstatus, + (regs->xstatus & SR_IE) ? "IE" : "--", + (regs->xstatus & SR_PIE) ? "PIE" : "---", + (regs->xstatus & SR_PP) ? "PP" : "--", + (regs->xstatus & SR_SUM) ? "SUM" : "---", + (regs->xstatus & SR_MXR) ? "MXR" : "---"); + rt_kprintf("satp : " F " (ASID: " F ", %s)\n", satp, (satp >> SATP_ASID_SHIFT) & SATP_ASID_MASK, sv_mode); +#endif + rt_kprintf("cause : " F "\n", regs->xcause); + rt_kprintf("pc : " F "\n", regs->epc); +#undef F +} + +static const char * const irq_name[] = +{ + [IRQ_S_SOFT] = "Software Interrupt (User)", + [IRQ_VS_SOFT] = "Software Interrupt (Supervisor)", + [IRQ_M_SOFT] = "Software Interrupt (Machine)", + [IRQ_S_TIMER] = "Timer Interrupt (User)", + [IRQ_VS_TIMER] = "Timer Interrupt (Supervisor)", + [IRQ_M_TIMER] = "Timer Interrupt (Machine)", + [IRQ_S_EXT] = "External Interrupt (User)", + [IRQ_VS_EXT] = "External Interrupt (Supervisor)", + [IRQ_M_EXT] = "External Interrupt (Machine)", + [IRQ_S_GEXT] = "External Interrupt (Supervisor Guest)", + [IRQ_PMU_OVF] = "Performance Monitor Unit Overflow", +}; + +static const char * const exception_name[] = +{ + [EXC_INST_MISALIGNED] = "INST Misaligned", + [EXC_INST_ACCESS] = "INST Access", + [EXC_INST_ILLEGAL] = "INST Illegal", + [EXC_BREAKPOINT] = "Breakpoint", + [EXC_LOAD_MISALIGNED] = "Load Misaligned", + [EXC_LOAD_ACCESS] = "Load Access", + [EXC_STORE_MISALIGNED] = "Store Misaligned", + [EXC_STORE_ACCESS] = "Store Access", + [EXC_SYSCALL] = "Syscall", + [EXC_HYPERVISOR_SYSCALL] = "Hypervisor Syscall", + [EXC_SUPERVISOR_SYSCALL] = "Supervisor Syscall", + [EXC_INST_PAGE_FAULT] = "INST Page Fault", + [EXC_LOAD_PAGE_FAULT] = "Load Page Fault", + [EXC_STORE_PAGE_FAULT] = "Store Page Fault", + [EXC_INST_GUEST_PAGE_FAULT] = "INST Guest Page Fault", + [EXC_LOAD_GUEST_PAGE_FAULT] = "Load Guest Page Fault", + [EXC_VIRTUAL_INST_FAULT] = "Virtual INST Fault", + [EXC_STORE_GUEST_PAGE_FAULT] = "Store Guest Page Fault", +}; + +void rt_hw_trap(struct rt_hw_stack_frame *regs) +{ + rt_ubase_t xcause = regs->xcause; + rt_ubase_t tval = csr_read_clear(CSR_TVAL, 0); + + /* IRQ */ + if (xcause & CAUSE_IRQ_FLAG) + { + rt_ubase_t cause; + + rt_interrupt_enter(); + + xcause &= ~CAUSE_IRQ_FLAG; + cause = xcause & 0xfff; + + if (cause == RV_IRQ_TIMER) + { + riscv_timer_isr(); + goto _irq_done; + } + else if (cause == RV_IRQ_SOFT + #if defined(RT_PIC_RISCV_CLIC) || INTC_TYPE == INTC_TYPE_CLIC + || cause >= RV_IRQ_EXT) + #else + || cause == RV_IRQ_EXT) + #endif + { + #ifdef RT_USING_DM + if (!rt_pic_do_traps()) + { + goto _irq_done; + } + #else + rt_hw_interrupt_handle(cause, regs); + goto _irq_done; + #endif + } + + LOG_E("Unhandle IRQ: (%d) %s", xcause, + xcause >= sizeof(irq_name) / sizeof(irq_name[0]) ? "Reserved" : irq_name[xcause]); + + _irq_done: + rt_interrupt_leave(); + } + /* Execption */ + else + { + struct rt_hw_backtrace_frame frame; + + switch (xcause) + { + case EXC_INST_ILLEGAL: + #ifdef ARCH_RISCV_VECTOR + if (!(regs->xstatus & SR_VS)) + { + if (illegal_inst_recoverable(regs, tval)) + { + return; + } + } + #endif /* ARCH_RISCV_VECTOR */ + break; + + case EXC_INST_MISALIGNED: + case EXC_INST_ACCESS: + case EXC_LOAD_MISALIGNED: + case EXC_LOAD_ACCESS: + case EXC_STORE_MISALIGNED: + case EXC_STORE_ACCESS: + case EXC_INST_PAGE_FAULT: + case EXC_LOAD_PAGE_FAULT: + case EXC_STORE_PAGE_FAULT: + #ifdef RT_USING_SMART + if (!(regs->xstatus & SR_PP) || + (tval >= USER_VADDR_START && tval < USER_VADDR_TOP)) + { + if (user_fault_fixable(regs, xcause, tval)) + { + return; + } + } + #endif /* RT_USING_SMART */ + + default: + LOG_E("Unhandle Execption: (%d) %s, TVAL = %p", xcause, + xcause >= sizeof(exception_name) / sizeof(exception_name[0]) ? "Reserved" : exception_name[xcause], + tval); + } + + rt_hw_show_register(regs); + + if (rt_thread_self()) + { + LOG_E("current thread: %s\n", rt_thread_self()->parent.name); + + #ifdef RT_USING_FINSH + list_thread(); + #endif + + #ifdef RT_USING_LWP + _check_fault(regs, 0, "user fault"); + #endif + } + + frame.fp = regs->s0_fp; + frame.pc = regs->epc; + rt_backtrace_frame(rt_thread_self(), &frame); + rt_hw_cpu_shutdown(); + } +} diff --git a/libcpu/risc-v/common/vector_gcc.S b/libcpu/risc-v/common/vector_gcc.S new file mode 100644 index 0000000000..dff9a48181 --- /dev/null +++ b/libcpu/risc-v/common/vector_gcc.S @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2018-10-02 Bernard The first version + * 2018-12-27 Jesven Add SMP schedule + * 2021-02-02 lizhirui Add userspace support + * 2021-12-24 JasonHu Add user setting save/restore + * 2022-10-22 Shell Support kernel mode RVV; + * Rewrite trap handling routine + * 2025-04-20 GuEe-GUI Support SMP + */ + +#include <asm.h> +#include <csr.h> + + .section ".text","ax" + .align 6 +START_POINT(system_vectors) +#ifdef ARCH_RISCV_USING_IRQ_STACK + /* + * Keep the exception frame on the interrupted stack so the scheduler can + * store it in the thread TCB. Only C-level trap handling uses the IRQ + * stack. A zero xscratch means that a nested trap is already on it. + */ + csrrw sp, CSR_SCRATCH, sp + beqz sp, _irq_stack_nested + + /* Recover the interrupted SP without losing the original t0 and t1. */ + addi sp, sp, -(2 * SZREG) + REG_S t0, 0 * SZREG(sp) + REG_S t1, 1 * SZREG(sp) + mv t1, sp + csrr t0, CSR_SCRATCH + mv sp, t0 + REG_L t0, 0 * SZREG(t1) + REG_L t1, 1 * SZREG(t1) + csrw CSR_SCRATCH, sp + + SAVE_ALL + mv s0, sp + li s1, 1 + call riscv_irq_stack_top + mv sp, a0 + csrw CSR_SCRATCH, zero + j _context_saved + +_irq_stack_nested: + csrr sp, CSR_SCRATCH + SAVE_ALL + mv s0, sp + mv s1, zero + csrw CSR_SCRATCH, zero +#else + /* Distingush exception from kernel or user */ + csrrw sp, CSR_SCRATCH, sp + bnez sp, _save_context + + /* + * BE REALLY careful with xscratch, + * if it's wrong, we could looping here forever + * or accessing random memory and seeing things totally + * messy after a long time and don't even know why + */ +_from_kernel: + csrr sp, CSR_SCRATCH + j _save_context + +_save_context: + SAVE_ALL + + /* Clear xscratch to say 'now in kernel mode' */ + csrw CSR_SCRATCH, zero +#endif /* ARCH_RISCV_USING_IRQ_STACK */ + +_context_saved: + LOAD_GLOBAL_POINTER + + /* Now we are ready to enter interrupt / excepiton handler */ +#ifdef RT_USING_SMART +_distinguish_syscall: + csrr t0, CSR_CAUSE + li t1, EXC_SYSCALL + bne t0, t1, _handle_interrupt_and_exception + + call syscall_entry + /* Syscall never return here */ +#endif /* RT_USING_SMART */ + +_handle_interrupt_and_exception: + mv a0, sp +#ifdef ARCH_RISCV_USING_IRQ_STACK + mv a0, s0 +#endif + call rt_hw_trap + +_interrupt_exit: +#ifdef ARCH_RISCV_USING_IRQ_STACK + /* Restore the exception stack before invoking the scheduler. */ + mv sp, s0 + beqz s1, 1f + call riscv_irq_stack_top + csrw CSR_SCRATCH, a0 + j 2f +1: + csrw CSR_SCRATCH, zero +2: +#endif +#ifdef RT_USING_SMP + mv a0, sp + call rt_scheduler_do_irq_switch +#else + la t0, rt_thread_switch_interrupt_flag + lw t1, 0(t0) + beqz t1, _resume_execution + sw zero, 0(t0) + +_context_switch: + la t0, rt_interrupt_from_thread + REG_L a0, 0(t0) + la t0, rt_interrupt_to_thread + REG_L a1, 0(t0) + csrr t0, CSR_STATUS + andi t0, t0, ~SR_PIE + csrw CSR_STATUS, t0 + jal rt_hw_context_switch +#endif /* RT_USING_SMP */ + +_resume_execution: +.globl rt_hw_resume_interrupt +rt_hw_resume_interrupt: +#ifdef RT_USING_SMART + REG_L t0, FRAME_OFF_XSTATUS(sp) + andi t0, t0, SR_PP + bnez t0, rt_hw_resume_kernel + call arch_ret_to_user +#endif /* RT_USING_SMART */ + + j rt_hw_resume_kernel +START_POINT_END(system_vectors) + +.globl rt_hw_resume_kernel +rt_hw_resume_kernel: + RESTORE_ALL +#ifndef ARCH_RISCV_USING_IRQ_STACK + csrw CSR_SCRATCH, zero +#endif + XRET + + .align 2 + .weak fast_irq_vectors +START_POINT(fast_irq_vectors) + j system_vectors +START_POINT_END(fast_irq_vectors) diff --git a/libcpu/risc-v/common64/README.md b/libcpu/risc-v/common64/README.md index 5e7ea4bf92..ec3ed3e603 100644 --- a/libcpu/risc-v/common64/README.md +++ b/libcpu/risc-v/common64/README.md @@ -45,8 +45,9 @@ （2）PLIC中断控制器支持 -`libcpu/risc-v/virt64/plic.[c|h]`提供了符合《RISC-V Platform-Level Interrupt Controller Specification version 1.0.0 》标准的PLIC中断控制器驱动代码，可作为移植参考。 +PLIC 驱动请参考 `libcpu/risc-v/common/pic/pic-sifive-plic.c`。 （3）串口uart支持 -目前串口驱动在各bsp目录下，可参考`bsp/qemu-virt64-riscv/driver/drv_uart.[c|h]`
\ No newline at end of file +目前串口驱动在各 BSP 目录下，可参考 +`bsp/allwinner/libraries/drivers/drv_uart.c`。 diff --git a/libcpu/risc-v/common64/SConscript b/libcpu/risc-v/common64/SConscript index 553a7429db..2499b8531c 100644 --- a/libcpu/risc-v/common64/SConscript +++ b/libcpu/risc-v/common64/SConscript @@ -7,7 +7,7 @@ CPPPATH = [cwd] if not GetDepend('ARCH_USING_ASID'): SrcRemove(src, ['asid.c']) -src.append('../common/atomic_riscv.c') +src.append('../legacy_common/atomic_riscv.c') group = DefineGroup('CPU', src, depend = [''], CPPPATH = CPPPATH) diff --git a/libcpu/risc-v/common64/stackframe.h b/libcpu/risc-v/common64/stackframe.h index 25346db9bd..bad60256e1 100644 --- a/libcpu/risc-v/common64/stackframe.h +++ b/libcpu/risc-v/common64/stackframe.h @@ -42,6 +42,7 @@ #define FRAME_OFF_SP BYTES(32) #define FRAME_OFF_GP BYTES(3) #define FRAME_OFF_TP BYTES(4) +#define FRAME_OFF_A7 BYTES(17) /* switch frame */ #define RT_HW_SWITCH_CONTEXT_SSTATUS 0 diff --git a/libcpu/risc-v/t-head/c906/SConscript b/libcpu/risc-v/legacy-t-head/c906/SConscript index e8d49aadb2..e8d49aadb2 100644 --- a/libcpu/risc-v/t-head/c906/SConscript +++ b/libcpu/risc-v/legacy-t-head/c906/SConscript diff --git a/libcpu/risc-v/t-head/c906/asm/sbiasm.h b/libcpu/risc-v/legacy-t-head/c906/asm/sbiasm.h index 4639fba68c..4639fba68c 100644 --- a/libcpu/risc-v/t-head/c906/asm/sbiasm.h +++ b/libcpu/risc-v/legacy-t-head/c906/asm/sbiasm.h diff --git a/libcpu/risc-v/t-head/c906/asm/sbidef.h b/libcpu/risc-v/legacy-t-head/c906/asm/sbidef.h index 5bcf58ade7..5bcf58ade7 100644 --- a/libcpu/risc-v/t-head/c906/asm/sbidef.h +++ b/libcpu/risc-v/legacy-t-head/c906/asm/sbidef.h diff --git a/libcpu/risc-v/t-head/c906/cache.c b/libcpu/risc-v/legacy-t-head/c906/cache.c index af493e3adf..af493e3adf 100644 --- a/libcpu/risc-v/t-head/c906/cache.c +++ b/libcpu/risc-v/legacy-t-head/c906/cache.c diff --git a/libcpu/risc-v/t-head/c906/cache.h b/libcpu/risc-v/legacy-t-head/c906/cache.h index 7476de83cf..7476de83cf 100644 --- a/libcpu/risc-v/t-head/c906/cache.h +++ b/libcpu/risc-v/legacy-t-head/c906/cache.h diff --git a/libcpu/risc-v/t-head/c906/interrupt.c b/libcpu/risc-v/legacy-t-head/c906/interrupt.c index 339dde808f..339dde808f 100644 --- a/libcpu/risc-v/t-head/c906/interrupt.c +++ b/libcpu/risc-v/legacy-t-head/c906/interrupt.c diff --git a/libcpu/risc-v/t-head/c906/interrupt.h b/libcpu/risc-v/legacy-t-head/c906/interrupt.h index bdc98972b2..bdc98972b2 100644 --- a/libcpu/risc-v/t-head/c906/interrupt.h +++ b/libcpu/risc-v/legacy-t-head/c906/interrupt.h diff --git a/libcpu/risc-v/t-head/c906/opcode.h b/libcpu/risc-v/legacy-t-head/c906/opcode.h index 27c2f91bf8..27c2f91bf8 100644 --- a/libcpu/risc-v/t-head/c906/opcode.h +++ b/libcpu/risc-v/legacy-t-head/c906/opcode.h diff --git a/libcpu/risc-v/t-head/c906/plic.c b/libcpu/risc-v/legacy-t-head/c906/plic.c index 829ee9e4fd..829ee9e4fd 100644 --- a/libcpu/risc-v/t-head/c906/plic.c +++ b/libcpu/risc-v/legacy-t-head/c906/plic.c diff --git a/libcpu/risc-v/t-head/c906/plic.h b/libcpu/risc-v/legacy-t-head/c906/plic.h index 93235279e2..93235279e2 100644 --- a/libcpu/risc-v/t-head/c906/plic.h +++ b/libcpu/risc-v/legacy-t-head/c906/plic.h diff --git a/libcpu/risc-v/t-head/c906/riscv_mmu.h b/libcpu/risc-v/legacy-t-head/c906/riscv_mmu.h index 2bf8e93305..2bf8e93305 100644 --- a/libcpu/risc-v/t-head/c906/riscv_mmu.h +++ b/libcpu/risc-v/legacy-t-head/c906/riscv_mmu.h diff --git a/libcpu/risc-v/t-head/c906/symbol_analysis.c b/libcpu/risc-v/legacy-t-head/c906/symbol_analysis.c index 83edacb1c4..83edacb1c4 100644 --- a/libcpu/risc-v/t-head/c906/symbol_analysis.c +++ b/libcpu/risc-v/legacy-t-head/c906/symbol_analysis.c diff --git a/libcpu/risc-v/t-head/c906/symbol_analysis.h b/libcpu/risc-v/legacy-t-head/c906/symbol_analysis.h index a9db46ffd3..a9db46ffd3 100644 --- a/libcpu/risc-v/t-head/c906/symbol_analysis.h +++ b/libcpu/risc-v/legacy-t-head/c906/symbol_analysis.h diff --git a/libcpu/risc-v/t-head/c908/SConscript b/libcpu/risc-v/legacy-t-head/c908/SConscript index 1562b9c14f..1562b9c14f 100644 --- a/libcpu/risc-v/t-head/c908/SConscript +++ b/libcpu/risc-v/legacy-t-head/c908/SConscript diff --git a/libcpu/risc-v/t-head/c908/asm/sbiasm.h b/libcpu/risc-v/legacy-t-head/c908/asm/sbiasm.h index 4639fba68c..4639fba68c 100644 --- a/libcpu/risc-v/t-head/c908/asm/sbiasm.h +++ b/libcpu/risc-v/legacy-t-head/c908/asm/sbiasm.h diff --git a/libcpu/risc-v/t-head/c908/asm/sbidef.h b/libcpu/risc-v/legacy-t-head/c908/asm/sbidef.h index 5bcf58ade7..5bcf58ade7 100644 --- a/libcpu/risc-v/t-head/c908/asm/sbidef.h +++ b/libcpu/risc-v/legacy-t-head/c908/asm/sbidef.h diff --git a/libcpu/risc-v/t-head/c908/cache.c b/libcpu/risc-v/legacy-t-head/c908/cache.c index db573fa5f8..db573fa5f8 100644 --- a/libcpu/risc-v/t-head/c908/cache.c +++ b/libcpu/risc-v/legacy-t-head/c908/cache.c diff --git a/libcpu/risc-v/t-head/c908/cache.h b/libcpu/risc-v/legacy-t-head/c908/cache.h index cf7b21c88a..cf7b21c88a 100644 --- a/libcpu/risc-v/t-head/c908/cache.h +++ b/libcpu/risc-v/legacy-t-head/c908/cache.h diff --git a/libcpu/risc-v/t-head/c908/interrupt.c b/libcpu/risc-v/legacy-t-head/c908/interrupt.c index 7caed0203f..7caed0203f 100644 --- a/libcpu/risc-v/t-head/c908/interrupt.c +++ b/libcpu/risc-v/legacy-t-head/c908/interrupt.c diff --git a/libcpu/risc-v/t-head/c908/interrupt.h b/libcpu/risc-v/legacy-t-head/c908/interrupt.h index d1b7ea13a3..d1b7ea13a3 100644 --- a/libcpu/risc-v/t-head/c908/interrupt.h +++ b/libcpu/risc-v/legacy-t-head/c908/interrupt.h diff --git a/libcpu/risc-v/t-head/c908/opcode.h b/libcpu/risc-v/legacy-t-head/c908/opcode.h index 90b002dc4e..90b002dc4e 100644 --- a/libcpu/risc-v/t-head/c908/opcode.h +++ b/libcpu/risc-v/legacy-t-head/c908/opcode.h diff --git a/libcpu/risc-v/t-head/c908/plic.c b/libcpu/risc-v/legacy-t-head/c908/plic.c index 7cb313280d..7cb313280d 100644 --- a/libcpu/risc-v/t-head/c908/plic.c +++ b/libcpu/risc-v/legacy-t-head/c908/plic.c diff --git a/libcpu/risc-v/t-head/c908/plic.h b/libcpu/risc-v/legacy-t-head/c908/plic.h index 94f00eacdb..94f00eacdb 100644 --- a/libcpu/risc-v/t-head/c908/plic.h +++ b/libcpu/risc-v/legacy-t-head/c908/plic.h diff --git a/libcpu/risc-v/t-head/c908/riscv_mmu.h b/libcpu/risc-v/legacy-t-head/c908/riscv_mmu.h index 84d764bd73..84d764bd73 100644 --- a/libcpu/risc-v/t-head/c908/riscv_mmu.h +++ b/libcpu/risc-v/legacy-t-head/c908/riscv_mmu.h diff --git a/libcpu/risc-v/t-head/c908/symbol_analysis.c b/libcpu/risc-v/legacy-t-head/c908/symbol_analysis.c index 83edacb1c4..83edacb1c4 100644 --- a/libcpu/risc-v/t-head/c908/symbol_analysis.c +++ b/libcpu/risc-v/legacy-t-head/c908/symbol_analysis.c diff --git a/libcpu/risc-v/t-head/c908/symbol_analysis.h b/libcpu/risc-v/legacy-t-head/c908/symbol_analysis.h index a9db46ffd3..a9db46ffd3 100644 --- a/libcpu/risc-v/t-head/c908/symbol_analysis.h +++ b/libcpu/risc-v/legacy-t-head/c908/symbol_analysis.h diff --git a/libcpu/risc-v/legacy_common/SConscript b/libcpu/risc-v/legacy_common/SConscript new file mode 100644 index 0000000000..814cfb01d5 --- /dev/null +++ b/libcpu/risc-v/legacy_common/SConscript @@ -0,0 +1,15 @@ +# RT-Thread building script for component + +from building import * + +cwd = GetCurrentDir() +src = Glob('*.c') + Glob('*.cpp') + Glob('*_gcc.S') +CPPPATH = [cwd] +ASFLAGS = ' -I ' + cwd + +if not GetDepend('RT_USING_HW_ATOMIC'): + SrcRemove(src, 'atomic_riscv.c') + +group = DefineGroup('CPU', src, depend = [''], CPPPATH = CPPPATH, ASFLAGS = ASFLAGS) + +Return('group') diff --git a/libcpu/risc-v/legacy_common/atomic_riscv.c b/libcpu/risc-v/legacy_common/atomic_riscv.c new file mode 100644 index 0000000000..bc1561f2ee --- /dev/null +++ b/libcpu/risc-v/legacy_common/atomic_riscv.c @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-03-14 WangShun first version + */ + +#include <rtthread.h> + +rt_atomic_t rt_hw_atomic_exchange(volatile rt_atomic_t *ptr, rt_atomic_t val) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile ("amoswap.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile ("amoswap.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#endif + return result; +} + +rt_atomic_t rt_hw_atomic_add(volatile rt_atomic_t *ptr, rt_atomic_t val) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile ("amoadd.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile ("amoadd.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#endif + return result; +} + +rt_atomic_t rt_hw_atomic_sub(volatile rt_atomic_t *ptr, rt_atomic_t val) +{ + rt_atomic_t result = 0; + val = -val; +#if __riscv_xlen == 32 + asm volatile ("amoadd.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile ("amoadd.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#endif + return result; +} + +rt_atomic_t rt_hw_atomic_xor(volatile rt_atomic_t *ptr, rt_atomic_t val) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile ("amoxor.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile ("amoxor.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#endif + return result; +} + +rt_atomic_t rt_hw_atomic_and(volatile rt_atomic_t *ptr, rt_atomic_t val) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile ("amoand.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile ("amoand.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#endif + return result; +} + +rt_atomic_t rt_hw_atomic_or(volatile rt_atomic_t *ptr, rt_atomic_t val) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile ("amoor.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile ("amoor.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#endif + return result; +} + +rt_atomic_t rt_hw_atomic_load(volatile rt_atomic_t *ptr) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile ("amoxor.w %0, x0, (%1)" : "=r"(result) : "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile ("amoxor.d %0, x0, (%1)" : "=r"(result) : "r"(ptr) : "memory"); +#endif + return result; +} + +void rt_hw_atomic_store(volatile rt_atomic_t *ptr, rt_atomic_t val) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile ("amoswap.w %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile ("amoswap.d %0, %1, (%2)" : "=r"(result) : "r"(val), "r"(ptr) : "memory"); +#endif +} + +rt_atomic_t rt_hw_atomic_flag_test_and_set(volatile rt_atomic_t *ptr) +{ + rt_atomic_t result = 0; + rt_atomic_t temp = 1; +#if __riscv_xlen == 32 + asm volatile ("amoor.w %0, %1, (%2)" : "=r"(result) : "r"(temp), "r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile ("amoor.d %0, %1, (%2)" : "=r"(result) : "r"(temp), "r"(ptr) : "memory"); +#endif + return result; +} + +void rt_hw_atomic_flag_clear(volatile rt_atomic_t *ptr) +{ + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile ("amoand.w %0, x0, (%1)" : "=r"(result) :"r"(ptr) : "memory"); +#elif __riscv_xlen == 64 + asm volatile ("amoand.d %0, x0, (%1)" : "=r"(result) :"r"(ptr) : "memory"); +#endif +} + +rt_atomic_t rt_hw_atomic_compare_exchange_strong(volatile rt_atomic_t *ptr, rt_atomic_t *old, rt_atomic_t desired) +{ + rt_atomic_t tmp = *old; + rt_atomic_t result = 0; +#if __riscv_xlen == 32 + asm volatile( + " fence iorw, ow\n" + "1: lr.w.aq %[result], (%[ptr])\n" + " bne %[result], %[tmp], 2f\n" + " sc.w.rl %[tmp], %[desired], (%[ptr])\n" + " bnez %[tmp], 1b\n" + " li %[result], 1\n" + " j 3f\n" + " 2:sw %[result], (%[old])\n" + " li %[result], 0\n" + " 3:\n" + : [result]"+r" (result), [tmp]"+r" (tmp), [ptr]"+r" (ptr) + : [desired]"r" (desired), [old]"r"(old) + : "memory"); +#elif __riscv_xlen == 64 + asm volatile( + " fence iorw, ow\n" + "1: lr.d.aq %[result], (%[ptr])\n" + " bne %[result], %[tmp], 2f\n" + " sc.d.rl %[tmp], %[desired], (%[ptr])\n" + " bnez %[tmp], 1b\n" + " li %[result], 1\n" + " j 3f\n" + " 2:sd %[result], (%[old])\n" + " li %[result], 0\n" + " 3:\n" + : [result]"+r" (result), [tmp]"+r" (tmp), [ptr]"+r" (ptr) + : [desired]"r" (desired), [old]"r"(old) + : "memory"); +#endif + return result; +} diff --git a/libcpu/risc-v/legacy_common/context_gcc.S b/libcpu/risc-v/legacy_common/context_gcc.S new file mode 100644 index 0000000000..550b6e4a4d --- /dev/null +++ b/libcpu/risc-v/legacy_common/context_gcc.S @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2006-2018, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2018/10/28 Bernard The unify RISC-V porting implementation + * 2018/12/27 Jesven Add SMP support + * 2020/11/20 BalanceTWK Add FPU support + * 2022/12/28 WangShun Add macro to distinguish whether FPU is supported + * 2023/03/19 Flyingcys Add riscv_32e support + */ + +#define __ASSEMBLY__ +#include "cpuport.h" + +#ifdef RT_USING_SMP +#define rt_hw_interrupt_disable rt_hw_local_irq_disable +#define rt_hw_interrupt_enable rt_hw_local_irq_enable +#endif + +/* + * rt_base_t rt_hw_interrupt_disable(void); + */ + .globl rt_hw_interrupt_disable +rt_hw_interrupt_disable: + csrrci a0, mstatus, 8 + ret + +/* + * void rt_hw_interrupt_enable(rt_base_t level); + */ + .globl rt_hw_interrupt_enable +rt_hw_interrupt_enable: + csrw mstatus, a0 + ret + +/* + * #ifdef RT_USING_SMP + * void rt_hw_context_switch_to(rt_ubase_t to, stuct rt_thread *to_thread); + * #else + * void rt_hw_context_switch_to(rt_ubase_t to); + * #endif + * a0 --> to + * a1 --> to_thread + */ + .globl rt_hw_context_switch_to +rt_hw_context_switch_to: + la t0, __rt_rvstack +#ifdef SOC_RISCV_FAMILY_CH32 +/* + * if it is an assembly entry code, the SP offset value is determined by the assembly code, + * but the C code is determined by the compiler, so we subtract 512 here as a reservation. + * When entering the interrupt function of C code, the compiler automatically presses the stack + * into the task stack. We can only change the SP value used by the calling function after switching + * the interrupt stack.This problem can be solved by modifying the interrupt to the assembly entry, + * and there is no need to reserve 512 bytes. You only need to switch the interrupt stack at the + * beginning of the interrupt function +*/ + addi t0, t0, -512 // for ch32 +#endif /* SOC_RISCV_FAMILY_CH32 */ + csrw mscratch,t0 + + LOAD sp, (a0) + +#ifdef RT_USING_SMP + mv a0, a1 + call rt_cpus_lock_status_restore +#endif + LOAD a0, 2 * REGBYTES(sp) + csrw mstatus, a0 + j rt_hw_context_switch_exit + +/* + * #ifdef RT_USING_SMP + * void rt_hw_context_switch(rt_ubase_t from, rt_ubase_t to, struct rt_thread *to_thread); + * #else + * void rt_hw_context_switch(rt_ubase_t from, rt_ubase_t to); + * #endif + * + * a0 --> from + * a1 --> to + * a2 --> to_thread + */ + .globl rt_hw_context_switch +rt_hw_context_switch: + /* saved from thread context + * x1/ra -> sp(0) + * x1/ra -> sp(1) + * mstatus.mie -> sp(2) + * x(i) -> sp(i-4) + */ +#ifdef ARCH_RISCV_FPU + addi sp, sp, -32 * FREGBYTES + + FSTORE f0, 0 * FREGBYTES(sp) + FSTORE f1, 1 * FREGBYTES(sp) + FSTORE f2, 2 * FREGBYTES(sp) + FSTORE f3, 3 * FREGBYTES(sp) + FSTORE f4, 4 * FREGBYTES(sp) + FSTORE f5, 5 * FREGBYTES(sp) + FSTORE f6, 6 * FREGBYTES(sp) + FSTORE f7, 7 * FREGBYTES(sp) + FSTORE f8, 8 * FREGBYTES(sp) + FSTORE f9, 9 * FREGBYTES(sp) + FSTORE f10, 10 * FREGBYTES(sp) + FSTORE f11, 11 * FREGBYTES(sp) + FSTORE f12, 12 * FREGBYTES(sp) + FSTORE f13, 13 * FREGBYTES(sp) + FSTORE f14, 14 * FREGBYTES(sp) + FSTORE f15, 15 * FREGBYTES(sp) + FSTORE f16, 16 * FREGBYTES(sp) + FSTORE f17, 17 * FREGBYTES(sp) + FSTORE f18, 18 * FREGBYTES(sp) + FSTORE f19, 19 * FREGBYTES(sp) + FSTORE f20, 20 * FREGBYTES(sp) + FSTORE f21, 21 * FREGBYTES(sp) + FSTORE f22, 22 * FREGBYTES(sp) + FSTORE f23, 23 * FREGBYTES(sp) + FSTORE f24, 24 * FREGBYTES(sp) + FSTORE f25, 25 * FREGBYTES(sp) + FSTORE f26, 26 * FREGBYTES(sp) + FSTORE f27, 27 * FREGBYTES(sp) + FSTORE f28, 28 * FREGBYTES(sp) + FSTORE f29, 29 * FREGBYTES(sp) + FSTORE f30, 30 * FREGBYTES(sp) + FSTORE f31, 31 * FREGBYTES(sp) + +#endif +#ifndef __riscv_32e + addi sp, sp, -32 * REGBYTES +#else + addi sp, sp, -16 * REGBYTES +#endif + + STORE sp, (a0) + + STORE x1, 0 * REGBYTES(sp) + STORE x1, 1 * REGBYTES(sp) + + csrr a0, mstatus + andi a0, a0, 8 + beqz a0, save_mpie + li a0, 0x80 +save_mpie: + STORE a0, 2 * REGBYTES(sp) + + STORE x4, 4 * REGBYTES(sp) + STORE x5, 5 * REGBYTES(sp) + STORE x6, 6 * REGBYTES(sp) + STORE x7, 7 * REGBYTES(sp) + STORE x8, 8 * REGBYTES(sp) + STORE x9, 9 * REGBYTES(sp) + STORE x10, 10 * REGBYTES(sp) + STORE x11, 11 * REGBYTES(sp) + STORE x12, 12 * REGBYTES(sp) + STORE x13, 13 * REGBYTES(sp) + STORE x14, 14 * REGBYTES(sp) + STORE x15, 15 * REGBYTES(sp) +#ifndef __riscv_32e + STORE x16, 16 * REGBYTES(sp) + STORE x17, 17 * REGBYTES(sp) + STORE x18, 18 * REGBYTES(sp) + STORE x19, 19 * REGBYTES(sp) + STORE x20, 20 * REGBYTES(sp) + STORE x21, 21 * REGBYTES(sp) + STORE x22, 22 * REGBYTES(sp) + STORE x23, 23 * REGBYTES(sp) + STORE x24, 24 * REGBYTES(sp) + STORE x25, 25 * REGBYTES(sp) + STORE x26, 26 * REGBYTES(sp) + STORE x27, 27 * REGBYTES(sp) + STORE x28, 28 * REGBYTES(sp) + STORE x29, 29 * REGBYTES(sp) + STORE x30, 30 * REGBYTES(sp) + STORE x31, 31 * REGBYTES(sp) +#endif + /* restore to thread context + * sp(0) -> epc; + * sp(1) -> ra; + * sp(i) -> x(i+2) + */ + LOAD sp, (a1) + +#ifdef RT_USING_SMP + mv a0, a2 + call rt_cpus_lock_status_restore +#endif /*RT_USING_SMP*/ + + j rt_hw_context_switch_exit + +#ifdef RT_USING_SMP +/* + * void rt_hw_context_switch_interrupt(void *context, rt_ubase_t from, rt_ubase_t to, struct rt_thread *to_thread); + * + * a0 --> context + * a1 --> from + * a2 --> to + * a3 --> to_thread + */ + .globl rt_hw_context_switch_interrupt +rt_hw_context_switch_interrupt: + + STORE a0, 0(a1) + + LOAD sp, 0(a2) + move a0, a3 + call rt_cpus_lock_status_restore + + j rt_hw_context_switch_exit + +#endif + +.global rt_hw_context_switch_exit +rt_hw_context_switch_exit: +#ifdef RT_USING_SMP +#ifdef RT_USING_SIGNALS + mv a0, sp + + csrr t0, mhartid + /* switch interrupt stack of current cpu */ + la sp, __stack_start__ + addi t1, t0, 1 + li t2, __STACKSIZE__ + mul t1, t1, t2 + add sp, sp, t1 /* sp = (cpuid + 1) * __STACKSIZE__ + __stack_start__ */ + + call rt_signal_check + mv sp, a0 +#endif +#endif + /* resw ra to mepc */ + LOAD a0, 0 * REGBYTES(sp) + csrw mepc, a0 + + LOAD x1, 1 * REGBYTES(sp) +#ifdef ARCH_RISCV_FPU + li t0, 0x7800 +#else + li t0, 0x1800 +#endif + csrw mstatus, t0 + LOAD a0, 2 * REGBYTES(sp) + csrs mstatus, a0 + + LOAD x4, 4 * REGBYTES(sp) + LOAD x5, 5 * REGBYTES(sp) + LOAD x6, 6 * REGBYTES(sp) + LOAD x7, 7 * REGBYTES(sp) + LOAD x8, 8 * REGBYTES(sp) + LOAD x9, 9 * REGBYTES(sp) + LOAD x10, 10 * REGBYTES(sp) + LOAD x11, 11 * REGBYTES(sp) + LOAD x12, 12 * REGBYTES(sp) + LOAD x13, 13 * REGBYTES(sp) + LOAD x14, 14 * REGBYTES(sp) + LOAD x15, 15 * REGBYTES(sp) +#ifndef __riscv_32e + LOAD x16, 16 * REGBYTES(sp) + LOAD x17, 17 * REGBYTES(sp) + LOAD x18, 18 * REGBYTES(sp) + LOAD x19, 19 * REGBYTES(sp) + LOAD x20, 20 * REGBYTES(sp) + LOAD x21, 21 * REGBYTES(sp) + LOAD x22, 22 * REGBYTES(sp) + LOAD x23, 23 * REGBYTES(sp) + LOAD x24, 24 * REGBYTES(sp) + LOAD x25, 25 * REGBYTES(sp) + LOAD x26, 26 * REGBYTES(sp) + LOAD x27, 27 * REGBYTES(sp) + LOAD x28, 28 * REGBYTES(sp) + LOAD x29, 29 * REGBYTES(sp) + LOAD x30, 30 * REGBYTES(sp) + LOAD x31, 31 * REGBYTES(sp) + + addi sp, sp, 32 * REGBYTES +#else + addi sp, sp, 16 * REGBYTES +#endif + +#ifdef ARCH_RISCV_FPU + FLOAD f0, 0 * FREGBYTES(sp) + FLOAD f1, 1 * FREGBYTES(sp) + FLOAD f2, 2 * FREGBYTES(sp) + FLOAD f3, 3 * FREGBYTES(sp) + FLOAD f4, 4 * FREGBYTES(sp) + FLOAD f5, 5 * FREGBYTES(sp) + FLOAD f6, 6 * FREGBYTES(sp) + FLOAD f7, 7 * FREGBYTES(sp) + FLOAD f8, 8 * FREGBYTES(sp) + FLOAD f9, 9 * FREGBYTES(sp) + FLOAD f10, 10 * FREGBYTES(sp) + FLOAD f11, 11 * FREGBYTES(sp) + FLOAD f12, 12 * FREGBYTES(sp) + FLOAD f13, 13 * FREGBYTES(sp) + FLOAD f14, 14 * FREGBYTES(sp) + FLOAD f15, 15 * FREGBYTES(sp) + FLOAD f16, 16 * FREGBYTES(sp) + FLOAD f17, 17 * FREGBYTES(sp) + FLOAD f18, 18 * FREGBYTES(sp) + FLOAD f19, 19 * FREGBYTES(sp) + FLOAD f20, 20 * FREGBYTES(sp) + FLOAD f21, 21 * FREGBYTES(sp) + FLOAD f22, 22 * FREGBYTES(sp) + FLOAD f23, 23 * FREGBYTES(sp) + FLOAD f24, 24 * FREGBYTES(sp) + FLOAD f25, 25 * FREGBYTES(sp) + FLOAD f26, 26 * FREGBYTES(sp) + FLOAD f27, 27 * FREGBYTES(sp) + FLOAD f28, 28 * FREGBYTES(sp) + FLOAD f29, 29 * FREGBYTES(sp) + FLOAD f30, 30 * FREGBYTES(sp) + FLOAD f31, 31 * FREGBYTES(sp) + + addi sp, sp, 32 * FREGBYTES +#endif + + mret diff --git a/libcpu/risc-v/legacy_common/cpuport.c b/libcpu/risc-v/legacy_common/cpuport.c new file mode 100644 index 0000000000..502a8fcbf3 --- /dev/null +++ b/libcpu/risc-v/legacy_common/cpuport.c @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2018/10/28 Bernard The unify RISC-V porting code. + * 2020/11/20 BalanceTWK Add FPU support + * 2023/01/04 WangShun Adapt to CH32 + */ + +#include <rthw.h> +#include <rtthread.h> + +#include "cpuport.h" +#include "rt_hw_stack_frame.h" + +#ifndef RT_USING_SMP +volatile rt_ubase_t rt_interrupt_from_thread = 0; +volatile rt_ubase_t rt_interrupt_to_thread = 0; +volatile rt_uint32_t rt_thread_switch_interrupt_flag = 0; +#endif + +/** + * This function will initialize thread stack + * + * @param tentry the entry of thread + * @param parameter the parameter of entry + * @param stack_addr the beginning stack address + * @param texit the function will be called when thread exit + * + * @return stack address + */ +rt_uint8_t *rt_hw_stack_init(void *tentry, + void *parameter, + rt_uint8_t *stack_addr, + void *texit) +{ + struct rt_hw_stack_frame *frame; + rt_uint8_t *stk; + int i; + + stk = stack_addr + sizeof(rt_ubase_t); + stk = (rt_uint8_t *)RT_ALIGN_DOWN((rt_ubase_t)stk, REGBYTES); + stk -= sizeof(struct rt_hw_stack_frame); + + frame = (struct rt_hw_stack_frame *)stk; + + for (i = 0; i < sizeof(struct rt_hw_stack_frame) / sizeof(rt_ubase_t); i++) + { + ((rt_ubase_t *)frame)[i] = 0xdeadbeef; + } + + frame->ra = (rt_ubase_t)texit; + frame->a0 = (rt_ubase_t)parameter; + frame->epc = (rt_ubase_t)tentry; + + /* force to machine mode(MPP=11) and set MPIE to 1 */ +#ifdef ARCH_RISCV_FPU + frame->mstatus = 0x7880; +#else + frame->mstatus = 0x1880; +#endif + + return stk; +} + +rt_weak void rt_trigger_software_interrupt(void) +{ + while (0); +} + +rt_weak void rt_hw_do_after_save_above(void) +{ + while (1); +} + +/* + * #ifdef RT_USING_SMP + * void rt_hw_context_switch_interrupt(void *context, rt_ubase_t from, rt_ubase_t to, struct rt_thread *to_thread); + * #else + * void rt_hw_context_switch_interrupt(rt_ubase_t from, rt_ubase_t to); + * #endif + */ +#ifndef RT_USING_SMP +rt_weak void rt_hw_context_switch_interrupt(rt_ubase_t from, rt_ubase_t to, rt_thread_t from_thread, rt_thread_t to_thread) +{ + if (rt_thread_switch_interrupt_flag == 0) + rt_interrupt_from_thread = from; + + rt_interrupt_to_thread = to; + rt_thread_switch_interrupt_flag = 1; + + rt_trigger_software_interrupt(); + + return ; +} +#endif /* end of RT_USING_SMP */ diff --git a/libcpu/risc-v/common/cpuport.h b/libcpu/risc-v/legacy_common/cpuport.h index e485e91a4f..e485e91a4f 100644 --- a/libcpu/risc-v/common/cpuport.h +++ b/libcpu/risc-v/legacy_common/cpuport.h diff --git a/libcpu/risc-v/common/interrupt_gcc.S b/libcpu/risc-v/legacy_common/interrupt_gcc.S index dad8cac424..dad8cac424 100644 --- a/libcpu/risc-v/common/interrupt_gcc.S +++ b/libcpu/risc-v/legacy_common/interrupt_gcc.S diff --git a/libcpu/risc-v/common/readme.md b/libcpu/risc-v/legacy_common/readme.md index 42be5a2025..1252329563 100644 --- a/libcpu/risc-v/common/readme.md +++ b/libcpu/risc-v/legacy_common/readme.md @@ -2,7 +2,7 @@ #### 1.概述 -为了简化32位RISC-V架构内核移植RT-Thread的流程，RT-Thread提供一分通用代码于common文件夹 +为了简化32位RISC-V架构内核移植RT-Thread的流程，RT-Thread提供一份兼容代码于legacy_common文件夹 | 文件名 | 文件内容 | | :-----------------: | :----------------------------: | @@ -49,7 +49,7 @@ void rt_hw_do_after_save_above(void) #### 4.移植步骤 -- 步骤一：配置中断管理入口，相关中断入口函数位于**common/interrupt_gcc.S**,入口函数为**SW_handler** +- 步骤一：配置中断管理入口，相关中断入口函数位于**legacy_common/interrupt_gcc.S**,入口函数为**SW_handler** - 根据使用的中断管理方式，执行下述操作 @@ -110,7 +110,7 @@ void rt_hw_do_after_save_above(void) - 步骤三：实现在中断上下文切换的函数接口 - <font color=red>RISC-V架构的内核通常采用非向量中断的管理方式，为了进一步降低难度，针对非向量模式的中断管理方式，common文件夹中的trap_common.c为用户提供了一套统一的中断查询分发、中断入口函数注册以及中断初始化函数，在rthw.h中声明，对于移植一个新的RV32内核，若采用非向量中断管理的方式，推荐使用方式一，若采用向量中断管理方式或针对中断的处理有专门的优化时推荐使用方式二,期望采用原有裸机工程的统一的中断查询与处理函数也可使用方式二。以下是两种实现方式的示例：</font> + <font color=red>RISC-V架构的内核通常采用非向量中断的管理方式，为了进一步降低难度，针对非向量模式的中断管理方式，legacy_common文件夹中的trap_common.c为用户提供了一套统一的中断查询分发、中断入口函数注册以及中断初始化函数，在rthw.h中声明，对于移植一个新的RV32内核，若采用非向量中断管理的方式，推荐使用方式一，若采用向量中断管理方式或针对中断的处理有专门的优化时推荐使用方式二,期望采用原有裸机工程的统一的中断查询与处理函数也可使用方式二。以下是两种实现方式的示例：</font> 方式一：面向非向量中断管理方式(例:core-v-mcu) @@ -184,4 +184,3 @@ void rt_hw_do_after_save_above(void) - 创建一个静态线程，在线程中调用RT-Thread提供的与系统时基相关函数接口，例如rt_thread_mdelay，调试观察系统是否可以正常运行 - 移植RT-Thread的shell进一步验证系统是否移植成功 - diff --git a/libcpu/risc-v/common/riscv-ops.h b/libcpu/risc-v/legacy_common/riscv-ops.h index 35d009ad53..35d009ad53 100644 --- a/libcpu/risc-v/common/riscv-ops.h +++ b/libcpu/risc-v/legacy_common/riscv-ops.h diff --git a/libcpu/risc-v/common/rt_hw_stack_frame.h b/libcpu/risc-v/legacy_common/rt_hw_stack_frame.h index 75a9c2d45a..75a9c2d45a 100644 --- a/libcpu/risc-v/common/rt_hw_stack_frame.h +++ b/libcpu/risc-v/legacy_common/rt_hw_stack_frame.h diff --git a/libcpu/risc-v/common/trap_common.c b/libcpu/risc-v/legacy_common/trap_common.c index 9f8b03885b..9f8b03885b 100644 --- a/libcpu/risc-v/common/trap_common.c +++ b/libcpu/risc-v/legacy_common/trap_common.c diff --git a/libcpu/risc-v/link.lds.S b/libcpu/risc-v/link.lds.S new file mode 100644 index 0000000000..67d49bf0e4 --- /dev/null +++ b/libcpu/risc-v/link.lds.S @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Date Author Notes + * 2020-12-12 bernard first version + */ + +#define __ASSEMBLY__ + +#include "rtconfig.h" + +#ifndef __riscv_xlen +#ifdef ARCH_RISCV64 +#define __riscv_xlen 64 +#else +#define __riscv_xlen 32 +#endif +#endif + +#include "page_def.h" + +OUTPUT_ARCH(riscv) + +#ifndef ARCH_TEXT_OFFSET +#if __riscv_xlen == 64 +#define ARCH_TEXT_OFFSET 0x200000 +#elif __riscv_xlen == 32 +#define ARCH_TEXT_OFFSET 0x400000 +#else +#error "Unexpected __riscv_xlen" +#endif +#endif + +#ifndef ARCH_RAM_OFFSET +#define ARCH_RAM_OFFSET 0 +#endif + +PHDRS +{ + text PT_LOAD FLAGS(5); /* PF_R(4) | PF_X(1) = 0x5 (RX) */ + data PT_LOAD FLAGS(6); /* PF_R(4) | PF_W(2) = 0x6 (RW) */ +} + +SECTIONS +{ + _text_offset = ARCH_TEXT_OFFSET; + +#ifdef RT_USING_SMART + . = KERNEL_VADDR_START + _text_offset; +#else + . = ARCH_RAM_OFFSET + _text_offset; +#endif + + .text : + { + PROVIDE(__text_start = .); + + KEEP(*(.start)) /* The start point */ + KEEP(*(.text.start)) + *(.text) /* Remaining code */ + *(.text.*) /* Remaining code */ + + *(.rodata) /* Read-only data (constants) */ + *(.rodata*) + *(.glue_7) + *(.glue_7t) + *(.gnu.linkonce.t*) + + /* section information for utest */ + . = ALIGN(RISCV_SZPTR); + PROVIDE(__rt_utest_tc_tab_start = .); + KEEP(*(UtestTcTab)) + PROVIDE(__rt_utest_tc_tab_end = .); + + /* section information for finsh shell */ + . = ALIGN(RISCV_SZPTR); + PROVIDE(__fsymtab_start = .); + KEEP(*(FSymTab)) + PROVIDE(__fsymtab_end = .); + . = ALIGN(RISCV_SZPTR); + PROVIDE(__vsymtab_start = .); + KEEP(*(VSymTab)) + PROVIDE(__vsymtab_end = .); + . = ALIGN(RISCV_SZPTR); + + /* section information for modules */ + . = ALIGN(RISCV_SZPTR); + PROVIDE(__rtmsymtab_start = .); + KEEP(*(RTMSymTab)) + PROVIDE(__rtmsymtab_end = .); + + /* section information for initialization */ + . = ALIGN(RISCV_SZPTR); + PROVIDE(__rt_init_start = .); + KEEP(*(SORT(.rti_fn*))) + PROVIDE(__rt_init_end = .); + + /* section information for rt_ofw. */ + . = ALIGN(RISCV_SZPTR); + PROVIDE(__rt_ofw_data_start = .); + KEEP(*(SORT(.rt_ofw_data.*))) + PROVIDE(__rt_ofw_data_end = .); + . = ALIGN(RISCV_SZPTR); + + /* section information for usb usbh_class_info */ + . = ALIGN(RISCV_SZPTR); + __usbh_class_info_start__ = .; + KEEP(*(.usbh_class_info)) + . = ALIGN(RISCV_SZPTR); + __usbh_class_info_end__ = .; + + PROVIDE(__text_end = .); + } :text + + .eh_frame_hdr : + { + *(.eh_frame_hdr) + *(.eh_frame_entry) + } + .eh_frame : ONLY_IF_RO { KEEP (*(.eh_frame)) } + + . = ALIGN(RISCV_SZPTR); + .data : + { + *(.data) + *(.data.*) + + *(.data1) + *(.data1.*) + + . = ALIGN(ARCH_PAGE_SIZE); + PROVIDE(__global_pointer$ = . + 0x800); + + *(.sdata) + *(.sdata.*) + + *(.rel.local) + } :data + + . = ALIGN(RISCV_SZPTR); + .ctors : + { + PROVIDE(__ctors_start__ = .); + KEEP(*(SORT(.init_array.*))) + KEEP(*(.init_array)) + PROVIDE(__ctors_end__ = .); + } + + .dtors : + { + PROVIDE(__dtors_start__ = .); + KEEP(*(SORT(.fini_array.*))) + KEEP(*(.fini_array)) + PROVIDE(__dtors_end__ = .); + } + + .bss : + { + /* + * We need some free space to page, move .bss.noclean.* + * to optimize size. + */ + PROVIDE(__bss_noclean_start = .); + *(.bss.noclean.*) + PROVIDE(__bss_noclean_end = .); + . = ALIGN(RISCV_SZPTR); + PROVIDE(__bss_start = .); + *(.bss) + *(.bss.*) + *(.sbss) + *(.sbss.*) + *(.dynbss) + *(COMMON) + . = ALIGN(RISCV_SZPTR); + PROVIDE(__bss_end = .); + } + + .percpu (NOLOAD) : + { + #if defined(RT_USING_SMP) && defined(ARCH_MM_MMU) + /* Align for MMU early map */ + . = ALIGN(SUPPER_PAGE_SIZE); + #endif /* RT_USING_SMP && ARCH_MM_MMU */ + PROVIDE(__percpu_start = .); + *(.percpu) + #if defined(RT_USING_SMP) && defined(ARCH_MM_MMU) + . = ALIGN(ARCH_PAGE_SIZE); + #endif /* RT_USING_SMP && ARCH_MM_MMU */ + PROVIDE(__percpu_end = .); + + /* Clone the area */ + . = __percpu_end + (__percpu_end - __percpu_start) * (RT_CPUS_NR - 1); + #if defined(RT_USING_SMP) && defined(ARCH_MM_MMU) + /* Align for MMU early map */ + . = ALIGN(SUPPER_PAGE_SIZE); + #endif /* RT_USING_SMP && ARCH_MM_MMU */ + PROVIDE(__percpu_real_end = .); + + #if defined(RT_USING_SMP) && defined(ARCH_MM_MMU) + ASSERT(((__percpu_real_end - __percpu_start) / ARCH_PAGE_SIZE) <= (ARCH_PAGE_SIZE / RISCV_SZPTR), "Not enough L0 pages to map per-CPU"); + #endif /* RT_USING_SMP && ARCH_MM_MMU */ + } + + /* + * We should make the bootloader know the size of memory we need, + * so we MUST calc the image's size with section '.percpu'. + */ + _end = .; + + /* Stabs debugging sections. */ + .stab 0 : { *(.stab) } + .stabstr 0 : { *(.stabstr) } + .stab.excl 0 : { *(.stab.excl) } + .stab.exclstr 0 : { *(.stab.exclstr) } + .stab.index 0 : { *(.stab.index) } + .stab.indexstr 0 : { *(.stab.indexstr) } + .comment 0 : { *(.comment) } + /* DWARF debug sections. + * Symbols in the DWARF debugging sections are relative to the beginning + * of the section so we begin them at 0. */ + /* DWARF 1 */ + .debug 0 : { *(.debug) } + .line 0 : { *(.line) } + /* GNU DWARF 1 extensions */ + .debug_srcinfo 0 : { *(.debug_srcinfo) } + .debug_sfnames 0 : { *(.debug_sfnames) } + /* DWARF 1.1 and DWARF 2 */ + .debug_aranges 0 : { *(.debug_aranges) } + .debug_pubnames 0 : { *(.debug_pubnames) } + /* DWARF 2 */ + .debug_info 0 : { *(.debug_info .gnu.linkonce.wi.*) } + .debug_abbrev 0 : { *(.debug_abbrev) } + .debug_line 0 : { *(.debug_line) } + .debug_frame 0 : { *(.debug_frame) } + .debug_str 0 : { *(.debug_str) } + .debug_loc 0 : { *(.debug_loc) } + .debug_macinfo 0 : { *(.debug_macinfo) } + /* SGI/MIPS DWARF 2 extensions */ + .debug_weaknames 0 : { *(.debug_weaknames) } + .debug_funcnames 0 : { *(.debug_funcnames) } + .debug_typenames 0 : { *(.debug_typenames) } + .debug_varnames 0 : { *(.debug_varnames) } + + __data_size = SIZEOF(.data); + __bss_size = SIZEOF(.bss); +} diff --git a/libcpu/risc-v/t-head/SConscript b/libcpu/risc-v/t-head/SConscript new file mode 100644 index 0000000000..7c0e922f19 --- /dev/null +++ b/libcpu/risc-v/t-head/SConscript @@ -0,0 +1,16 @@ +# RT-Thread building script for component + +from building import * + +group = [] + +if not GetDepend('ARCH_MM_MMU') and not GetDepend('RT_USING_DM'): + Return('group') + +cwd = GetCurrentDir() +src = Glob('*.c') + Glob('*.cpp') + Glob('*_gcc.S') +CPPPATH = [cwd] + +group = DefineGroup('libcpu', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') diff --git a/libcpu/risc-v/t-head/cache.c b/libcpu/risc-v/t-head/cache.c new file mode 100644 index 0000000000..5fabe80ddf --- /dev/null +++ b/libcpu/risc-v/t-head/cache.c @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2021-01-29 lizhirui first version + * 2021-11-05 JasonHu add c906/c908 cache inst + * 2022-11-09 WangXiaoyao Support cache coherence operations; + * improve portability and make + * no assumption on undefined behavior + * 2025-04-26 GuEe-GUI Follow the specific and vendor + */ + +#include <rthw.h> +#include <rtcompiler.h> + +#include <cache.h> +#include <opcode.h> +#ifdef RT_USING_SMP +#include <smp_call.h> +#endif + +#define THEAD_CMD_OP(_op, _sync_op, _start, _size, _cachesize) \ + rt_ubase_t _block = (rt_ubase_t)_start & ~(_cachesize - 1UL); \ + rt_ubase_t _end = (rt_ubase_t)_start + _size; \ + for (; _block < _end; _block += _cachesize) \ + { \ + __asm__ volatile (_op(%0)::"r"(_block):"memory"); \ + } \ + __asm__ volatile (_sync_op:::"memory"); + +static void thead_icache_invalidate_local(void *data) +{ + rt_ubase_t *start_size = data; + rt_uint32_t cachesize = riscv_cache_get_cbom_block_size(); + + THEAD_CMD_OP(OPC_ICACHE_IVA, OPC_SYNC_IS, start_size[0], start_size[1], cachesize); +} + +static void thead_icache_invalidate(void *addr, int size) +{ + rt_ubase_t start_size[2] = { (rt_ubase_t)addr, size }; + + thead_icache_invalidate_local(start_size); + +#ifdef RT_USING_SMP + rt_smp_call_each_cpu(thead_icache_invalidate_local, start_size, SMP_CALL_WAIT_ALL); +#endif +} + +static void thead_icache_invalidate_all_local(void *data) +{ + RT_UNUSED(data); + + __asm__ volatile(OPC_ICACHE_IALL "\n\t" OPC_SYNC_I:::"memory"); +} + +static void thead_icache_invalidate_all(void) +{ + thead_icache_invalidate_all_local(RT_NULL); + +#ifdef RT_USING_SMP + rt_smp_call_each_cpu(thead_icache_invalidate_all_local, RT_NULL, SMP_CALL_WAIT_ALL); +#endif +} + +static void thead_dcache_invalidate(void *addr, int size) +{ + rt_uint32_t cachesize = riscv_cache_get_cbom_block_size(); + + THEAD_CMD_OP(OPC_DCACHE_IVA, OPC_SYNC_S, addr, size, cachesize); +} + +static void thead_dcache_clean(void *addr, int size) +{ + rt_uint32_t cachesize = riscv_cache_get_cbom_block_size(); + + THEAD_CMD_OP(OPC_DCACHE_CVA, OPC_SYNC_S, addr, size, cachesize); +} + +static void thead_dcache_flush(void *addr, int size) +{ + rt_uint32_t cachesize = riscv_cache_get_cbom_block_size(); + + THEAD_CMD_OP(OPC_DCACHE_CIVA, OPC_SYNC_S, addr, size, cachesize); +} + +/** + * ===================================================== + * Architecture Independent API + * ===================================================== + */ + +void rt_hw_cpu_icache_ops(int ops, void *addr, int size) +{ + if (ops == RT_HW_CACHE_INVALIDATE) + { + rt_hw_cpu_icache_invalidate(addr, size); + } +} + +void rt_hw_cpu_dcache_ops(int ops, void *addr, int size) +{ + if (ops == RT_HW_CACHE_FLUSH) + { + thead_dcache_clean(addr, size); + } + else + { + thead_dcache_invalidate(addr, size); + } +} + +void rt_hw_cpu_icache_invalidate_all(void) +{ + thead_icache_invalidate_all(); +} + +void rt_hw_cpu_icache_invalidate(void *addr, rt_size_t size) +{ + thead_icache_invalidate(addr, size); +} + +void rt_hw_cpu_dcache_clean_and_invalidate(void *addr, rt_size_t size) +{ + thead_dcache_flush(addr, size); +} + +void rt_hw_icache_invalidate_all(void) +{ + thead_icache_invalidate_all(); +} + +/** + * @brief Synchronize cache to Point of Unification + */ +void rt_hw_sync_cache_local(void *addr, int size) +{ + __asm__ volatile(OPC_DCACHE_CVAL1(a0)::: "memory"); + + thead_icache_invalidate(addr, size); +} diff --git a/libcpu/risc-v/t-head/mmu_vendor.h b/libcpu/risc-v/t-head/mmu_vendor.h new file mode 100644 index 0000000000..6b05cea3a2 --- /dev/null +++ b/libcpu/risc-v/t-head/mmu_vendor.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2021-01-30 lizhirui first version + * 2021-05-03 lizhirui porting to c906/c908 + * 2023-10-12 Shell Add permission control API + * 2025-04-26 GuEe-GUI Follow the specific and vendor + */ + +#ifndef __MMU_VENDOR_H__ +#define __MMU_VENDOR_H__ + +#include <asm-generic.h> + +/* Sv32 has no memory-type bits; use the common PMA defaults on RV32. */ +#if __riscv_xlen == 64 +/* C-SKY extend */ +#define __PTE_SEC BIT(59) /* Security */ +#define __PTE_SHARE BIT(60) /* Shareable */ +#define __PTE_BUF BIT(61) /* Bufferable */ +#define __PTE_CACHE BIT(62) /* Cacheable */ +#define __PTE_SO BIT(63) /* Strong Order */ + +#define PTE_VENDOR_PMA (__PTE_CACHE | __PTE_BUF | __PTE_SHARE) +#define PTE_VENDOR_NOCACHE (__PTE_BUF | __PTE_SHARE) +#define PTE_VENDOR_IO (__PTE_SO | __PTE_SHARE) +#endif /* __riscv_xlen == 64 */ + +#define MMU_VENDOR_MAP_K_RW (PTE_ATTR_KERNEL | PTE_G) +#define MMU_VENDOR_MAP_EARLY (MMU_VENDOR_MAP_K_RW | PTE_VENDOR_PMA) + +#endif /* __MMU_VENDOR_H__ */ diff --git a/libcpu/risc-v/t-head/opcode_vendor.h b/libcpu/risc-v/t-head/opcode_vendor.h new file mode 100644 index 0000000000..7fb201b750 --- /dev/null +++ b/libcpu/risc-v/t-head/opcode_vendor.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2006-2021, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-11-09 WangXiaoyao Add portable asm support + * 2022-03-16 WangXiaoyao Porting to xtheadsync & xtheadcmo ISA extension + */ + +#ifndef __VENDOR_OPCODE_H__ +#define __VENDOR_OPCODE_H__ + +/** + * @brief binary opcode pseudo operations + * Used to bypass toolchain restriction on extension ISA + * + * WARNING: Xuantie ISAs are not compatible to each other in opcode. + * It's painful to port this file, and should be really careful. + */ + +/** + * @brief RISC-V instruction formats + */ + +/** + * @brief Xuantie T-HEAD extension ISA format + * Compatible to Xuantie C906R2S1 user manual v06 and C908 user manual v03 + */ +#define __OPC_INSN_FORMAT_CACHE(func7, rs2, rs1) \ + __OPC_INSN_FORMAT_R(0x0b, 0x0, func7, x0, rs1, rs2) + +#ifdef ARCH_TOOLCHAIN_SUPP_XTHEADE_ISA +#define OPC_SYNC "sync" +#define OPC_SYNC_S "sync.s" +#define OPC_SYNC_I "sync.i" +#define OPC_SYNC_IS "sync.is" + +#define OPC_DCACHE_CALL "dcache.call" +#define OPC_DCACHE_IALL "dcache.iall" +#define OPC_DCACHE_CIALL "dcache.ciall" +#define OPC_DCACHE_CVAL1(rs1) "dcache.cval1 "_TOSTR(rs1) + +#define OPC_ICACHE_IALL "icache.iall" + +#define OPC_DCACHE_CVA(rs1) "dcache.cva "RT_STRINGIFY(rs1) +#define OPC_DCACHE_IVA(rs1) "dcache.iva "RT_STRINGIFY(rs1) +#define OPC_DCACHE_CIVA(rs1) "dcache.civa "RT_STRINGIFY(rs1) + +#define OPC_ICACHE_IVA(rs1) "icache.iva "RT_STRINGIFY(rs1) +#else /* !ARCH_TOOLCHAIN_NOT_SUPP_THEAD_ISA */ + +#define OPC_SYNC ".long 0x0180000B" +#define OPC_SYNC_S ".long 0x0190000B" +#define OPC_SYNC_I ".long 0x01A0000B" +#define OPC_SYNC_IS ".long 0x01B0000B" + +#define OPC_DCACHE_CALL ".long 0x0010000B" +#define OPC_DCACHE_IALL ".long 0x0020000B" +#define OPC_DCACHE_CIALL ".long 0x0030000B" +#define OPC_DCACHE_CVAL1(rs1) __OPC_INSN_FORMAT_CACHE(0x1, x4, rs1) + +#define OPC_ICACHE_IALL ".long 0x0100000B" + +#define OPC_DCACHE_CVA(rs1) __OPC_INSN_FORMAT_CACHE(0x1, x5, rs1) +#define OPC_DCACHE_IVA(rs1) __OPC_INSN_FORMAT_CACHE(0x1, x6, rs1) +#define OPC_DCACHE_CIVA(rs1) __OPC_INSN_FORMAT_CACHE(0x1, x7, rs1) + +#define OPC_ICACHE_IVA(rs1) __OPC_INSN_FORMAT_CACHE(0x1, x16, rs1) +#endif /* ARCH_TOOLCHAIN_NOT_SUPP_THEAD_ISA */ + +/** + * @brief RISC-V zifencei ISA + */ +#ifdef ARCH_TOOLCHAIN_SUPP_ZIFENCEI_ISA +#define OPC_FENCE_I "fence.i" +#else /* !ARCH_TOOLCHAIN_SUPP_ZIFENCEI_ISA */ +#define OPC_FENCE_I ".long 0x0000100F" +#endif /* ARCH_TOOLCHAIN_SUPP_ZIFENCEI_ISA */ + +#endif /* __VENDOR_OPCODE_H__ */ diff --git a/libcpu/risc-v/t-head/setup.c b/libcpu/risc-v/t-head/setup.c new file mode 100644 index 0000000000..6580a562aa --- /dev/null +++ b/libcpu/risc-v/t-head/setup.c @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2006-2025, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2025-04-20 GuEe-GUI the first version + */ + +#include <cache.h> +#include <isa_ext.h> + +rt_weak void fdt_thead_install_early(void *fdt) +{ + RT_UNUSED(fdt); +} + +void rt_hw_fdt_vendor_install_early(void *fdt) +{ + fdt_thead_install_early(fdt); + + riscv_isa_ext_set(ZICBOM); + riscv_cache_set_cbom_block_size(L1_CACHE_BYTES); +} diff --git a/libcpu/risc-v/vector/rvv-1.0/rvv_context.h b/libcpu/risc-v/vector/rvv-1.0/rvv_context.h index f5ec4d4ae8..fd66166d2d 100644 --- a/libcpu/risc-v/vector/rvv-1.0/rvv_context.h +++ b/libcpu/risc-v/vector/rvv-1.0/rvv_context.h @@ -11,15 +11,56 @@ #ifndef __RVV_CONTEXT_1_0_H__ #define __RVV_CONTEXT_1_0_H__ +#ifdef __ASSEMBLY__ +#include "vector_encoding.h" + +/* common64 uses the older REGBYTES/STORE/LOAD assembler ABI. */ +#ifndef SZREG +#define SZREG REGBYTES +#endif +#ifndef REG_S +#define REG_S STORE +#endif +#ifndef REG_L +#define REG_L LOAD +#endif + +/* The unified common supplies these through csr.h. */ +#ifndef CSR_VSTART +#define CSR_VSTART 0x008 +#endif +#ifndef CSR_VCSR +#define CSR_VCSR 0x00f +#endif +#ifndef CSR_VL +#define CSR_VL 0xc20 +#endif +#ifndef CSR_VTYPE +#define CSR_VTYPE 0xc21 +#endif +#ifndef CSR_VLENB +#define CSR_VLENB 0xc22 +#endif +#endif + #if defined(ARCH_VECTOR_VLEN_128) -#define CTX_VECTOR_REGS 64 + #define CTX_VECTOR_VLEN 128 #elif defined(ARCH_VECTOR_VLEN_256) -#define CTX_VECTOR_REGS 128 + #define CTX_VECTOR_VLEN 256 +#elif defined(ARCH_VECTOR_VLEN_512) + #define CTX_VECTOR_VLEN 512 +#elif defined(ARCH_VECTOR_VLEN_1024) + #define CTX_VECTOR_VLEN 1024 +#elif defined(ARCH_VECTOR_VLEN_2048) + #define CTX_VECTOR_VLEN 2048 +#elif defined(ARCH_VECTOR_VLEN_4096) + #define CTX_VECTOR_VLEN 4096 #else -#error "No supported VLEN" +#error "No RISC-V vector length configured" #endif /* VLEN */ -#define CTX_VECTOR_REG_NR (CTX_VECTOR_REGS + 4) +/* Number of XLEN-sized stack slots occupied by vector CSRs and registers. */ +#define CTX_VECTOR_REG_NR ((CTX_VECTOR_VLEN * 32 / __riscv_xlen) + 4) #ifdef __ASSEMBLY__ @@ -32,16 +73,16 @@ * ================================== */ -#define VEC_FRAME_VSTART (0 * REGBYTES) -#define VEC_FRAME_VTYPE (1 * REGBYTES) -#define VEC_FRAME_VL (2 * REGBYTES) -#define VEC_FRAME_VCSR (3 * REGBYTES) -#define VEC_FRAME_V0 (4 * REGBYTES) +#define VEC_FRAME_VSTART (0 * SZREG) +#define VEC_FRAME_VTYPE (1 * SZREG) +#define VEC_FRAME_VL (2 * SZREG) +#define VEC_FRAME_VCSR (3 * SZREG) +#define VEC_FRAME_V0 (4 * SZREG) .macro GET_VEC_FRAME_LEN, xreg - csrr \xreg, vlenb + csrr \xreg, CSR_VLENB slli \xreg, \xreg, 5 - addi \xreg, \xreg, 4 * REGBYTES + addi \xreg, \xreg, 4 * SZREG .endm /** @@ -53,28 +94,30 @@ .macro SAVE_VECTOR, dst mv t1, \dst - csrr t0, vstart - STORE t0, VEC_FRAME_VSTART(t1) - csrr t0, vtype - STORE t0, VEC_FRAME_VTYPE(t1) - csrr t0, vl - STORE t0, VEC_FRAME_VL(t1) - csrr t0, vcsr - STORE t0, VEC_FRAME_VCSR(t1) + csrr t0, CSR_VSTART + REG_S t0, VEC_FRAME_VSTART(t1) + csrr t0, CSR_VTYPE + REG_S t0, VEC_FRAME_VTYPE(t1) + csrr t0, CSR_VL + REG_S t0, VEC_FRAME_VL(t1) + csrr t0, CSR_VCSR + REG_S t0, VEC_FRAME_VCSR(t1) addi t1, t1, VEC_FRAME_V0 - // config vector setting, - // t2 is updated to length of a vector group in bytes - VEC_CONFIG_SETVLI(t2, x0, VEC_IMM_SEW_8, VEC_IMM_LMUL_8) + /* + * config vector setting, + * t2 is updated to length of a vector group in bytes + */ + VEC_INSN_VSETVLI_T2_X0_E8_M8 - vse8.v v0, (t1) + VEC_INSN_VSE8_V0_T1 add t1, t1, t2 - vse8.v v8, (t1) + VEC_INSN_VSE8_V8_T1 add t1, t1, t2 - vse8.v v16, (t1) + VEC_INSN_VSE8_V16_T1 add t1, t1, t2 - vse8.v v24, (t1) + VEC_INSN_VSE8_V24_T1 .endm /** @@ -84,35 +127,35 @@ * */ .macro RESTORE_VECTOR, dst - // restore vector registers first since it will modify vector states + /* Restore vector registers first since it will modify vector states */ mv t0, \dst addi t1, t0, VEC_FRAME_V0 - VEC_CONFIG_SETVLI(t2, x0, VEC_IMM_SEW_8, VEC_IMM_LMUL_8) + VEC_INSN_VSETVLI_T2_X0_E8_M8 - vle8.v v0, (t1) + VEC_INSN_VLE8_V0_T1 add t1, t1, t2 - vle8.v v8, (t1) + VEC_INSN_VLE8_V8_T1 add t1, t1, t2 - vle8.v v16, (t1) + VEC_INSN_VLE8_V16_T1 add t1, t1, t2 - vle8.v v24, (t1) + VEC_INSN_VLE8_V24_T1 mv t1, t0 - LOAD t0, VEC_FRAME_VCSR(t1) - csrw vcsr, t0 + REG_L t0, VEC_FRAME_VCSR(t1) + csrw CSR_VCSR, t0 - LOAD t0, VEC_FRAME_VTYPE(t1) - LOAD t3, VEC_FRAME_VL(t1) - VEC_CONFIG_SET_VL_VTYPE(t3, t0) + REG_L t0, VEC_FRAME_VTYPE(t1) + REG_L t3, VEC_FRAME_VL(t1) + VEC_INSN_VSETVL_X0_T3_T0 /* vsetvl resets vstart, so restore the restart index last. */ - LOAD t0, VEC_FRAME_VSTART(t1) - csrw vstart, t0 + REG_L t0, VEC_FRAME_VSTART(t1) + csrw CSR_VSTART, t0 .endm /* clang-format on */ -#endif +#endif /* __ASSEMBLY__ */ #endif /* __RVV_CONTEXT_H__ */ diff --git a/libcpu/risc-v/vector/rvv-1.0/vector_encoding.h b/libcpu/risc-v/vector/rvv-1.0/vector_encoding.h index 022beb0603..cb19b21d05 100644 --- a/libcpu/risc-v/vector/rvv-1.0/vector_encoding.h +++ b/libcpu/risc-v/vector/rvv-1.0/vector_encoding.h @@ -25,18 +25,18 @@ * assembler names used for vset{i}vli vtypei immediate */ -#define VEC_IMM_SEW_8 e8 -#define VEC_IMM_SEW_16 e16 -#define VEC_IMM_SEW_32 e32 -#define VEC_IMM_SEW_64 e64 +#define VEC_IMM_SEW_8 e8 +#define VEC_IMM_SEW_16 e16 +#define VEC_IMM_SEW_32 e32 +#define VEC_IMM_SEW_64 e64 /* group setting, encoding by multiplier */ -#define VEC_IMM_LMUL_F8 mf8 -#define VEC_IMM_LMUL_F4 mf4 -#define VEC_IMM_LMUL_F2 mf2 -#define VEC_IMM_LMUL_1 m1 -#define VEC_IMM_LMUL_2 m2 -#define VEC_IMM_LMUL_4 m4 -#define VEC_IMM_LMUL_8 m8 +#define VEC_IMM_LMUL_F8 mf8 +#define VEC_IMM_LMUL_F4 mf4 +#define VEC_IMM_LMUL_F2 mf2 +#define VEC_IMM_LMUL_1 m1 +#define VEC_IMM_LMUL_2 m2 +#define VEC_IMM_LMUL_4 m4 +#define VEC_IMM_LMUL_8 m8 /* TAIL & MASK agnostic bits */ #define VEC_IMM_TAIL_AGNOSTIC ta #define VEC_IMM_MASK_AGNOSTIC ma @@ -47,9 +47,28 @@ /** * configuration setting instruction */ -#define VEC_CONFIG_SETVLI(xVl, xAvl, vtype...) vsetvli xVl, xAvl, ##vtype -#define VEC_CONFIG_SET_VL_VTYPE(xVl, xVtype) vsetvl x0, xVl, xVtype +#define VEC_CONFIG_SETVLI(xVl, xAvl, vtype...) vsetvli xVl, xAvl, ##vtype +#define VEC_CONFIG_SET_VL_VTYPE(xVl, xVtype) vsetvl x0, xVl, xVtype -#endif +/* + * Some maintained RISC-V GCC 10 based toolchains can build RT-Thread but use + * binutils without RVV mnemonic support. Keep the fixed encodings used by the + * context save/restore path here so ARCH_RISCV_VECTOR does not require a newer + * assembler just to build the trap entry. + */ +#define VEC_INSN_VSETVLI_T2_X0_E8_M8 .word 0x003073d7 +#define VEC_INSN_VSETVL_X0_T3_T0 .word 0x805e7057 + +#define VEC_INSN_VLE8_V0_T1 .word 0x02030007 +#define VEC_INSN_VLE8_V8_T1 .word 0x02030407 +#define VEC_INSN_VLE8_V16_T1 .word 0x02030807 +#define VEC_INSN_VLE8_V24_T1 .word 0x02030c07 + +#define VEC_INSN_VSE8_V0_T1 .word 0x02030027 +#define VEC_INSN_VSE8_V8_T1 .word 0x02030427 +#define VEC_INSN_VSE8_V16_T1 .word 0x02030827 +#define VEC_INSN_VSE8_V24_T1 .word 0x02030c27 + +#endif /* __ASSEMBLY__ */ #endif /* __VECTOR_ENCODING_H__ */ |