🎈 perf: perf rt_hw_interrupt_disable/enable (#8042)

Signed-off-by: Shell <[email protected]>
Co-authored-by: Shell <[email protected]>

5 files changed, 129 insertions, 48 deletions

diff --git a/libcpu/aarch64/common/context_gcc.S b/libcpu/aarch64/common/context_gcc.S
index 1e36cbc9f4..c671f9ba14 100644
--- a/libcpu/aarch64/common/context_gcc.S
+++ b/libcpu/aarch64/common/context_gcc.S
@@ -9,10 +9,14 @@
  * 2023-06-24     WangXiaoyao  Support backtrace for user thread
  */
 
+#ifndef __ASSEMBLY__
+#define __ASSEMBLY__
+#endif
+
 #include "rtconfig.h"
 #include "asm-generic.h"
-
 #include "asm-fpu.h"
+#include "armv8.h"
 
 .text
 .weak rt_hw_cpu_id_set
@@ -279,12 +283,30 @@ START_POINT_END(_thread_start)
 1:
 .endm
 
+.macro RESTORE_USER_CTX, ctx
+    LDR     X1, [\ctx, #CONTEXT_OFFSET_SPSR_EL1]
+    AND     X1, X1, 0x1f
+    CMP     X1, XZR
+
+    BNE     1f
+    BL      lwp_uthread_ctx_restore
+1:
+.endm
+
 #ifdef RT_USING_SMP
 #define rt_hw_interrupt_disable rt_hw_local_irq_disable
 #define rt_hw_interrupt_enable rt_hw_local_irq_enable
 #endif
 
 .text
+
+.global rt_hw_interrupt_is_disabled
+rt_hw_interrupt_is_disabled:
+    MRS     X0, DAIF
+    TST     X0, #0xc0
+    CSET    X0, NE
+    RET
+
 /*
  * rt_base_t rt_hw_interrupt_disable();
  */
@@ -387,8 +409,7 @@ rt_hw_context_switch_interrupt:
 vector_fiq:
     B       .
 
-.globl vector_irq
-vector_irq:
+START_POINT(vector_irq)
     SAVE_CONTEXT
     STP     X0, X1, [SP, #-0x10]!   /* X0 is thread sp */
 
@@ -399,13 +420,15 @@ vector_irq:
 #endif
     BL      rt_hw_trap_irq
 #ifdef RT_USING_LWP
-    BL      lwp_uthread_ctx_restore
+    LDP     X0, X1, [SP]
+    RESTORE_USER_CTX X0
 #endif
     BL      rt_interrupt_leave
 
     LDP     X0, X1, [SP], #0x10
     BL      rt_scheduler_do_irq_switch
     B       rt_hw_context_switch_exit
+START_POINT_END(vector_irq)
 
 .global rt_hw_context_switch_exit
 rt_hw_context_switch_exit:
@@ -540,7 +563,8 @@ START_POINT(vector_exception)
 
     BL      rt_hw_trap_exception
 #ifdef RT_USING_LWP
-    BL      lwp_uthread_ctx_restore
+    LDP     X0, X1, [SP]
+    RESTORE_USER_CTX X0
 #endif
 
     LDP     X0, X1, [SP], #0x10
diff --git a/libcpu/aarch64/common/cpu.c b/libcpu/aarch64/common/cpu.c
index 7aeb59f634..5dbb45af71 100644
--- a/libcpu/aarch64/common/cpu.c
+++ b/libcpu/aarch64/common/cpu.c
@@ -59,52 +59,65 @@ rt_weak rt_uint64_t rt_cpu_mpidr_early[] =
 };
 #endif /* RT_USING_SMART */
 
-void rt_hw_spin_lock_init(rt_hw_spinlock_t *lock)
+typedef rt_hw_spinlock_t arch_spinlock_t;
+
+static inline void arch_spin_lock(arch_spinlock_t *lock)
 {
-    lock->slock = 0;
+    unsigned int tmp;
+
+    asm volatile(
+    "   sevl\n"
+    "1: wfe\n"
+    "2: ldaxr   %w0, %1\n"
+    "   cbnz    %w0, 1b\n"
+    "   stxr    %w0, %w2, %1\n"
+    "   cbnz    %w0, 2b\n"
+    : "=&r" (tmp), "+Q" (lock->lock)
+    : "r" (1)
+    : "cc", "memory");
 }
 
-#define TICKET_SHIFT 16
-void rt_hw_spin_lock(rt_hw_spinlock_t *lock)
+static inline int arch_spin_trylock(arch_spinlock_t *lock)
 {
     unsigned int tmp;
-    struct __arch_tickets lockval, newval;
 
     asm volatile(
-        /* Atomically increment the next ticket. */
-        "   prfm    pstl1strm, %3\n"
-        "1: ldaxr   %w0, %3\n"
-        "   add %w1, %w0, %w5\n"
-        "   stxr    %w2, %w1, %3\n"
-        "   cbnz    %w2, 1b\n"
-        /* Did we get the lock? */
-        "   eor %w1, %w0, %w0, ror #16\n"
-        "   cbz %w1, 3f\n"
-        /*
-         * No: spin on the owner. Send a local event to avoid missing an
-         * unlock before the exclusive load.
-         */
-        "   sevl\n"
-        "2: wfe\n"
-        "   ldaxrh  %w2, %4\n"
-        "   eor %w1, %w2, %w0, lsr #16\n"
-        "   cbnz    %w1, 2b\n"
-        /* We got the lock. Critical section starts here. */
-        "3:"
-        : "=&r"(lockval), "=&r"(newval), "=&r"(tmp), "+Q"(*lock)
-        : "Q"(lock->tickets.owner), "I"(1 << TICKET_SHIFT)
-        : "memory");
-    rt_hw_dmb();
+    "  ldaxr   %w0, %1\n"
+    "  cbnz    %w0, 1f\n"
+    "  stxr    %w0, %w2, %1\n"
+    "1:\n"
+    : "=&r" (tmp), "+Q" (lock->lock)
+    : "r" (1)
+    : "cc", "memory");
+
+    return !tmp;
 }
 
-void rt_hw_spin_unlock(rt_hw_spinlock_t *lock)
+static inline void arch_spin_unlock(arch_spinlock_t *lock)
 {
-    rt_hw_dmb();
     asm volatile(
-        "   stlrh   %w1, %0\n"
-        : "=Q"(lock->tickets.owner)
-        : "r"(lock->tickets.owner + 1)
-        : "memory");
+    " stlr    %w1, %0\n"
+    : "=Q" (lock->lock) : "r" (0) : "memory");
+}
+
+void rt_hw_spin_lock_init(arch_spinlock_t *lock)
+{
+    lock->lock = 0;
+}
+
+void rt_hw_spin_lock(rt_hw_spinlock_t *lock)
+{
+    arch_spin_lock(lock);
+}
+
+void rt_hw_spin_unlock(rt_hw_spinlock_t *lock)
+{
+    arch_spin_unlock(lock);
+}
+
+rt_bool_t rt_hw_spin_trylock(rt_hw_spinlock_t *lock)
+{
+    return arch_spin_trylock(lock);
 }
 
 static int _cpus_init_data_hardcoded(int num_cpus, rt_uint64_t *cpu_hw_ids, struct cpu_ops_t *cpu_ops[])
diff --git a/libcpu/aarch64/common/cpuport.h b/libcpu/aarch64/common/cpuport.h
index 2fe6afb2b9..00618fdfa0 100644
--- a/libcpu/aarch64/common/cpuport.h
+++ b/libcpu/aarch64/common/cpuport.h
@@ -14,12 +14,8 @@
 #include <rtdef.h>
 
 #ifdef RT_USING_SMP
-typedef union {
-    unsigned long slock;
-    struct __arch_tickets {
-        unsigned short owner;
-        unsigned short next;
-    } tickets;
+typedef struct {
+    volatile unsigned int lock;
 } rt_hw_spinlock_t;
 #endif
 
diff --git a/libcpu/aarch64/common/interrupt.c b/libcpu/aarch64/common/interrupt.c
index 7a93bfb250..8a5ed7bdd9 100644
--- a/libcpu/aarch64/common/interrupt.c
+++ b/libcpu/aarch64/common/interrupt.c
@@ -38,7 +38,7 @@ extern void *system_vectors;
 #ifdef RT_USING_SMP
 #define rt_interrupt_nest rt_cpu_self()->irq_nest
 #else
-extern volatile rt_uint8_t rt_interrupt_nest;
+extern volatile rt_atomic_t rt_interrupt_nest;
 #endif
 
 #ifdef SOC_BCM283x
@@ -85,7 +85,7 @@ void rt_hw_interrupt_init(void)
     }
 
     /* init interrupt nest, and context in thread sp */
-    rt_interrupt_nest = 0;
+    rt_atomic_store(&rt_interrupt_nest, 0);
     rt_interrupt_from_thread = 0;
     rt_interrupt_to_thread = 0;
     rt_thread_switch_interrupt_flag = 0;
@@ -415,3 +415,48 @@ void rt_hw_ipi_handler_install(int ipi_vector, rt_isr_handler_t ipi_isr_handler)
     rt_hw_interrupt_install(ipi_vector, ipi_isr_handler, 0, "IPI_HANDLER");
 }
 #endif
+
+#if defined(FINSH_USING_MSH) && defined(RT_USING_INTERRUPT_INFO)
+int list_isr()
+{
+    int idx;
+
+    rt_kprintf("%-*.*s nr   handler            param              counter         ", RT_NAME_MAX, RT_NAME_MAX, "irq");
+#ifdef RT_USING_SMP
+    for (int i = 0; i < RT_CPUS_NR; i++)
+    {
+        rt_kprintf(" cpu%2d  ", i);
+    }
+#endif
+    rt_kprintf("\n");
+    for (int i = 0; i < RT_NAME_MAX; i++)
+    {
+        rt_kprintf("-");
+    }
+    rt_kprintf(" ---- ------------------ ------------------ ----------------");
+#ifdef RT_USING_SMP
+    for (int i = 0; i < RT_CPUS_NR; i++)
+    {
+        rt_kprintf(" -------");
+    }
+#endif
+    rt_kprintf("\n");
+    for (idx = 0; idx < MAX_HANDLERS; idx++)
+    {
+        if (isr_table[idx].handler != RT_NULL)
+        {
+            rt_kprintf("%*.s %4d %p %p %16d", RT_NAME_MAX, isr_table[idx].name, idx, isr_table[idx].handler,
+                       isr_table[idx].param, isr_table[idx].counter);
+#ifdef RT_USING_SMP
+            for (int i = 0; i < RT_CPUS_NR; i++)
+                 rt_kprintf(" %7d", isr_table[idx].cpu_counter[i]);
+#endif
+            rt_kprintf("\n");
+        }
+    }
+    return 0;
+}
+
+#include "finsh.h"
+MSH_CMD_EXPORT(list_isr, list isr)
+#endif
diff --git a/libcpu/aarch64/common/trap.c b/libcpu/aarch64/common/trap.c
index e3e4bb43eb..bac4ceaf21 100644
--- a/libcpu/aarch64/common/trap.c
+++ b/libcpu/aarch64/common/trap.c
@@ -274,6 +274,9 @@ void rt_hw_trap_irq(void)
     isr_func = isr_table[ir_self].handler;
 #ifdef RT_USING_INTERRUPT_INFO
     isr_table[ir_self].counter++;
+#ifdef RT_USING_SMP
+    isr_table[ir_self].cpu_counter[rt_hw_cpu_id()]++;
+#endif
 #endif
     if (isr_func)
     {