🎈 perf: perf rt_hw_interrupt_disable/enable (#8042)

Signed-off-by: Shell <[email protected]>
Co-authored-by: Shell <[email protected]>

4 files changed, 100 insertions, 73 deletions

diff --git a/include/rtatomic.h b/include/rtatomic.h
index 9f361fd925..5363ee5426 100644
--- a/include/rtatomic.h
+++ b/include/rtatomic.h
@@ -27,13 +27,6 @@ void rt_hw_atomic_flag_clear(volatile rt_atomic_t *ptr);
 rt_atomic_t rt_hw_atomic_flag_test_and_set(volatile rt_atomic_t *ptr);
 rt_atomic_t rt_hw_atomic_compare_exchange_strong(volatile rt_atomic_t *ptr, rt_atomic_t *expected, rt_atomic_t desired);
 
-/* To detect stdatomic */
-#if !defined(RT_USING_HW_ATOMIC) && !defined(RT_USING_STDC_ATOMIC)
-#if defined(__GNUC__) && defined(RT_USING_LIBC) && !defined(__STDC_NO_ATOMICS__)
-#define RT_USING_STDC_ATOMIC
-#endif /* __GNUC__ && .. */
-#endif /* !RT_USING_HW_ATOMIC && !RT_USING_STDC_ATOMIC */
-
 #if defined(RT_USING_HW_ATOMIC)
 #define rt_atomic_load(ptr) rt_hw_atomic_load(ptr)
 #define rt_atomic_store(ptr, v) rt_hw_atomic_store(ptr, v)
diff --git a/include/rtdef.h b/include/rtdef.h
index b4047d3cdf..e9afa06ff0 100644
--- a/include/rtdef.h
+++ b/include/rtdef.h
@@ -49,6 +49,7 @@
  * 2022-12-20     Meco Man     add const name for rt_object
  * 2023-04-01     Chushicheng  change version number to v5.0.1
  * 2023-05-20     Bernard      add stdc atomic detection.
+ * 2023-09-15     xqyjlj       perf rt_hw_interrupt_disable/enable
  * 2023-09-17     Meco Man     add RT_USING_LIBC_ISO_ONLY macro
  * 2023-10-10     Chushicheng  change version number to v5.1.0
  * 2023-10-11     zmshahaha    move specific devices related and driver to components/drivers
@@ -146,22 +147,16 @@ typedef rt_base_t                       rt_flag_t;      /**< Type for flags */
 typedef rt_ubase_t                      rt_dev_t;       /**< Type for device */
 typedef rt_base_t                       rt_off_t;       /**< Type for offset */
 
-#if !defined(__cplusplus)
+#ifdef __cplusplus
+    typedef rt_base_t rt_atomic_t;
+#else
 #if defined(RT_USING_HW_ATOMIC)
     typedef rt_base_t rt_atomic_t;
 #elif defined(RT_USING_STDC_ATOMIC)
     #include <stdatomic.h>
     typedef atomic_size_t rt_atomic_t;
 #else
-
-    /* To detect std atomic */
-    #if defined(RT_USING_LIBC) && defined(__GNUC__) && !defined(__STDC_NO_ATOMICS__)
-        #include <stdatomic.h>
-        typedef atomic_size_t rt_atomic_t;
-    #else
-        typedef rt_base_t rt_atomic_t;
-    #endif /* __GNUC__ && !__STDC_NO_ATOMICS__ */
-
+    typedef rt_base_t rt_atomic_t;
 #endif /* RT_USING_STDC_ATOMIC */
 #endif /* __cplusplus */
 
@@ -179,10 +174,12 @@ typedef rt_base_t                       rt_off_t;       /**< Type for offset */
 #define RT_UINT8_MAX                    UINT8_MAX       /**< Maximum number of UINT8 */
 #define RT_UINT16_MAX                   UINT16_MAX      /**< Maximum number of UINT16 */
 #define RT_UINT32_MAX                   UINT32_MAX      /**< Maximum number of UINT32 */
+#define RT_UINT64_MAX                   UINT64_MAX      /**< Maximum number of UINT64 */
 #else
 #define RT_UINT8_MAX                    0xff            /**< Maximum number of UINT8 */
 #define RT_UINT16_MAX                   0xffff          /**< Maximum number of UINT16 */
 #define RT_UINT32_MAX                   0xffffffff      /**< Maximum number of UINT32 */
+#define RT_UINT64_MAX                   0xffffffffffffffff
 #endif /* RT_USING_LIBC */
 
 #define RT_TICK_MAX                     RT_UINT32_MAX   /**< Maximum number of tick */
@@ -492,6 +489,34 @@ struct rt_slist_node
 };
 typedef struct rt_slist_node rt_slist_t;                /**< Type for single list. */
 
+#ifdef RT_USING_SMP
+#include <cpuport.h> /* for spinlock from arch */
+
+struct rt_spinlock
+{
+    rt_hw_spinlock_t lock;
+#if defined(RT_DEBUGING_SPINLOCK)
+    void *owner;
+    void *pc;
+#endif /* RT_DEBUGING_SPINLOCK */
+};
+typedef struct rt_spinlock rt_spinlock_t;
+
+#ifndef RT_SPINLOCK_INIT
+#define RT_SPINLOCK_INIT {{0}} // default
+#endif /* RT_SPINLOCK_INIT */
+
+#else
+typedef rt_ubase_t rt_spinlock_t;
+struct rt_spinlock
+{
+    rt_spinlock_t lock;
+};
+#define RT_SPINLOCK_INIT {0}
+#endif /* RT_USING_SMP */
+
+#define RT_DEFINE_SPINLOCK(x)  struct rt_spinlock x = RT_SPINLOCK_INIT
+
 /**
  * @addtogroup KernelObject
  */
@@ -521,7 +546,7 @@ struct rt_object
 #endif /* RT_USING_MODULE */
 
 #ifdef RT_USING_SMART
-    int         lwp_ref_count;                           /**< ref count for lwp */
+    rt_atomic_t lwp_ref_count;                           /**< ref count for lwp */
 #endif /* RT_USING_SMART */
 
     rt_list_t   list;                                    /**< list node of kernel object */
@@ -574,6 +599,7 @@ struct rt_object_information
     enum rt_object_class_type type;                     /**< object class type */
     rt_list_t                 object_list;              /**< object list */
     rt_size_t                 object_size;              /**< object size */
+    struct rt_spinlock        spinlock;
 };
 
 /**
@@ -757,23 +783,25 @@ typedef struct rt_cpu_usage_stats *rt_cpu_usage_stats_t;
  */
 struct rt_cpu
 {
-    struct rt_thread *current_thread;
-    struct rt_thread *idle_thread;
-    rt_uint16_t irq_nest;
-    rt_uint8_t  irq_switch_flag;
+    struct rt_thread            *current_thread;
+    struct rt_thread            *idle_thread;
+    rt_atomic_t                 irq_nest;
+    rt_uint8_t                  irq_switch_flag;
 
-    rt_uint8_t current_priority;
-    rt_list_t priority_table[RT_THREAD_PRIORITY_MAX];
+    rt_uint8_t                  current_priority;
+    rt_list_t                   priority_table[RT_THREAD_PRIORITY_MAX];
 #if RT_THREAD_PRIORITY_MAX > 32
-    rt_uint32_t priority_group;
-    rt_uint8_t ready_table[32];
+    rt_uint32_t                 priority_group;
+    rt_uint8_t                  ready_table[32];
 #else
-    rt_uint32_t priority_group;
+    rt_uint32_t                 priority_group;
 #endif /* RT_THREAD_PRIORITY_MAX > 32 */
 
-    rt_tick_t tick;
+    rt_atomic_t                 tick;
+
+    struct rt_spinlock          spinlock;
 #ifdef RT_USING_SMART
-    struct rt_cpu_usage_stats cpu_stat;
+    struct rt_cpu_usage_stats   cpu_stat;
 #endif
 };
 typedef struct rt_cpu *rt_cpu_t;
@@ -865,6 +893,7 @@ struct rt_thread
 {
     struct rt_object            parent;
     rt_list_t                   tlist;                  /**< the thread list */
+    rt_list_t                   tlist_schedule;         /**< the thread list */
 
     /* stack point and entry */
     void                        *sp;                    /**< stack point */
@@ -882,9 +911,8 @@ struct rt_thread
     rt_uint8_t                  bind_cpu;               /**< thread is bind to cpu */
     rt_uint8_t                  oncpu;                  /**< process on cpu */
 
-    rt_uint16_t                 scheduler_lock_nest;    /**< scheduler lock count */
-    rt_int16_t                  cpus_lock_nest;         /**< cpus lock count */
-    rt_uint16_t                 critical_lock_nest;     /**< critical lock count */
+    rt_atomic_t                 cpus_lock_nest;         /**< cpus lock count */
+    rt_atomic_t                 critical_lock_nest;     /**< critical lock count */
 #endif /*RT_USING_SMP*/
 
     /* priority */
@@ -919,8 +947,8 @@ struct rt_thread
     void                        *si_list;               /**< the signal infor list */
 #endif /* RT_USING_SIGNALS */
 
-    rt_ubase_t                  init_tick;              /**< thread's initialized tick */
-    rt_ubase_t                  remaining_tick;         /**< remaining tick */
+    rt_atomic_t                 init_tick;              /**< thread's initialized tick */
+    rt_atomic_t                 remaining_tick;         /**< remaining tick */
 
 #ifdef RT_USING_CPU_USAGE
     rt_uint64_t                 duration_tick;          /**< cpu usage tick */
@@ -949,8 +977,10 @@ struct rt_thread
     struct lwp_thread_signal    signal;                 /**< lwp signal for user-space thread */
     struct rt_user_context      user_ctx;               /**< user space context */
     struct rt_wakeup            wakeup;                 /**< wakeup data */
-    int                         exit_request;
-    int                         tid;
+    int                         exit_request;           /**< pending exit request of thread */
+    int                         tid;                    /**< thread ID used by process */
+    int                         tid_ref_count;          /**< reference of tid */
+    void                        *susp_recycler;         /**< suspended recycler on this thread */
 
     rt_uint64_t                 user_time;
     rt_uint64_t                 system_time;
@@ -967,6 +997,8 @@ struct rt_thread
     int                         *clear_child_tid;
 #endif /* ARCH_MM_MMU */
 #endif /* RT_USING_SMART */
+    rt_atomic_t                 ref_count;
+    struct rt_spinlock          spinlock;
     rt_ubase_t                  user_data;              /**< private user data beyond this thread */
 };
 typedef struct rt_thread *rt_thread_t;
@@ -977,6 +1009,11 @@ typedef struct rt_thread *rt_thread_t;
 
 /**@}*/
 
+#define rt_atomic_inc(v)                rt_atomic_add((v), 1)
+#define rt_atomic_dec(v)                rt_atomic_sub((v), 1)
+#define rt_get_thread_struct(object)    do { rt_atomic_inc(&(object)->ref_count); } while(0)
+#define rt_put_thread_struct(object)    do { rt_atomic_dec(&(object)->ref_count); } while(0)
+
 /**
  * @addtogroup IPC
  */
@@ -1016,6 +1053,7 @@ struct rt_semaphore
 
     rt_uint16_t          value;                         /**< value of semaphore. */
     rt_uint16_t          reserved;                      /**< reserved field */
+    struct rt_spinlock   spinlock;
 };
 typedef struct rt_semaphore *rt_sem_t;
 #endif /* RT_USING_SEMAPHORE */
@@ -1035,6 +1073,7 @@ struct rt_mutex
 
     struct rt_thread    *owner;                         /**< current owner of mutex */
     rt_list_t            taken_list;                    /**< the object list taken by thread */
+    struct rt_spinlock   spinlock;
 };
 typedef struct rt_mutex *rt_mutex_t;
 #endif /* RT_USING_MUTEX */
@@ -1055,6 +1094,7 @@ struct rt_event
     struct rt_ipc_object parent;                        /**< inherit from ipc_object */
 
     rt_uint32_t          set;                           /**< event set */
+    struct rt_spinlock   spinlock;
 };
 typedef struct rt_event *rt_event_t;
 #endif /* RT_USING_EVENT */
@@ -1076,6 +1116,7 @@ struct rt_mailbox
     rt_uint16_t          out_offset;                    /**< output offset of the message buffer */
 
     rt_list_t            suspend_sender_thread;         /**< sender thread suspended on this mailbox */
+    struct rt_spinlock   spinlock;
 };
 typedef struct rt_mailbox *rt_mailbox_t;
 #endif /* RT_USING_MAILBOX */
@@ -1100,6 +1141,7 @@ struct rt_messagequeue
     void                *msg_queue_free;                /**< pointer indicated the free node of queue */
 
     rt_list_t            suspend_sender_thread;         /**< sender thread suspended on this message queue */
+    struct rt_spinlock   spinlock;
 };
 typedef struct rt_messagequeue *rt_mq_t;
 #endif /* RT_USING_MESSAGEQUEUE */
@@ -1189,18 +1231,19 @@ struct rt_memheap
  */
 struct rt_mempool
 {
-    struct rt_object parent;                            /**< inherit from rt_object */
+    struct rt_object    parent;                            /**< inherit from rt_object */
 
-    void            *start_address;                     /**< memory pool start */
-    rt_size_t        size;                              /**< size of memory pool */
+    void                *start_address;                     /**< memory pool start */
+    rt_size_t           size;                              /**< size of memory pool */
 
-    rt_size_t        block_size;                        /**< size of memory blocks */
-    rt_uint8_t      *block_list;                        /**< memory blocks list */
+    rt_size_t           block_size;                        /**< size of memory blocks */
+    rt_uint8_t          *block_list;                        /**< memory blocks list */
 
-    rt_size_t        block_total_count;                 /**< numbers of memory block */
-    rt_size_t        block_free_count;                  /**< numbers of free memory block */
+    rt_size_t           block_total_count;                 /**< numbers of memory block */
+    rt_size_t           block_free_count;                  /**< numbers of free memory block */
 
-    rt_list_t        suspend_thread;                    /**< threads pended on this resource */
+    rt_list_t           suspend_thread;                    /**< threads pended on this resource */
+    struct rt_spinlock  spinlock;
 };
 typedef struct rt_mempool *rt_mp_t;
 #endif /* RT_USING_MEMPOOL */
@@ -1327,6 +1370,7 @@ struct rt_wqueue
 {
     rt_uint32_t flag;
     rt_list_t waiting_list;
+    struct rt_spinlock spinlock;
 };
 typedef struct rt_wqueue rt_wqueue_t;
 
diff --git a/include/rthw.h b/include/rthw.h
index 65764d5635..6e7773805a 100644
--- a/include/rthw.h
+++ b/include/rthw.h
@@ -13,6 +13,7 @@
  * 2018-11-17     Jesven       add rt_hw_spinlock_t
  *                             add smp support
  * 2019-05-18     Bernard      add empty definition for not enable cache case
+ * 2023-09-15     xqyjlj       perf rt_hw_interrupt_disable/enable
  * 2023-10-16     Shell        Support a new backtrace framework
  */
 
@@ -111,6 +112,9 @@ struct rt_irq_desc
 #ifdef RT_USING_INTERRUPT_INFO
     char             name[RT_NAME_MAX];
     rt_uint32_t      counter;
+#ifdef RT_USING_SMP
+    rt_ubase_t       cpu_counter[RT_CPUS_NR];
+#endif
 #endif
 };
 
@@ -129,11 +133,18 @@ rt_isr_handler_t rt_hw_interrupt_install(int              vector,
 rt_base_t rt_hw_local_irq_disable();
 void rt_hw_local_irq_enable(rt_base_t level);
 
+rt_base_t rt_cpus_lock(void);
+void rt_cpus_unlock(rt_base_t level);
+
 #define rt_hw_interrupt_disable rt_cpus_lock
 #define rt_hw_interrupt_enable rt_cpus_unlock
 #else
 rt_base_t rt_hw_interrupt_disable(void);
 void rt_hw_interrupt_enable(rt_base_t level);
+
+#define rt_hw_local_irq_disable rt_hw_interrupt_disable
+#define rt_hw_local_irq_enable rt_hw_interrupt_enable
+
 #endif /*RT_USING_SMP*/
 rt_bool_t rt_hw_interrupt_is_disabled(void);
 
@@ -187,25 +198,18 @@ void rt_hw_ipi_send(int ipi_vector, unsigned int cpu_mask);
 
 #ifdef RT_USING_SMP
 
-struct rt_spinlock
-{
-    rt_hw_spinlock_t lock;
-};
-
 void rt_hw_spin_lock_init(rt_hw_spinlock_t *lock);
 void rt_hw_spin_lock(rt_hw_spinlock_t *lock);
 void rt_hw_spin_unlock(rt_hw_spinlock_t *lock);
 
 extern rt_hw_spinlock_t _cpus_lock;
-extern rt_hw_spinlock_t _rt_critical_lock;
 
 #define __RT_HW_SPIN_LOCK_INITIALIZER(lockname) {0}
 
 #define __RT_HW_SPIN_LOCK_UNLOCKED(lockname)    \
     (rt_hw_spinlock_t) __RT_HW_SPIN_LOCK_INITIALIZER(lockname)
 
-#define RT_DEFINE_SPINLOCK(x)  rt_hw_spinlock_t x = __RT_HW_SPIN_LOCK_UNLOCKED(x)
-#define RT_DECLARE_SPINLOCK(x)
+#define RT_DEFINE_HW_SPINLOCK(x)  rt_hw_spinlock_t x = __RT_HW_SPIN_LOCK_UNLOCKED(x)
 
 /**
  * boot secondary cpu
@@ -218,17 +222,12 @@ void rt_hw_secondary_cpu_up(void);
 void rt_hw_secondary_cpu_idle_exec(void);
 #else
 
-#define RT_DEFINE_SPINLOCK(x)    rt_ubase_t x
-#define RT_DECLARE_SPINLOCK(x)
+#define RT_DEFINE_HW_SPINLOCK(x)    rt_ubase_t x
 
 #define rt_hw_spin_lock(lock)     *(lock) = rt_hw_interrupt_disable()
 #define rt_hw_spin_unlock(lock)   rt_hw_interrupt_enable(*(lock))
 
-typedef rt_ubase_t rt_spinlock_t;
-struct rt_spinlock
-{
-    rt_spinlock_t lock;
-};
+
 #endif
 
 #ifndef RT_USING_CACHE
diff --git a/include/rtthread.h b/include/rtthread.h
index f693b4f856..47d03119eb 100644
--- a/include/rtthread.h
+++ b/include/rtthread.h
@@ -553,11 +553,11 @@ void rt_spin_unlock(struct rt_spinlock *lock);
 rt_base_t rt_spin_lock_irqsave(struct rt_spinlock *lock);
 void rt_spin_unlock_irqrestore(struct rt_spinlock *lock, rt_base_t level);
 #else
-#define rt_spin_lock_init(lock)                 /* nothing */
-#define rt_spin_lock(lock)                      rt_enter_critical()
-#define rt_spin_unlock(lock)                    rt_exit_critical()
-#define rt_spin_lock_irqsave(lock)              rt_hw_interrupt_disable()
-#define rt_spin_unlock_irqrestore(lock, level)  rt_hw_interrupt_enable(level)
+#define rt_spin_lock_init(lock)                  { RT_UNUSED(lock);                                 }
+#define rt_spin_lock(lock)                       { RT_UNUSED(lock);                                 }
+#define rt_spin_unlock(lock)                     { RT_UNUSED(lock);                                 }
+#define rt_spin_lock_irqsave(lock)              ({ RT_UNUSED(lock); rt_hw_interrupt_disable();      })
+#define rt_spin_unlock_irqrestore(lock, level)   { RT_UNUSED(lock); rt_hw_interrupt_enable(level);  }
 #endif /* RT_USING_SMP */
 
 /**@}*/
@@ -742,14 +742,11 @@ if (!(EX))                                                                    \
 #define RT_DEBUG_NOT_IN_INTERRUPT                                             \
 do                                                                            \
 {                                                                             \
-    rt_base_t level;                                                          \
-    level = rt_hw_interrupt_disable();                                        \
     if (rt_interrupt_get_nest() != 0)                                         \
     {                                                                         \
         rt_kprintf("Function[%s] shall not be used in ISR\n", __FUNCTION__);  \
         RT_ASSERT(0)                                                          \
     }                                                                         \
-    rt_hw_interrupt_enable(level);                                            \
 }                                                                             \
 while (0)
 
@@ -760,8 +757,6 @@ while (0)
 #define RT_DEBUG_IN_THREAD_CONTEXT                                            \
 do                                                                            \
 {                                                                             \
-    rt_base_t level;                                                          \
-    level = rt_hw_interrupt_disable();                                        \
     if (rt_thread_self() == RT_NULL)                                          \
     {                                                                         \
         rt_kprintf("Function[%s] shall not be used before scheduler start\n", \
@@ -769,7 +764,6 @@ do                                                                            \
         RT_ASSERT(0)                                                          \
     }                                                                         \
     RT_DEBUG_NOT_IN_INTERRUPT;                                                \
-    rt_hw_interrupt_enable(level);                                            \
 }                                                                             \
 while (0)
 
@@ -785,9 +779,7 @@ do                                                                            \
     if (need_check)                                                           \
     {                                                                         \
         rt_bool_t interrupt_disabled;                                         \
-        rt_base_t level;                                                      \
         interrupt_disabled = rt_hw_interrupt_is_disabled();                   \
-        level = rt_hw_interrupt_disable();                                    \
         if (rt_critical_level() != 0)                                         \
         {                                                                     \
             rt_kprintf("Function[%s]: scheduler is not available\n",          \
@@ -801,7 +793,6 @@ do                                                                            \
             RT_ASSERT(0)                                                      \
         }                                                                     \
         RT_DEBUG_IN_THREAD_CONTEXT;                                           \
-        rt_hw_interrupt_enable(level);                                        \
     }                                                                         \
 }                                                                             \
 while (0)