sync smart & dfs (#8672)

Signed-off-by: xqyjlj <[email protected]>
Signed-off-by: Shell <[email protected]>
Co-authored-by: xqyjlj <[email protected]>

1 files changed, 475 insertions, 187 deletions

diff --git a/components/lwp/lwp_pid.c b/components/lwp/lwp_pid.c
index 7479df0357..5b9f1f4098 100644
--- a/components/lwp/lwp_pid.c
+++ b/components/lwp/lwp_pid.c
@@ -14,32 +14,47 @@
  *                             error
  * 2023-10-27     shell        Format codes of sys_exit(). Fix the data racing where lock is missed
  *                             Add reference on pid/tid, so the resource is not freed while using.
+ *                             Add support for waitpid(options=WNOHANG)
+ * 2023-11-16     xqyjlj       Fix the case where pid is 0
+ * 2023-11-17     xqyjlj       add process group and session support
+ * 2023-11-24     shell        Support of waitpid(options=WNOTRACED|WCONTINUED);
+ *                             Reimplement the waitpid with a wait queue method, and fixup problem
+ *                             with waitpid(pid=-1)/waitpid(pid=-pgid)/waitpid(pid=0) that only one
+ *                             process can be traced while waiter suspend
  * 2024-01-25     shell        porting to new sched API
  */
 
 /* includes scheduler related API */
 #define __RT_IPC_SOURCE__
 
-#include <rthw.h>
-#include <rtthread.h>
+/* for waitpid, we are compatible to GNU extension */
+#define _GNU_SOURCE
 
 #define DBG_TAG "lwp.pid"
 #define DBG_LVL DBG_INFO
 #include <rtdbg.h>
 
+#include "lwp_internal.h"
+
+#include <rthw.h>
+#include <rtthread.h>
 #include <dfs_file.h>
 #include <unistd.h>
 #include <stdio.h> /* rename() */
+#include <stdlib.h>
 #include <sys/stat.h>
 #include <sys/statfs.h> /* statfs() */
-
-#include "lwp_internal.h"
-#include "tty.h"
+#include <stdatomic.h>
 
 #ifdef ARCH_MM_MMU
 #include "lwp_user_mm.h"
 #endif
 
+#ifdef RT_USING_DFS_PROCFS
+#include "proc.h"
+#include "procfs.h"
+#endif
+
 #define PID_MAX 10000
 
 #define PID_CT_ASSERT(name, x) \
@@ -147,8 +162,23 @@ static void lwp_pid_put_locked(pid_t pid)
     }
 }
 
+#ifdef RT_USING_DFS_PROCFS
+    rt_inline void _free_proc_dentry(rt_lwp_t lwp)
+    {
+        char pid_str[64] = {0};
+
+        rt_snprintf(pid_str, 64, "%d", lwp->pid);
+        pid_str[63] = 0;
+        proc_remove_dentry(pid_str, 0);
+    }
+#else
+    #define _free_proc_dentry(lwp)
+#endif
+
 void lwp_pid_put(struct rt_lwp *lwp)
 {
+    _free_proc_dentry(lwp);
+
     lwp_pid_lock_take();
     lwp_pid_put_locked(lwp->pid);
     lwp_pid_lock_release();
@@ -168,6 +198,13 @@ static void lwp_pid_set_lwp_locked(pid_t pid, struct rt_lwp *lwp)
     {
         p->data = lwp;
         lwp_ref_inc(lwp);
+
+#ifdef RT_USING_DFS_PROCFS
+        if (pid)
+        {
+            proc_pid(pid);
+        }
+#endif
     }
 }
 
@@ -352,19 +389,23 @@ rt_lwp_t lwp_create(rt_base_t flags)
     if (new_lwp)
     {
         /* minimal setup of lwp object */
-        new_lwp->session = -1;
         new_lwp->ref = 1;
 #ifdef RT_USING_SMP
         new_lwp->bind_cpu = RT_CPUS_NR;
 #endif
-        rt_list_init(&new_lwp->wait_list);
+        new_lwp->exe_file = RT_NULL;
         rt_list_init(&new_lwp->t_grp);
+        rt_list_init(&new_lwp->pgrp_node);
         rt_list_init(&new_lwp->timer);
         lwp_user_object_lock_init(new_lwp);
         rt_wqueue_init(&new_lwp->wait_queue);
+        rt_wqueue_init(&new_lwp->waitpid_waiters);
         lwp_signal_init(&new_lwp->signal);
         rt_mutex_init(&new_lwp->lwp_lock, "lwp_lock", RT_IPC_FLAG_PRIO);
 
+        if (flags & LWP_CREATE_FLAG_NOTRACE_EXEC)
+            new_lwp->did_exec = RT_TRUE;
+
         /* lwp with pid */
         if (flags & LWP_CREATE_FLAG_ALLOC_PID)
         {
@@ -375,7 +416,7 @@ rt_lwp_t lwp_create(rt_base_t flags)
                 lwp_user_object_lock_destroy(new_lwp);
                 rt_free(new_lwp);
                 new_lwp = RT_NULL;
-                LOG_E("pid slot fulled!\n");
+                LOG_E("%s: pid slot fulled", __func__);
             }
             else
             {
@@ -384,6 +425,20 @@ rt_lwp_t lwp_create(rt_base_t flags)
             }
             lwp_pid_lock_release();
         }
+        rt_memset(&new_lwp->rt_rusage,0, sizeof(new_lwp->rt_rusage));
+
+        if (flags & LWP_CREATE_FLAG_INIT_USPACE)
+        {
+            rt_err_t error = lwp_user_space_init(new_lwp, 0);
+            if (error)
+            {
+                lwp_pid_put(new_lwp);
+                lwp_user_object_lock_destroy(new_lwp);
+                rt_free(new_lwp);
+                new_lwp = RT_NULL;
+                LOG_E("%s: failed to initialize user space", __func__);
+            }
+        }
     }
 
     LOG_D("%s(pid=%d) => %p", __func__, new_lwp ? new_lwp->pid : -1, new_lwp);
@@ -393,6 +448,8 @@ rt_lwp_t lwp_create(rt_base_t flags)
 /** when reference is 0, a lwp can be released */
 void lwp_free(struct rt_lwp* lwp)
 {
+    rt_processgroup_t group = RT_NULL;
+
     if (lwp == RT_NULL)
     {
         return;
@@ -406,6 +463,10 @@ void lwp_free(struct rt_lwp* lwp)
      *   all the reference is clear)
      */
     LOG_D("lwp free: %p", lwp);
+    rt_free(lwp->exe_file);
+    group = lwp_pgrp_find(lwp_pgid_get_byprocess(lwp));
+    if (group)
+        lwp_pgrp_remove(group, lwp);
 
     LWP_LOCK(lwp);
 
@@ -472,6 +533,15 @@ void lwp_free(struct rt_lwp* lwp)
 rt_inline rt_noreturn
 void _thread_exit(rt_lwp_t lwp, rt_thread_t thread)
 {
+    LWP_LOCK(lwp);
+    lwp->rt_rusage.ru_stime.tv_sec += thread->system_time / RT_TICK_PER_SECOND;
+    lwp->rt_rusage.ru_stime.tv_usec += thread->system_time % RT_TICK_PER_SECOND * (1000000 / RT_TICK_PER_SECOND);
+    lwp->rt_rusage.ru_utime.tv_sec += thread->user_time / RT_TICK_PER_SECOND;
+    lwp->rt_rusage.ru_utime.tv_usec += thread->user_time % RT_TICK_PER_SECOND * (1000000 / RT_TICK_PER_SECOND);
+    rt_list_remove(&thread->sibling);
+    LWP_UNLOCK(lwp);
+    lwp_futex_exit_robust_list(thread);
+
     /**
      * Note: the tid tree always hold a reference to thread, hence the tid must
      * be release before cleanup of thread
@@ -479,10 +549,6 @@ void _thread_exit(rt_lwp_t lwp, rt_thread_t thread)
     lwp_tid_put(thread->tid);
     thread->tid = 0;
 
-    LWP_LOCK(lwp);
-    rt_list_remove(&thread->sibling);
-    LWP_UNLOCK(lwp);
-
     rt_thread_delete(thread);
     rt_schedule();
     while (1) ;
@@ -497,11 +563,11 @@ rt_inline void _clear_child_tid(rt_thread_t thread)
 
         thread->clear_child_tid = RT_NULL;
         lwp_put_to_user(clear_child_tid, &t, sizeof t);
-        sys_futex(clear_child_tid, FUTEX_WAKE | FUTEX_PRIVATE, 1, RT_NULL, RT_NULL, 0);
+        sys_futex(clear_child_tid, FUTEX_WAKE, 1, RT_NULL, RT_NULL, 0);
     }
 }
 
-void lwp_exit(rt_lwp_t lwp, rt_base_t status)
+void lwp_exit(rt_lwp_t lwp, lwp_status_t status)
 {
     rt_thread_t thread;
 
@@ -523,7 +589,7 @@ void lwp_exit(rt_lwp_t lwp, rt_base_t status)
      * Brief: only one thread should calls exit_group(),
      * but we can not ensured that during run-time
      */
-    lwp->lwp_ret = LWP_CREATE_STAT(status);
+    lwp->lwp_status = status;
     LWP_UNLOCK(lwp);
 
     lwp_terminate(lwp);
@@ -550,7 +616,7 @@ void lwp_exit(rt_lwp_t lwp, rt_base_t status)
     _thread_exit(lwp, thread);
 }
 
-void lwp_thread_exit(rt_thread_t thread, rt_base_t status)
+void lwp_thread_exit(rt_thread_t thread, int status)
 {
     rt_thread_t header_thr;
     struct rt_lwp *lwp;
@@ -568,7 +634,11 @@ void lwp_thread_exit(rt_thread_t thread, rt_base_t status)
     header_thr = rt_list_entry(lwp->t_grp.prev, struct rt_thread, sibling);
     if (header_thr == thread && thread->sibling.prev == &lwp->t_grp)
     {
-        lwp->lwp_ret = LWP_CREATE_STAT(status);
+        /**
+         * if thread exit, treated as process exit normally.
+         * This is reasonable since trap event is exited through lwp_exit()
+         */
+        lwp->lwp_status = LWP_CREATE_STAT_EXIT(status);
         LWP_UNLOCK(lwp);
 
         lwp_terminate(lwp);
@@ -582,38 +652,6 @@ void lwp_thread_exit(rt_thread_t thread, rt_base_t status)
     _thread_exit(lwp, thread);
 }
 
-static void _pop_tty(rt_lwp_t lwp)
-{
-    if (!lwp->background)
-    {
-        struct termios *old_stdin_termios = get_old_termios();
-        struct rt_lwp *old_lwp = NULL;
-
-        if (lwp->session == -1)
-        {
-            tcsetattr(1, 0, old_stdin_termios);
-        }
-        if (lwp->tty != RT_NULL)
-        {
-            rt_mutex_take(&lwp->tty->lock, RT_WAITING_FOREVER);
-            if (lwp->tty->foreground == lwp)
-            {
-                old_lwp = tty_pop(&lwp->tty->head, RT_NULL);
-                lwp->tty->foreground = old_lwp;
-            }
-            else
-            {
-                tty_pop(&lwp->tty->head, lwp);
-            }
-            rt_mutex_release(&lwp->tty->lock);
-
-            LWP_LOCK(lwp);
-            lwp->tty = RT_NULL;
-            LWP_UNLOCK(lwp);
-        }
-    }
-}
-
 /** @note the reference is not for synchronization, but for the release of resource. the synchronization is done through lwp & pid lock */
 int lwp_ref_inc(struct rt_lwp *lwp)
 {
@@ -657,7 +695,7 @@ int lwp_ref_dec(struct rt_lwp *lwp)
     return ref;
 }
 
-struct rt_lwp* lwp_from_pid_locked(pid_t pid)
+struct rt_lwp* lwp_from_pid_raw_locked(pid_t pid)
 {
     struct lwp_avl_struct *p;
     struct rt_lwp *lwp = RT_NULL;
@@ -671,6 +709,13 @@ struct rt_lwp* lwp_from_pid_locked(pid_t pid)
     return lwp;
 }
 
+struct rt_lwp* lwp_from_pid_locked(pid_t pid)
+{
+    struct rt_lwp* lwp;
+    lwp = pid ? lwp_from_pid_raw_locked(pid) : lwp_self();
+    return lwp;
+}
+
 pid_t lwp_to_pid(struct rt_lwp* lwp)
 {
     if (!lwp)
@@ -733,162 +778,377 @@ pid_t lwp_name2pid(const char *name)
 
 int lwp_getpid(void)
 {
-    return ((struct rt_lwp *)rt_thread_self()->lwp)->pid;
+    rt_lwp_t lwp = lwp_self();
+    return lwp ? lwp->pid : 1;
+    // return ((struct rt_lwp *)rt_thread_self()->lwp)->pid;
 }
 
-/**
- * @brief Wait for a child lwp to terminate. Do the essential recycling. Setup
- *        status code for user
- */
-static sysret_t _lwp_wait_and_recycle(struct rt_lwp *child, rt_thread_t cur_thr,
-                                      struct rt_lwp *self_lwp, int *status,
-                                      int options)
+rt_inline void _update_ru(struct rt_lwp *child, struct rt_lwp *self_lwp, struct rusage *uru)
+{
+    struct rusage rt_rusage;
+    if (uru != RT_NULL)
+    {
+        rt_rusage.ru_stime.tv_sec = child->rt_rusage.ru_stime.tv_sec;
+        rt_rusage.ru_stime.tv_usec = child->rt_rusage.ru_stime.tv_usec;
+        rt_rusage.ru_utime.tv_sec = child->rt_rusage.ru_utime.tv_sec;
+        rt_rusage.ru_utime.tv_usec = child->rt_rusage.ru_utime.tv_usec;
+        lwp_data_put(self_lwp, uru, &rt_rusage, sizeof(*uru));
+    }
+}
+
+/* do statistical summary and reap the child if neccessary */
+static rt_err_t _stats_and_reap_child(rt_lwp_t child, rt_thread_t cur_thr,
+                                      struct rt_lwp *self_lwp, int *ustatus,
+                                      int options, struct rusage *uru)
 {
-    sysret_t error;
-    int lwp_stat;
-    int terminated;
+    int lwp_stat = child->lwp_status;
 
-    if (!child)
+    /* report statistical data to process */
+    _update_ru(child, self_lwp, uru);
+
+    if (child->terminated && !(options & WNOWAIT))
+    {
+        /** Reap the child process if it's exited */
+        LOG_D("func %s: child detached", __func__);
+        lwp_pid_put(child);
+        lwp_children_unregister(self_lwp, child);
+    }
+
+    if (ustatus)
+        lwp_data_put(self_lwp, ustatus, &lwp_stat, sizeof(*ustatus));
+
+    return RT_EOK;
+}
+
+#define HAS_CHILD_BUT_NO_EVT (-1024)
+
+/* check if the process is already terminate */
+static sysret_t _query_event_from_lwp(rt_lwp_t child, rt_thread_t cur_thr, rt_lwp_t self_lwp,
+                                      int options, int *status)
+{
+    sysret_t rc;
+
+    LWP_LOCK(child);
+    if (child->terminated)
+    {
+        rc = child->pid;
+    }
+    else if ((options & WSTOPPED) && child->jobctl_stopped && !child->wait_reap_stp)
     {
-        error = -RT_ERROR;
+        child->wait_reap_stp = 1;
+        rc = child->pid;
     }
     else
     {
+        rc = HAS_CHILD_BUT_NO_EVT;
+    }
+    LWP_UNLOCK(child);
+
+    LOG_D("%s(child_pid=%d ('%s'), stopped=%d) => %d", __func__, child->pid, child->cmd, child->jobctl_stopped, rc);
+    return rc;
+}
+
+/* verify if the process is child, and reap it */
+static pid_t _verify_child_and_reap(rt_thread_t cur_thr, rt_lwp_t self_lwp,
+                                       pid_t wait_pid, int options, int *ustatus,
+                                       struct rusage *uru)
+{
+    sysret_t rc;
+    struct rt_lwp *child;
+
+    /* check if pid is reference to a valid child */
+    lwp_pid_lock_take();
+    child = lwp_from_pid_locked(wait_pid);
+    if (!child)
+        rc = -EINVAL;
+    else if (child->parent != self_lwp)
+        rc = -ESRCH;
+    else
+        rc = wait_pid;
+
+    lwp_pid_lock_release();
+
+    if (rc > 0)
+    {
+        rc = _query_event_from_lwp(child, cur_thr, self_lwp, options, ustatus);
+        if (rc > 0)
+        {
+            _stats_and_reap_child(child, cur_thr, self_lwp, ustatus, options, uru);
+        }
+    }
+    return rc;
+}
+
+/* try to reap any child */
+static pid_t _reap_any_child_pid(rt_thread_t cur_thr, rt_lwp_t self_lwp, pid_t pair_pgid,
+                                 int options, int *ustatus, struct rusage *uru)
+{
+    sysret_t rc = -ECHILD;
+    struct rt_lwp *child;
+
+    LWP_LOCK(self_lwp);
+    child = self_lwp->first_child;
+
+    /* find a exited child if any */
+    while (child)
+    {
+        if (pair_pgid && child->pgid != pair_pgid)
+            continue;
+
+        rc = _query_event_from_lwp(child, cur_thr, self_lwp, options, ustatus);
+        if (rc > 0)
+            break;
+
+        child = child->sibling;
+    }
+    LWP_UNLOCK(self_lwp);
+
+    if (rc > 0)
+    {
+        _stats_and_reap_child(child, cur_thr, self_lwp, ustatus, options, uru);
+    }
+    return rc;
+}
+
+rt_err_t lwp_waitpid_kick(rt_lwp_t parent, rt_lwp_t self_lwp)
+{
+    /* waker provide the message mainly through its lwp_status */
+    rt_wqueue_wakeup(&parent->waitpid_waiters, self_lwp);
+    return RT_EOK;
+}
+
+struct waitpid_handle {
+    struct rt_wqueue_node wq_node;
+    int options;
+    rt_lwp_t waker_lwp;
+};
+
+/* the IPC message is setup and notify the parent */
+static int _waitq_filter(struct rt_wqueue_node *wait_node, void *key)
+{
+    int can_accept_evt = 0;
+    rt_thread_t waiter = wait_node->polling_thread;
+    pid_t destiny = (pid_t)wait_node->key;
+    rt_lwp_t waker_lwp = key;
+    struct waitpid_handle *handle;
+    rt_ubase_t options;
+
+    handle = rt_container_of(wait_node, struct waitpid_handle, wq_node);
+
+    RT_ASSERT(waiter != RT_NULL);
+    options = handle->options;
+
+    /* filter out if waker is not the one */
+    if (destiny > 0)
+    {
         /**
-         * Note: Critical Section
-         * - child lwp (RW. This will modify its parent if valid)
+         * in waitpid immediately return routine, we already do the check
+         * that pid is one of the child process of waiting thread
          */
-        LWP_LOCK(child);
-        if (child->terminated)
+        can_accept_evt = waker_lwp->pid == destiny;
+    }
+    else if (destiny == -1)
+    {
+        can_accept_evt = waker_lwp->parent == waiter->lwp;
+    }
+    else
+    {
+        /* destiny == 0 || destiny == -pgid */
+        pid_t waiter_pgid;
+        if (destiny == 0)
         {
-            error = child->pid;
+            waiter_pgid = lwp_pgid_get_byprocess(waiter->lwp);
         }
-        else if (rt_list_isempty(&child->wait_list))
+        else
         {
-            /**
-             * Note: only one thread can wait on wait_list.
-             * dont reschedule before mutex unlock
-             */
-            rt_enter_critical();
+            waiter_pgid = -destiny;
+        }
+        can_accept_evt = waiter_pgid == lwp_pgid_get_byprocess(waker_lwp);
+    }
 
-            error = rt_thread_suspend_with_flag(cur_thr, RT_INTERRUPTIBLE);
-            if (error == 0)
-            {
-                rt_list_insert_before(&child->wait_list, &RT_THREAD_LIST_NODE(cur_thr));
-                LWP_UNLOCK(child);
+    /* filter out if event is not desired */
+    if (can_accept_evt)
+    {
+        if ((options & WEXITED) && waker_lwp->terminated)
+            can_accept_evt = 1;
+        else if ((options & WSTOPPED) && WIFSTOPPED(waker_lwp->lwp_status))
+            can_accept_evt = 1;
+        else if ((options & WCONTINUED) && WIFCONTINUED(waker_lwp->lwp_status))
+            can_accept_evt = 1;
+        else
+            can_accept_evt = 0;
+    }
 
-                rt_set_errno(RT_EINTR);
-                rt_exit_critical();
-                rt_schedule();
+    /* setup message for waiter if accepted */
+    if (can_accept_evt)
+        handle->waker_lwp = waker_lwp;
 
-                /**
-                 * Since parent is holding a reference to children this lock will
-                 * not be freed before parent dereference to it.
-                 */
-                LWP_LOCK(child);
+    /* 0 if event is accepted, otherwise discard */
+    return !can_accept_evt;
+}
 
-                error = rt_get_errno();
-                if (error == RT_EINTR)
-                {
-                    error = -EINTR;
-                }
-                else if (error != RT_EOK)
-                {
-                    LOG_W("%s: unexpected error code %ld", __func__, error);
-                }
-                else
-                {
-                    error = child->pid;
-                }
-            }
-            else
-                rt_exit_critical();
-        }
-        else
-            error = -RT_EINTR;
+/* the waiter cleanup IPC message and wait for desired event here */
+static rt_err_t _wait_for_event(rt_thread_t cur_thr, rt_lwp_t self_lwp,
+                                struct waitpid_handle *handle, pid_t destiny)
+{
+    rt_err_t ret;
 
-        lwp_stat = child->lwp_ret;
-        terminated = child->terminated;
-        LWP_UNLOCK(child);
+    /* current context checking */
+    RT_DEBUG_SCHEDULER_AVAILABLE(RT_TRUE);
 
-        if (error > 0)
-        {
-            if (terminated)
-            {
-                LOG_D("func %s: child detached", __func__);
-                /** Reap the child process if it's exited */
-                lwp_pid_put(child);
-                lwp_children_unregister(self_lwp, child);
-            }
+    handle->wq_node.polling_thread = cur_thr;
+    handle->wq_node.key = destiny;
+    handle->wq_node.wakeup = _waitq_filter;
+    handle->wq_node.wqueue = &self_lwp->waitpid_waiters;
+    rt_list_init(&handle->wq_node.list);
 
-            if (status)
-                lwp_data_put(self_lwp, status, &lwp_stat, sizeof(*status));
-        }
+    cur_thr->error = RT_EOK;
+
+    LOG_D("%s(self_lwp=%d) wait for event", __func__, self_lwp->pid);
+
+    rt_enter_critical();
+    ret = rt_thread_suspend_with_flag(cur_thr, RT_INTERRUPTIBLE);
+    if (ret == RT_EOK)
+    {
+        rt_wqueue_add(handle->wq_node.wqueue, &handle->wq_node);
+        rt_exit_critical();
+
+        rt_schedule();
+
+        ret = cur_thr->error;
+
+        /**
+         * cur_thr error is a positive value, but some legacy implementation
+         * use a negative one. So we check to avoid errors
+         */
+        ret = ret > 0 ? -ret : ret;
+
+        /**
+         * we dont rely on this actually, but we cleanup it since wakeup API
+         * set this up durint operation, and this will cause some messy condition
+         */
+        handle->wq_node.wqueue->flag = RT_WQ_FLAG_CLEAN;
+        rt_wqueue_remove(&handle->wq_node);
+    }
+    else
+    {
+        /* failed to suspend, return immediately with failure */
+        rt_exit_critical();
     }
 
-    return error;
+    return ret;
 }
 
-pid_t waitpid(pid_t pid, int *status, int options) __attribute__((alias("lwp_waitpid")));
+/* wait for IPC event and do the cleanup if neccessary */
+static sysret_t _wait_and_reap(rt_thread_t cur_thr, rt_lwp_t self_lwp, const pid_t pid,
+                               int options, int *ustatus, struct rusage *uru)
+{
+    sysret_t rc;
+    struct waitpid_handle handle;
+    rt_lwp_t waker;
+
+    /* wait for SIGCHLD or other async events */
+    handle.options = options;
+    handle.waker_lwp = 0;
+    rc = _wait_for_event(cur_thr, self_lwp, &handle, pid);
+
+    waker = handle.waker_lwp;
+    if (waker != RT_NULL)
+    {
+        rc = waker->pid;
+
+        /* check out if any process exited */
+        LOG_D("%s: woken up by lwp=%d", __func__, waker->pid);
+        _stats_and_reap_child(waker, cur_thr, self_lwp, ustatus, options, uru);
+    }
+    /**
+     * else if (rc != RT_EOK)
+     * unable to do a suspend, or wakeup unexpectedly
+     * -> then returned a failure
+     */
+
+    return rc;
+}
 
-pid_t lwp_waitpid(const pid_t pid, int *status, int options)
+pid_t lwp_waitpid(const pid_t pid, int *status, int options, struct rusage *ru)
 {
     pid_t rc = -1;
-    struct rt_thread *thread;
-    struct rt_lwp *child;
+    struct rt_thread *cur_thr;
     struct rt_lwp *self_lwp;
 
-    thread = rt_thread_self();
+    cur_thr = rt_thread_self();
     self_lwp = lwp_self();
 
-    if (!self_lwp)
+    if (!cur_thr || !self_lwp)
     {
-        rc = -RT_EINVAL;
+        rc = -EINVAL;
     }
     else
     {
+        /* check if able to reap desired child immediately */
         if (pid > 0)
         {
-            lwp_pid_lock_take();
-            child = lwp_from_pid_locked(pid);
-            if (child->parent != self_lwp)
-                rc = -RT_ERROR;
-            else
-                rc = RT_EOK;
-            lwp_pid_lock_release();
-
-            if (rc == RT_EOK)
-                rc = _lwp_wait_and_recycle(child, thread, self_lwp, status, options);
+            /* if pid is child then try to reap it */
+            rc = _verify_child_and_reap(cur_thr, self_lwp, pid, options, status, ru);
         }
         else if (pid == -1)
         {
-            LWP_LOCK(self_lwp);
-            child = self_lwp->first_child;
-            LWP_UNLOCK(self_lwp);
-            RT_ASSERT(!child || child->parent == self_lwp);
-
-            rc = _lwp_wait_and_recycle(child, thread, self_lwp, status, options);
+            /* any terminated child */
+            rc = _reap_any_child_pid(cur_thr, self_lwp, 0, options, status, ru);
         }
         else
         {
-            /* not supported yet */
-            rc = -RT_EINVAL;
+            /**
+             * (pid < -1 || pid == 0)
+             * any terminated child with matched pgid
+             */
+
+            pid_t pair_pgid;
+            if (pid == 0)
+            {
+                pair_pgid = lwp_pgid_get_byprocess(self_lwp);
+            }
+            else
+            {
+                pair_pgid = -pid;
+            }
+            rc = _reap_any_child_pid(cur_thr, self_lwp, pair_pgid, options, status, ru);
         }
-    }
 
-    if (rc > 0)
-    {
-        LOG_D("%s: recycle child id %ld (status=0x%x)", __func__, (long)rc, status ? *status : 0);
-    }
-    else
-    {
-        RT_ASSERT(rc != 0);
-        LOG_D("%s: wait failed with code %ld", __func__, (long)rc);
+        if (rc == HAS_CHILD_BUT_NO_EVT)
+        {
+            if (!(options & WNOHANG))
+            {
+                /* otherwise, arrange a suspend and wait for async event */
+                options |= WEXITED;
+                rc = _wait_and_reap(cur_thr, self_lwp, pid, options, status, ru);
+            }
+            else
+            {
+                /**
+                 * POSIX.1: If waitpid() was invoked with WNOHANG set in options,
+                 * it has at least one child process specified by pid for which
+                 * status is not available, and status is not available for any
+                 * process specified by pid, 0 is returned
+                 */
+                rc = 0;
+            }
+        }
+        else
+        {
+            RT_ASSERT(rc != 0);
+        }
     }
 
+    LOG_D("waitpid() => %d, *status=0x%x", rc, status ? *status:0);
     return rc;
 }
 
+pid_t waitpid(pid_t pid, int *status, int options)
+{
+    return lwp_waitpid(pid, status, options, RT_NULL);
+}
+
 #ifdef RT_USING_FINSH
 /* copy from components/finsh/cmd.c */
 static void object_split(int len)
@@ -910,7 +1170,7 @@ static void print_thread_info(struct rt_thread* thread, int maxlen)
     else
         rt_kprintf("%-*.*s N/A %3d ", maxlen, RT_NAME_MAX, thread->parent.name, RT_SCHED_PRIV(thread).current_priority);
 #else
-    rt_kprintf("%-*.*s %3d ", maxlen, RT_NAME_MAX, thread->parent.name, thread->current_priority);
+    rt_kprintf("%-*.*s %3d ", maxlen, RT_NAME_MAX, thread->parent.name, RT_SCHED_PRIV(thread).current_priority);
 #endif /*RT_USING_SMP*/
 
     stat = (RT_SCHED_CTX(thread).stat & RT_THREAD_STAT_MASK);
@@ -1044,7 +1304,7 @@ static void cmd_kill(int argc, char** argv)
         }
     }
     lwp_pid_lock_take();
-    lwp_signal_kill(lwp_from_pid_locked(pid), sig, SI_USER, 0);
+    lwp_signal_kill(lwp_from_pid_raw_locked(pid), sig, SI_USER, 0);
     lwp_pid_lock_release();
 }
 MSH_CMD_EXPORT_ALIAS(cmd_kill, kill, send a signal to a process);
@@ -1061,7 +1321,7 @@ static void cmd_killall(int argc, char** argv)
     while((pid = lwp_name2pid(argv[1])) > 0)
     {
         lwp_pid_lock_take();
-        lwp_signal_kill(lwp_from_pid_locked(pid), SIGKILL, SI_USER, 0);
+        lwp_signal_kill(lwp_from_pid_raw_locked(pid), SIGKILL, SI_USER, 0);
         lwp_pid_lock_release();
         rt_thread_mdelay(100);
     }
@@ -1073,15 +1333,15 @@ MSH_CMD_EXPORT_ALIAS(cmd_killall, killall, kill processes by name);
 int lwp_check_exit_request(void)
 {
     rt_thread_t thread = rt_thread_self();
-    rt_base_t expected = LWP_EXIT_REQUEST_TRIGGERED;
+    rt_size_t expected = LWP_EXIT_REQUEST_TRIGGERED;
 
     if (!thread->lwp)
     {
         return 0;
     }
 
-    return rt_atomic_compare_exchange_strong(&thread->exit_request, &expected,
-                                             LWP_EXIT_REQUEST_IN_PROCESS);
+    return atomic_compare_exchange_strong(&thread->exit_request, &expected,
+                                          LWP_EXIT_REQUEST_IN_PROCESS);
 }
 
 static void _wait_sibling_exit(rt_lwp_t lwp, rt_thread_t curr_thread);
@@ -1119,7 +1379,7 @@ static void _wait_sibling_exit(rt_lwp_t lwp, rt_thread_t curr_thread)
     rt_sched_lock_level_t slvl;
     rt_list_t *list;
     rt_thread_t thread;
-    rt_base_t expected = LWP_EXIT_REQUEST_NONE;
+    rt_size_t expected = LWP_EXIT_REQUEST_NONE;
 
     /* broadcast exit request for sibling threads */
     LWP_LOCK(lwp);
@@ -1127,8 +1387,8 @@ static void _wait_sibling_exit(rt_lwp_t lwp, rt_thread_t curr_thread)
     {
         thread = rt_list_entry(list, struct rt_thread, sibling);
 
-        rt_atomic_compare_exchange_strong(&thread->exit_request, &expected,
-                                          LWP_EXIT_REQUEST_TRIGGERED);
+        atomic_compare_exchange_strong(&thread->exit_request, &expected,
+                                       LWP_EXIT_REQUEST_TRIGGERED);
 
         rt_sched_lock(&slvl);
         /* dont release, otherwise thread may have been freed */
@@ -1213,8 +1473,43 @@ static void _wait_sibling_exit(rt_lwp_t lwp, rt_thread_t curr_thread)
     }
 }
 
+static void _notify_parent(rt_lwp_t lwp)
+{
+    int si_code;
+    int signo_or_exitcode;
+    lwp_siginfo_ext_t ext;
+    lwp_status_t lwp_status = lwp->lwp_status;
+    rt_lwp_t parent = lwp->parent;
+
+    if (WIFSIGNALED(lwp_status))
+    {
+        si_code = (lwp_status & LWP_COREDUMP_FLAG) ? CLD_DUMPED : CLD_KILLED;
+        signo_or_exitcode = WTERMSIG(lwp_status);
+    }
+    else
+    {
+        si_code = CLD_EXITED;
+        signo_or_exitcode = WEXITSTATUS(lwp->lwp_status);
+    }
+
+    lwp_waitpid_kick(parent, lwp);
+
+    ext = rt_malloc(sizeof(struct lwp_siginfo));
+
+    if (ext)
+    {
+        rt_thread_t cur_thr = rt_thread_self();
+        ext->sigchld.status = signo_or_exitcode;
+        ext->sigchld.stime = cur_thr->system_time;
+        ext->sigchld.utime = cur_thr->user_time;
+    }
+    lwp_signal_kill(parent, SIGCHLD, si_code, ext);
+}
+
 static void _resr_cleanup(struct rt_lwp *lwp)
 {
+    lwp_jobctrl_on_exit(lwp);
+
     LWP_LOCK(lwp);
     lwp_signal_detach(&lwp->signal);
 
@@ -1254,8 +1549,6 @@ static void _resr_cleanup(struct rt_lwp *lwp)
     }
     LWP_UNLOCK(lwp);
 
-    _pop_tty(lwp);
-
     /**
      * @brief Wakeup parent if it's waiting for this lwp, otherwise a signal
      *        will be sent to parent
@@ -1264,29 +1557,26 @@ static void _resr_cleanup(struct rt_lwp *lwp)
      * - the parent lwp (RW.)
      */
     LWP_LOCK(lwp);
-    if (lwp->parent)
+    if (lwp->parent &&
+        !lwp_sigismember(&lwp->parent->signal.sig_action_nocldwait, SIGCHLD))
     {
-        struct rt_thread *thread;
-
+        /* if successfully race to setup lwp->terminated before parent detach */
         LWP_UNLOCK(lwp);
-        if (!rt_list_isempty(&lwp->wait_list))
-        {
-            thread = RT_THREAD_LIST_NODE_ENTRY(lwp->wait_list.next);
-            thread->error = RT_EOK;
-            thread->msg_ret = (void*)(rt_size_t)lwp->lwp_ret;
-            rt_thread_resume(thread);
-        }
-        else
-        {
-            /* children cannot detach itself and must wait for parent to take care of it */
-            lwp_signal_kill(lwp->parent, SIGCHLD, CLD_EXITED, 0);
-        }
+
+        /**
+         * Note: children cannot detach itself and must wait for parent to take
+         * care of it
+         */
+        _notify_parent(lwp);
     }
     else
     {
         LWP_UNLOCK(lwp);
 
-        /* INFO: orphan hasn't parents to do the reap of pid */
+        /**
+         * if process is orphan, it doesn't have parent to do the recycling.
+         * Otherwise, its parent had setup a flag to mask out recycling event
+         */
         lwp_pid_put(lwp);
     }
 
@@ -1315,10 +1605,8 @@ static int _lwp_setaffinity(pid_t pid, int cpu)
     int ret = -1;
 
     lwp_pid_lock_take();
-    if(pid == 0)
-        lwp = lwp_self();
-    else
-        lwp = lwp_from_pid_locked(pid);
+    lwp = lwp_from_pid_locked(pid);
+
     if (lwp)
     {
 #ifdef RT_USING_SMP