[smart][tty] guard NULL lwp in tty_wait_background

Fix #11717

On RT-Smart a kernel thread (lwp == NULL) that reads a tty reaches
tty_wait_background() and dereferences td->lwp as a process. The shell
thread is a kernel thread while finsh reads /dev/ttyS0, so the first key
press faults on p->pgrp.

Return 0 for a NULL lwp: a kernel thread has no process group, so no
background-job decision applies. Apply the same guard to the sibling
paths that also dereference td->lwp: tty_is_ctty() returns false,
tty_drop_ctty() returns -ENOTTY, and lwp_tty_set_ctrl_proc() returns
-EPERM.

No ttydisc refactor and no change to the ioctl argument marshalling.

3 files changed, 23 insertions, 0 deletions

diff --git a/components/lwp/terminal/freebsd/tty.c b/components/lwp/terminal/freebsd/tty.c
index 622c4dd975..e4bf6fd680 100644
--- a/components/lwp/terminal/freebsd/tty.c
+++ b/components/lwp/terminal/freebsd/tty.c
@@ -421,6 +421,11 @@ int tty_wait_background(struct lwp_tty *tp, struct rt_thread *td, int sig)
     tty_assert_locked(tp);
 
     p = td->lwp;
+    if (p == RT_NULL)
+    {
+        /* kernel threads have no process group: nothing to wait for */
+        return 0;
+    }
     for (;;)
     {
         pg = p->pgrp;
@@ -925,6 +930,12 @@ static int tty_drop_ctty(struct lwp_tty *tp, struct rt_lwp *p)
     struct vnode *vp;
 #endif
 
+    /* a kernel thread owns no session or process group */
+    if (p == RT_NULL)
+    {
+        return -ENOTTY;
+    }
+
     /*
      * This looks terrible, but it's generally safe as long as the tty
      * hasn't gone away while we had the lock dropped.  All of our sanity
diff --git a/components/lwp/terminal/freebsd/tty_compat.c b/components/lwp/terminal/freebsd/tty_compat.c
index 10e97ed9f6..503a7587c8 100644
--- a/components/lwp/terminal/freebsd/tty_compat.c
+++ b/components/lwp/terminal/freebsd/tty_compat.c
@@ -148,6 +148,12 @@ int lwp_tty_set_ctrl_proc(lwp_tty_t tp, rt_thread_t td)
     int rc = -1;
     struct rt_lwp *p = td->lwp;
 
+    /* a kernel thread cannot own a controlling terminal */
+    if (p == RT_NULL)
+    {
+        return -EPERM;
+    }
+
     tty_unlock(tp);
     LWP_LOCK(p);
     tty_lock(tp);
diff --git a/components/lwp/terminal/tty_internal.h b/components/lwp/terminal/tty_internal.h
index fdd563e09a..cb57383529 100644
--- a/components/lwp/terminal/tty_internal.h
+++ b/components/lwp/terminal/tty_internal.h
@@ -86,6 +86,12 @@ rt_inline int tty_is_ctty(struct lwp_tty *tp, struct rt_lwp *p)
 {
     tty_assert_locked(tp);
 
+    /* kernel threads have no process group or controlling terminal */
+    if (p == RT_NULL)
+    {
+        return RT_FALSE;
+    }
+
     return p->pgrp->session == tp->t_session && p->term_ctrlterm;
 }