tty: never hold BTM while getting tty_mutex

tty_mutex is never taken with the BTM held, except for
two corner cases that are worked around here.
We give up the BTM before calling tty_release() in the
error path of tty_open().
Similarly, we reorder the locking in ptmx_open()
to get tty_mutex before the BTM.

Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Cc: Alan Cox <alan@lxorguk.ukuu.org.uk>
Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de>

2 files changed, 17 insertions, 19 deletions

diff --git a/drivers/char/pty.c b/drivers/char/pty.c
index 622e21ca9a5..de22ea95283 100644
--- a/drivers/char/pty.c
+++ b/drivers/char/pty.c
@@ -647,7 +647,7 @@ static const struct tty_operations pty_unix98_ops = {
  *              allocated_ptys_lock handles the list of free pty numbers
  */
 
-static int __ptmx_open(struct inode *inode, struct file *filp)
+static int ptmx_open(struct inode *inode, struct file *filp)
 {
         struct tty_struct *tty;
         int retval;
@@ -656,11 +656,14 @@ static int __ptmx_open(struct inode *inode, struct file *filp)
         nonseekable_open(inode, filp);
 
         /* find a device that is not in use. */
+        tty_lock();
         index = devpts_new_index(inode);
+        tty_unlock();
         if (index < 0)
                 return index;
 
         mutex_lock(&tty_mutex);
+        tty_lock();
         tty = tty_init_dev(ptm_driver, index, 1);
         mutex_unlock(&tty_mutex);
 
@@ -678,24 +681,19 @@ static int __ptmx_open(struct inode *inode, struct file *filp)
                 goto out1;
 
         retval = ptm_driver->ops->open(tty, filp);
-        if (!retval)
-                return 0;
+        if (retval)
+                goto out2;
 out1:
+        tty_unlock();
+        return retval;
+out2:
+        tty_unlock();
         tty_release(inode, filp);
         return retval;
 out:
         devpts_kill_index(inode, index);
-        return retval;
-}
-
-static int ptmx_open(struct inode *inode, struct file *filp)
-{
-        int ret;
-
-        tty_lock();
-        ret = __ptmx_open(inode, filp);
         tty_unlock();
-        return ret;
+        return retval;
 }
 
 static struct file_operations ptmx_fops;
diff --git a/drivers/char/tty_io.c b/drivers/char/tty_io.c
index 5ee9081a560..bb22cf49543 100644
--- a/drivers/char/tty_io.c
+++ b/drivers/char/tty_io.c
@@ -1866,19 +1866,19 @@ got_driver:
                 printk(KERN_DEBUG "error %d in opening %s...", retval,
                        tty->name);
 #endif
+                tty_unlock(); /* need to call tty_release without BTM */
                 tty_release(inode, filp);
-                if (retval != -ERESTARTSYS) {
-                        tty_unlock();
+                if (retval != -ERESTARTSYS)
                         return retval;
-                }
-                if (signal_pending(current)) {
-                        tty_unlock();
+
+                if (signal_pending(current))
                         return retval;
-                }
+
                 schedule();
                 /*
                  * Need to reset f_op in case a hangup happened.
                  */
+                tty_lock();
                 if (filp->f_op == &hung_up_tty_fops)
                         filp->f_op = &tty_fops;
                 tty_unlock();