1 files changed, 81 insertions, 23 deletions

diff --git a/ipc/mqueue.c b/ipc/mqueue.c
index af1692556c52..8ce57691e7b6 100644
--- a/ipc/mqueue.c
+++ b/ipc/mqueue.c
@@ -69,6 +69,7 @@ struct mqueue_inode_info {
         wait_queue_head_t wait_q;
 
         struct rb_root msg_tree;
+        struct posix_msg_tree_node *node_cache;
         struct mq_attr attr;
 
         struct sigevent notify;
@@ -134,15 +135,20 @@ static int msg_insert(struct msg_msg *msg, struct mqueue_inode_info *info)
                 else
                         p = &(*p)->rb_right;
         }
-        leaf = kzalloc(sizeof(*leaf), GFP_ATOMIC);
-        if (!leaf)
-                return -ENOMEM;
-        rb_init_node(&leaf->rb_node);
-        INIT_LIST_HEAD(&leaf->msg_list);
+        if (info->node_cache) {
+                leaf = info->node_cache;
+                info->node_cache = NULL;
+        } else {
+                leaf = kmalloc(sizeof(*leaf), GFP_ATOMIC);
+                if (!leaf)
+                        return -ENOMEM;
+                rb_init_node(&leaf->rb_node);
+                INIT_LIST_HEAD(&leaf->msg_list);
+                info->qsize += sizeof(*leaf);
+        }
         leaf->priority = msg->m_type;
         rb_link_node(&leaf->rb_node, parent, p);
         rb_insert_color(&leaf->rb_node, &info->msg_tree);
-        info->qsize += sizeof(struct posix_msg_tree_node);
 insert_msg:
         info->attr.mq_curmsgs++;
         info->qsize += msg->m_ts;
@@ -177,13 +183,17 @@ try_again:
                 return NULL;
         }
         leaf = rb_entry(parent, struct posix_msg_tree_node, rb_node);
-        if (list_empty(&leaf->msg_list)) {
+        if (unlikely(list_empty(&leaf->msg_list))) {
                 pr_warn_once("Inconsistency in POSIX message queue, "
                              "empty leaf node but we haven't implemented "
                              "lazy leaf delete!\n");
                 rb_erase(&leaf->rb_node, &info->msg_tree);
-                info->qsize -= sizeof(struct posix_msg_tree_node);
-                kfree(leaf);
+                if (info->node_cache) {
+                        info->qsize -= sizeof(*leaf);
+                        kfree(leaf);
+                } else {
+                        info->node_cache = leaf;
+                }
                 goto try_again;
         } else {
                 msg = list_first_entry(&leaf->msg_list,
@@ -191,8 +201,12 @@ try_again:
                 list_del(&msg->m_list);
                 if (list_empty(&leaf->msg_list)) {
                         rb_erase(&leaf->rb_node, &info->msg_tree);
-                        info->qsize -= sizeof(struct posix_msg_tree_node);
-                        kfree(leaf);
+                        if (info->node_cache) {
+                                info->qsize -= sizeof(*leaf);
+                                kfree(leaf);
+                        } else {
+                                info->node_cache = leaf;
+                        }
                 }
         }
         info->attr.mq_curmsgs--;
@@ -235,6 +249,7 @@ static struct inode *mqueue_get_inode(struct super_block *sb,
                 info->qsize = 0;
                 info->user = NULL;      /* set when all is ok */
                 info->msg_tree = RB_ROOT;
+                info->node_cache = NULL;
                 memset(&info->attr, 0, sizeof(info->attr));
                 info->attr.mq_maxmsg = min(ipc_ns->mq_msg_max,
                                            ipc_ns->mq_msg_default);
@@ -367,6 +382,7 @@ static void mqueue_evict_inode(struct inode *inode)
         spin_lock(&info->lock);
         while ((msg = msg_get(info)) != NULL)
                 free_msg(msg);
+        kfree(info->node_cache);
         spin_unlock(&info->lock);
 
         /* Total amount of bytes accounted for the mqueue */
@@ -964,7 +980,8 @@ SYSCALL_DEFINE5(mq_timedsend, mqd_t, mqdes, const char __user *, u_msg_ptr,
         struct mqueue_inode_info *info;
         ktime_t expires, *timeout = NULL;
         struct timespec ts;
-        int ret;
+        struct posix_msg_tree_node *new_leaf = NULL;
+        int ret = 0;
 
         if (u_abs_timeout) {
                 int res = prepare_timeout(u_abs_timeout, &expires, &ts);
@@ -1012,39 +1029,60 @@ SYSCALL_DEFINE5(mq_timedsend, mqd_t, mqdes, const char __user *, u_msg_ptr,
         msg_ptr->m_ts = msg_len;
         msg_ptr->m_type = msg_prio;
 
+        /*
+         * msg_insert really wants us to have a valid, spare node struct so
+         * it doesn't have to kmalloc a GFP_ATOMIC allocation, but it will
+         * fall back to that if necessary.
+         */
+        if (!info->node_cache)
+                new_leaf = kmalloc(sizeof(*new_leaf), GFP_KERNEL);
+
         spin_lock(&info->lock);
 
+        if (!info->node_cache && new_leaf) {
+                /* Save our speculative allocation into the cache */
+                rb_init_node(&new_leaf->rb_node);
+                INIT_LIST_HEAD(&new_leaf->msg_list);
+                info->node_cache = new_leaf;
+                info->qsize += sizeof(*new_leaf);
+                new_leaf = NULL;
+        } else {
+                kfree(new_leaf);
+        }
+
         if (info->attr.mq_curmsgs == info->attr.mq_maxmsg) {
                 if (filp->f_flags & O_NONBLOCK) {
-                        spin_unlock(&info->lock);
                         ret = -EAGAIN;
                 } else {
                         wait.task = current;
                         wait.msg = (void *) msg_ptr;
                         wait.state = STATE_NONE;
                         ret = wq_sleep(info, SEND, timeout, &wait);
+                        /*
+                         * wq_sleep must be called with info->lock held, and
+                         * returns with the lock released
+                         */
+                        goto out_free;
                 }
-                if (ret < 0)
-                        free_msg(msg_ptr);
         } else {
                 receiver = wq_get_first_waiter(info, RECV);
                 if (receiver) {
                         pipelined_send(info, msg_ptr, receiver);
                 } else {
                         /* adds message to the queue */
-                        if (msg_insert(msg_ptr, info)) {
-                                free_msg(msg_ptr);
-                                ret = -ENOMEM;
-                                spin_unlock(&info->lock);
-                                goto out_fput;
-                        }
+                        ret = msg_insert(msg_ptr, info);
+                        if (ret)
+                                goto out_unlock;
                         __do_notify(info);
                 }
                 inode->i_atime = inode->i_mtime = inode->i_ctime =
                                 CURRENT_TIME;
-                spin_unlock(&info->lock);
-                ret = 0;
         }
+out_unlock:
+        spin_unlock(&info->lock);
+out_free:
+        if (ret)
+                free_msg(msg_ptr);
 out_fput:
         fput(filp);
 out:
@@ -1063,6 +1101,7 @@ SYSCALL_DEFINE5(mq_timedreceive, mqd_t, mqdes, char __user *, u_msg_ptr,
         struct ext_wait_queue wait;
         ktime_t expires, *timeout = NULL;
         struct timespec ts;
+        struct posix_msg_tree_node *new_leaf = NULL;
 
         if (u_abs_timeout) {
                 int res = prepare_timeout(u_abs_timeout, &expires, &ts);
@@ -1098,7 +1137,26 @@ SYSCALL_DEFINE5(mq_timedreceive, mqd_t, mqdes, char __user *, u_msg_ptr,
                 goto out_fput;
         }
 
+        /*
+         * msg_insert really wants us to have a valid, spare node struct so
+         * it doesn't have to kmalloc a GFP_ATOMIC allocation, but it will
+         * fall back to that if necessary.
+         */
+        if (!info->node_cache)
+                new_leaf = kmalloc(sizeof(*new_leaf), GFP_KERNEL);
+
         spin_lock(&info->lock);
+
+        if (!info->node_cache && new_leaf) {
+                /* Save our speculative allocation into the cache */
+                rb_init_node(&new_leaf->rb_node);
+                INIT_LIST_HEAD(&new_leaf->msg_list);
+                info->node_cache = new_leaf;
+                info->qsize += sizeof(*new_leaf);
+        } else {
+                kfree(new_leaf);
+        }
+
         if (info->attr.mq_curmsgs == 0) {
                 if (filp->f_flags & O_NONBLOCK) {
                         spin_unlock(&info->lock);