3 files changed, 162 insertions, 60 deletions

diff --git a/include/linux/plist.h b/include/linux/plist.h
index 7254eda078e5..c9b9f322c8d8 100644
--- a/include/linux/plist.h
+++ b/include/linux/plist.h
@@ -31,15 +31,17 @@
  *
  * Simple ASCII art explanation:
  *
- * |HEAD          |
- * |              |
- * |prio_list.prev|<------------------------------------|
- * |prio_list.next|<->|pl|<->|pl|<--------------->|pl|<-|
- * |10            |   |10|   |21|   |21|   |21|   |40|   (prio)
- * |              |   |  |   |  |   |  |   |  |   |  |
- * |              |   |  |   |  |   |  |   |  |   |  |
- * |node_list.next|<->|nl|<->|nl|<->|nl|<->|nl|<->|nl|<-|
- * |node_list.prev|<------------------------------------|
+ * pl:prio_list (only for plist_node)
+ * nl:node_list
+ *   HEAD|             NODE(S)
+ *       |
+ *       ||------------------------------------|
+ *       ||->|pl|<->|pl|<--------------->|pl|<-|
+ *       |   |10|   |21|   |21|   |21|   |40|   (prio)
+ *       |   |  |   |  |   |  |   |  |   |  |
+ *       |   |  |   |  |   |  |   |  |   |  |
+ * |->|nl|<->|nl|<->|nl|<->|nl|<->|nl|<->|nl|<-|
+ * |-------------------------------------------|
  *
  * The nodes on the prio_list list are sorted by priority to simplify
  * the insertion of new nodes. There are no nodes with duplicate
@@ -78,7 +80,6 @@
 #include <linux/spinlock_types.h>
 
 struct plist_head {
-        struct list_head prio_list;
         struct list_head node_list;
 #ifdef CONFIG_DEBUG_PI_LIST
         raw_spinlock_t *rawlock;
@@ -88,7 +89,8 @@ struct plist_head {
 
 struct plist_node {
         int                     prio;
-        struct plist_head       plist;
+        struct list_head        prio_list;
+        struct list_head        node_list;
 };
 
 #ifdef CONFIG_DEBUG_PI_LIST
@@ -100,7 +102,6 @@ struct plist_node {
 #endif
 
 #define _PLIST_HEAD_INIT(head)                          \
-        .prio_list = LIST_HEAD_INIT((head).prio_list),  \
         .node_list = LIST_HEAD_INIT((head).node_list)
 
 /**
@@ -133,7 +134,8 @@ struct plist_node {
 #define PLIST_NODE_INIT(node, __prio)                   \
 {                                                       \
         .prio  = (__prio),                              \
-        .plist = { _PLIST_HEAD_INIT((node).plist) },    \
+        .prio_list = LIST_HEAD_INIT((node).prio_list),  \
+        .node_list = LIST_HEAD_INIT((node).node_list),  \
 }
 
 /**
@@ -144,7 +146,6 @@ struct plist_node {
 static inline void
 plist_head_init(struct plist_head *head, spinlock_t *lock)
 {
-        INIT_LIST_HEAD(&head->prio_list);
         INIT_LIST_HEAD(&head->node_list);
 #ifdef CONFIG_DEBUG_PI_LIST
         head->spinlock = lock;
@@ -160,7 +161,6 @@ plist_head_init(struct plist_head *head, spinlock_t *lock)
 static inline void
 plist_head_init_raw(struct plist_head *head, raw_spinlock_t *lock)
 {
-        INIT_LIST_HEAD(&head->prio_list);
         INIT_LIST_HEAD(&head->node_list);
 #ifdef CONFIG_DEBUG_PI_LIST
         head->rawlock = lock;
@@ -176,7 +176,8 @@ plist_head_init_raw(struct plist_head *head, raw_spinlock_t *lock)
 static inline void plist_node_init(struct plist_node *node, int prio)
 {
         node->prio = prio;
-        plist_head_init(&node->plist, NULL);
+        INIT_LIST_HEAD(&node->prio_list);
+        INIT_LIST_HEAD(&node->node_list);
 }
 
 extern void plist_add(struct plist_node *node, struct plist_head *head);
@@ -188,7 +189,7 @@ extern void plist_del(struct plist_node *node, struct plist_head *head);
  * @head:       the head for your list
  */
 #define plist_for_each(pos, head)       \
-         list_for_each_entry(pos, &(head)->node_list, plist.node_list)
+         list_for_each_entry(pos, &(head)->node_list, node_list)
 
 /**
  * plist_for_each_safe - iterate safely over a plist of given type
@@ -199,7 +200,7 @@ extern void plist_del(struct plist_node *node, struct plist_head *head);
  * Iterate over a plist of given type, safe against removal of list entry.
  */
 #define plist_for_each_safe(pos, n, head)       \
-         list_for_each_entry_safe(pos, n, &(head)->node_list, plist.node_list)
+         list_for_each_entry_safe(pos, n, &(head)->node_list, node_list)
 
 /**
  * plist_for_each_entry - iterate over list of given type
@@ -208,7 +209,7 @@ extern void plist_del(struct plist_node *node, struct plist_head *head);
  * @mem:        the name of the list_struct within the struct
  */
 #define plist_for_each_entry(pos, head, mem)    \
-         list_for_each_entry(pos, &(head)->node_list, mem.plist.node_list)
+         list_for_each_entry(pos, &(head)->node_list, mem.node_list)
 
 /**
  * plist_for_each_entry_safe - iterate safely over list of given type
@@ -220,7 +221,7 @@ extern void plist_del(struct plist_node *node, struct plist_head *head);
  * Iterate over list of given type, safe against removal of list entry.
  */
 #define plist_for_each_entry_safe(pos, n, head, m)      \
-        list_for_each_entry_safe(pos, n, &(head)->node_list, m.plist.node_list)
+        list_for_each_entry_safe(pos, n, &(head)->node_list, m.node_list)
 
 /**
  * plist_head_empty - return !0 if a plist_head is empty
@@ -237,7 +238,7 @@ static inline int plist_head_empty(const struct plist_head *head)
  */
 static inline int plist_node_empty(const struct plist_node *node)
 {
-        return plist_head_empty(&node->plist);
+        return list_empty(&node->node_list);
 }
 
 /* All functions below assume the plist_head is not empty. */
@@ -285,7 +286,7 @@ static inline int plist_node_empty(const struct plist_node *node)
 static inline struct plist_node *plist_first(const struct plist_head *head)
 {
         return list_entry(head->node_list.next,
-                          struct plist_node, plist.node_list);
+                          struct plist_node, node_list);
 }
 
 /**
@@ -297,7 +298,7 @@ static inline struct plist_node *plist_first(const struct plist_head *head)
 static inline struct plist_node *plist_last(const struct plist_head *head)
 {
         return list_entry(head->node_list.prev,
-                          struct plist_node, plist.node_list);
+                          struct plist_node, node_list);
 }
 
 #endif
diff --git a/kernel/futex.c b/kernel/futex.c
index 237f14bfc022..c6bef6e404fe 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -772,6 +772,24 @@ retry:
         return ret;
 }
 
+/**
+ * __unqueue_futex() - Remove the futex_q from its futex_hash_bucket
+ * @q:  The futex_q to unqueue
+ *
+ * The q->lock_ptr must not be NULL and must be held by the caller.
+ */
+static void __unqueue_futex(struct futex_q *q)
+{
+        struct futex_hash_bucket *hb;
+
+        if (WARN_ON(!q->lock_ptr || !spin_is_locked(q->lock_ptr)
+                        || plist_node_empty(&q->list)))
+                return;
+
+        hb = container_of(q->lock_ptr, struct futex_hash_bucket, lock);
+        plist_del(&q->list, &hb->chain);
+}
+
 /*
  * The hash bucket lock must be held when this is called.
  * Afterwards, the futex_q must not be accessed.
@@ -789,7 +807,7 @@ static void wake_futex(struct futex_q *q)
          */
         get_task_struct(p);
 
-        plist_del(&q->list, &q->list.plist);
+        __unqueue_futex(q);
         /*
          * The waiting task can free the futex_q as soon as
          * q->lock_ptr = NULL is written, without taking any locks. A
@@ -1064,9 +1082,6 @@ void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,
                 plist_del(&q->list, &hb1->chain);
                 plist_add(&q->list, &hb2->chain);
                 q->lock_ptr = &hb2->lock;
-#ifdef CONFIG_DEBUG_PI_LIST
-                q->list.plist.spinlock = &hb2->lock;
-#endif
         }
         get_futex_key_refs(key2);
         q->key = *key2;
@@ -1093,16 +1108,12 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,
         get_futex_key_refs(key);
         q->key = *key;
 
-        WARN_ON(plist_node_empty(&q->list));
-        plist_del(&q->list, &q->list.plist);
+        __unqueue_futex(q);
 
         WARN_ON(!q->rt_waiter);
         q->rt_waiter = NULL;
 
         q->lock_ptr = &hb->lock;
-#ifdef CONFIG_DEBUG_PI_LIST
-        q->list.plist.spinlock = &hb->lock;
-#endif
 
         wake_up_state(q->task, TASK_NORMAL);
 }
@@ -1450,9 +1461,6 @@ static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb)
         prio = min(current->normal_prio, MAX_RT_PRIO);
 
         plist_node_init(&q->list, prio);
-#ifdef CONFIG_DEBUG_PI_LIST
-        q->list.plist.spinlock = &hb->lock;
-#endif
         plist_add(&q->list, &hb->chain);
         q->task = current;
         spin_unlock(&hb->lock);
@@ -1497,8 +1505,7 @@ retry:
                         spin_unlock(lock_ptr);
                         goto retry;
                 }
-                WARN_ON(plist_node_empty(&q->list));
-                plist_del(&q->list, &q->list.plist);
+                __unqueue_futex(q);
 
                 BUG_ON(q->pi_state);
 
@@ -1518,8 +1525,7 @@ retry:
 static void unqueue_me_pi(struct futex_q *q)
         __releases(q->lock_ptr)
 {
-        WARN_ON(plist_node_empty(&q->list));
-        plist_del(&q->list, &q->list.plist);
+        __unqueue_futex(q);
 
         BUG_ON(!q->pi_state);
         free_pi_state(q->pi_state);
@@ -2156,7 +2162,7 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
                  * We were woken prior to requeue by a timeout or a signal.
                  * Unqueue the futex_q and determine which it was.
                  */
-                plist_del(&q->list, &q->list.plist);
+                plist_del(&q->list, &hb->chain);
 
                 /* Handle spurious wakeups gracefully */
                 ret = -EWOULDBLOCK;
diff --git a/lib/plist.c b/lib/plist.c
index 1471988d9190..0ae7e6431726 100644
--- a/lib/plist.c
+++ b/lib/plist.c
@@ -28,6 +28,8 @@
 
 #ifdef CONFIG_DEBUG_PI_LIST
 
+static struct plist_head test_head;
+
 static void plist_check_prev_next(struct list_head *t, struct list_head *p,
                                   struct list_head *n)
 {
@@ -54,12 +56,13 @@ static void plist_check_list(struct list_head *top)
 
 static void plist_check_head(struct plist_head *head)
 {
-        WARN_ON(!head->rawlock && !head->spinlock);
+        WARN_ON(head != &test_head && !head->rawlock && !head->spinlock);
         if (head->rawlock)
                 WARN_ON_SMP(!raw_spin_is_locked(head->rawlock));
         if (head->spinlock)
                 WARN_ON_SMP(!spin_is_locked(head->spinlock));
-        plist_check_list(&head->prio_list);
+        if (!plist_head_empty(head))
+                plist_check_list(&plist_first(head)->prio_list);
         plist_check_list(&head->node_list);
 }
 
@@ -75,25 +78,33 @@ static void plist_check_head(struct plist_head *head)
  */
 void plist_add(struct plist_node *node, struct plist_head *head)
 {
-        struct plist_node *iter;
+        struct plist_node *first, *iter, *prev = NULL;
+        struct list_head *node_next = &head->node_list;
 
         plist_check_head(head);
         WARN_ON(!plist_node_empty(node));
+        WARN_ON(!list_empty(&node->prio_list));
+
+        if (plist_head_empty(head))
+                goto ins_node;
 
-        list_for_each_entry(iter, &head->prio_list, plist.prio_list) {
-                if (node->prio < iter->prio)
-                        goto lt_prio;
-                else if (node->prio == iter->prio) {
-                        iter = list_entry(iter->plist.prio_list.next,
-                                        struct plist_node, plist.prio_list);
-                        goto eq_prio;
+        first = iter = plist_first(head);
+
+        do {
+                if (node->prio < iter->prio) {
+                        node_next = &iter->node_list;
+                        break;
                 }
-        }
 
-lt_prio:
-        list_add_tail(&node->plist.prio_list, &iter->plist.prio_list);
-eq_prio:
-        list_add_tail(&node->plist.node_list, &iter->plist.node_list);
+                prev = iter;
+                iter = list_entry(iter->prio_list.next,
+                                struct plist_node, prio_list);
+        } while (iter != first);
+
+        if (!prev || prev->prio != node->prio)
+                list_add_tail(&node->prio_list, &iter->prio_list);
+ins_node:
+        list_add_tail(&node->node_list, node_next);
 
         plist_check_head(head);
 }
@@ -108,14 +119,98 @@ void plist_del(struct plist_node *node, struct plist_head *head)
 {
         plist_check_head(head);
 
-        if (!list_empty(&node->plist.prio_list)) {
-                struct plist_node *next = plist_first(&node->plist);
+        if (!list_empty(&node->prio_list)) {
+                if (node->node_list.next != &head->node_list) {
+                        struct plist_node *next;
+
+                        next = list_entry(node->node_list.next,
+                                        struct plist_node, node_list);
 
-                list_move_tail(&next->plist.prio_list, &node->plist.prio_list);
-                list_del_init(&node->plist.prio_list);
+                        /* add the next plist_node into prio_list */
+                        if (list_empty(&next->prio_list))
+                                list_add(&next->prio_list, &node->prio_list);
+                }
+                list_del_init(&node->prio_list);
         }
 
-        list_del_init(&node->plist.node_list);
+        list_del_init(&node->node_list);
 
         plist_check_head(head);
 }
+
+#ifdef CONFIG_DEBUG_PI_LIST
+#include <linux/sched.h>
+#include <linux/module.h>
+#include <linux/init.h>
+
+static struct plist_node __initdata test_node[241];
+
+static void __init plist_test_check(int nr_expect)
+{
+        struct plist_node *first, *prio_pos, *node_pos;
+
+        if (plist_head_empty(&test_head)) {
+                BUG_ON(nr_expect != 0);
+                return;
+        }
+
+        prio_pos = first = plist_first(&test_head);
+        plist_for_each(node_pos, &test_head) {
+                if (nr_expect-- < 0)
+                        break;
+                if (node_pos == first)
+                        continue;
+                if (node_pos->prio == prio_pos->prio) {
+                        BUG_ON(!list_empty(&node_pos->prio_list));
+                        continue;
+                }
+
+                BUG_ON(prio_pos->prio > node_pos->prio);
+                BUG_ON(prio_pos->prio_list.next != &node_pos->prio_list);
+                prio_pos = node_pos;
+        }
+
+        BUG_ON(nr_expect != 0);
+        BUG_ON(prio_pos->prio_list.next != &first->prio_list);
+}
+
+static int  __init plist_test(void)
+{
+        int nr_expect = 0, i, loop;
+        unsigned int r = local_clock();
+
+        printk(KERN_INFO "start plist test\n");
+        plist_head_init(&test_head, NULL);
+        for (i = 0; i < ARRAY_SIZE(test_node); i++)
+                plist_node_init(test_node + i, 0);
+
+        for (loop = 0; loop < 1000; loop++) {
+                r = r * 193939 % 47629;
+                i = r % ARRAY_SIZE(test_node);
+                if (plist_node_empty(test_node + i)) {
+                        r = r * 193939 % 47629;
+                        test_node[i].prio = r % 99;
+                        plist_add(test_node + i, &test_head);
+                        nr_expect++;
+                } else {
+                        plist_del(test_node + i, &test_head);
+                        nr_expect--;
+                }
+                plist_test_check(nr_expect);
+        }
+
+        for (i = 0; i < ARRAY_SIZE(test_node); i++) {
+                if (plist_node_empty(test_node + i))
+                        continue;
+                plist_del(test_node + i, &test_head);
+                nr_expect--;
+                plist_test_check(nr_expect);
+        }
+
+        printk(KERN_INFO "end plist test\n");
+        return 0;
+}
+
+module_init(plist_test);
+
+#endif