rbtree: move augmented rbtree functionality to rbtree_augmented.h

Provide rb_insert_augmented() and rb_erase_augmented() through a new
rbtree_augmented.h include file.  rb_erase_augmented() is defined there as
an __always_inline function, in order to allow inlining of augmented
rbtree callbacks into it.  Since this generates a relatively large
function, each augmented rbtree user should make sure to have a single
call site.

Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Hillf Danton <dhillf@gmail.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

2 files changed, 12 insertions, 152 deletions

diff --git a/lib/rbtree.c b/lib/rbtree.c
index c0088ca345f9..4f56a11d67fa 100644
--- a/lib/rbtree.c
+++ b/lib/rbtree.c
@@ -21,7 +21,7 @@
   linux/lib/rbtree.c
 */
 
-#include <linux/rbtree.h>
+#include <linux/rbtree_augmented.h>
 #include <linux/export.h>
 
 /*
@@ -44,52 +44,16 @@
  *  parentheses and have some accompanying text comment.
  */
 
-#define RB_RED          0
-#define RB_BLACK        1
-
-#define __rb_parent(pc)    ((struct rb_node *)(pc & ~3))
-
-#define __rb_color(pc)     ((pc) & 1)
-#define __rb_is_black(pc)  __rb_color(pc)
-#define __rb_is_red(pc)    (!__rb_color(pc))
-#define rb_color(rb)       __rb_color((rb)->__rb_parent_color)
-#define rb_is_red(rb)      __rb_is_red((rb)->__rb_parent_color)
-#define rb_is_black(rb)    __rb_is_black((rb)->__rb_parent_color)
-
 static inline void rb_set_black(struct rb_node *rb)
 {
         rb->__rb_parent_color |= RB_BLACK;
 }
 
-static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p)
-{
-        rb->__rb_parent_color = rb_color(rb) | (unsigned long)p;
-}
-
-static inline void rb_set_parent_color(struct rb_node *rb,
-                                       struct rb_node *p, int color)
-{
-        rb->__rb_parent_color = (unsigned long)p | color;
-}
-
 static inline struct rb_node *rb_red_parent(struct rb_node *red)
 {
         return (struct rb_node *)red->__rb_parent_color;
 }
 
-static inline void
-__rb_change_child(struct rb_node *old, struct rb_node *new,
-                  struct rb_node *parent, struct rb_root *root)
-{
-        if (parent) {
-                if (parent->rb_left == old)
-                        parent->rb_left = new;
-                else
-                        parent->rb_right = new;
-        } else
-                root->rb_node = new;
-}
-
 /*
  * Helper function for rotations:
  * - old's parent and color get assigned to new
@@ -230,9 +194,9 @@ __rb_insert(struct rb_node *node, struct rb_root *root,
         }
 }
 
-static __always_inline void
+__always_inline void
 __rb_erase_color(struct rb_node *parent, struct rb_root *root,
-                 const struct rb_augment_callbacks *augment)
+        void (*augment_rotate)(struct rb_node *old, struct rb_node *new))
 {
         struct rb_node *node = NULL, *sibling, *tmp1, *tmp2;
 
@@ -261,7 +225,7 @@ __rb_erase_color(struct rb_node *parent, struct rb_root *root,
                                 rb_set_parent_color(tmp1, parent, RB_BLACK);
                                 __rb_rotate_set_parents(parent, sibling, root,
                                                         RB_RED);
-                                augment->rotate(parent, sibling);
+                                augment_rotate(parent, sibling);
                                 sibling = tmp1;
                         }
                         tmp1 = sibling->rb_right;
@@ -313,7 +277,7 @@ __rb_erase_color(struct rb_node *parent, struct rb_root *root,
                                 if (tmp1)
                                         rb_set_parent_color(tmp1, sibling,
                                                             RB_BLACK);
-                                augment->rotate(sibling, tmp2);
+                                augment_rotate(sibling, tmp2);
                                 tmp1 = sibling;
                                 sibling = tmp2;
                         }
@@ -336,7 +300,7 @@ __rb_erase_color(struct rb_node *parent, struct rb_root *root,
                                 rb_set_parent(tmp2, parent);
                         __rb_rotate_set_parents(parent, sibling, root,
                                                 RB_BLACK);
-                        augment->rotate(parent, sibling);
+                        augment_rotate(parent, sibling);
                         break;
                 } else {
                         sibling = parent->rb_left;
@@ -347,7 +311,7 @@ __rb_erase_color(struct rb_node *parent, struct rb_root *root,
                                 rb_set_parent_color(tmp1, parent, RB_BLACK);
                                 __rb_rotate_set_parents(parent, sibling, root,
                                                         RB_RED);
-                                augment->rotate(parent, sibling);
+                                augment_rotate(parent, sibling);
                                 sibling = tmp1;
                         }
                         tmp1 = sibling->rb_left;
@@ -374,7 +338,7 @@ __rb_erase_color(struct rb_node *parent, struct rb_root *root,
                                 if (tmp1)
                                         rb_set_parent_color(tmp1, sibling,
                                                             RB_BLACK);
-                                augment->rotate(sibling, tmp2);
+                                augment_rotate(sibling, tmp2);
                                 tmp1 = sibling;
                                 sibling = tmp2;
                         }
@@ -386,109 +350,12 @@ __rb_erase_color(struct rb_node *parent, struct rb_root *root,
                                 rb_set_parent(tmp2, parent);
                         __rb_rotate_set_parents(parent, sibling, root,
                                                 RB_BLACK);
-                        augment->rotate(parent, sibling);
+                        augment_rotate(parent, sibling);
                         break;
                 }
         }
 }
-
-static __always_inline void
-__rb_erase(struct rb_node *node, struct rb_root *root,
-           const struct rb_augment_callbacks *augment)
-{
-        struct rb_node *child = node->rb_right, *tmp = node->rb_left;
-        struct rb_node *parent, *rebalance;
-        unsigned long pc;
-
-        if (!tmp) {
-                /*
-                 * Case 1: node to erase has no more than 1 child (easy!)
-                 *
-                 * Note that if there is one child it must be red due to 5)
-                 * and node must be black due to 4). We adjust colors locally
-                 * so as to bypass __rb_erase_color() later on.
-                 */
-                pc = node->__rb_parent_color;
-                parent = __rb_parent(pc);
-                __rb_change_child(node, child, parent, root);
-                if (child) {
-                        child->__rb_parent_color = pc;
-                        rebalance = NULL;
-                } else
-                        rebalance = __rb_is_black(pc) ? parent : NULL;
-                tmp = parent;
-        } else if (!child) {
-                /* Still case 1, but this time the child is node->rb_left */
-                tmp->__rb_parent_color = pc = node->__rb_parent_color;
-                parent = __rb_parent(pc);
-                __rb_change_child(node, tmp, parent, root);
-                rebalance = NULL;
-                tmp = parent;
-        } else {
-                struct rb_node *successor = child, *child2;
-                tmp = child->rb_left;
-                if (!tmp) {
-                        /*
-                         * Case 2: node's successor is its right child
-                         *
-                         *    (n)          (s)
-                         *    / \          / \
-                         *  (x) (s)  ->  (x) (c)
-                         *        \
-                         *        (c)
-                         */
-                        parent = successor;
-                        child2 = successor->rb_right;
-                        augment->copy(node, successor);
-                } else {
-                        /*
-                         * Case 3: node's successor is leftmost under
-                         * node's right child subtree
-                         *
-                         *    (n)          (s)
-                         *    / \          / \
-                         *  (x) (y)  ->  (x) (y)
-                         *      /            /
-                         *    (p)          (p)
-                         *    /            /
-                         *  (s)          (c)
-                         *    \
-                         *    (c)
-                         */
-                        do {
-                                parent = successor;
-                                successor = tmp;
-                                tmp = tmp->rb_left;
-                        } while (tmp);
-                        parent->rb_left = child2 = successor->rb_right;
-                        successor->rb_right = child;
-                        rb_set_parent(child, successor);
-                        augment->copy(node, successor);
-                        augment->propagate(parent, successor);
-                }
-
-                successor->rb_left = tmp = node->rb_left;
-                rb_set_parent(tmp, successor);
-
-                pc = node->__rb_parent_color;
-                tmp = __rb_parent(pc);
-                __rb_change_child(node, successor, tmp, root);
-                if (child2) {
-                        successor->__rb_parent_color = pc;
-                        rb_set_parent_color(child2, parent, RB_BLACK);
-                        rebalance = NULL;
-                } else {
-                        unsigned long pc2 = successor->__rb_parent_color;
-                        successor->__rb_parent_color = pc;
-                        rebalance = __rb_is_black(pc2) ? parent : NULL;
-                }
-                tmp = successor;
-        }
-
-        augment->propagate(tmp, NULL);
-        if (rebalance)
-                __rb_erase_color(rebalance, root, augment);
-}
+EXPORT_SYMBOL(__rb_erase_color);
 
 /*
  * Non-augmented rbtree manipulation functions.
@@ -513,7 +380,7 @@ EXPORT_SYMBOL(rb_insert_color);
 
 void rb_erase(struct rb_node *node, struct rb_root *root)
 {
-        __rb_erase(node, root, &dummy_callbacks);
+        rb_erase_augmented(node, root, &dummy_callbacks);
 }
 EXPORT_SYMBOL(rb_erase);
 
@@ -531,13 +398,6 @@ void __rb_insert_augmented(struct rb_node *node, struct rb_root *root,
 }
 EXPORT_SYMBOL(__rb_insert_augmented);
 
-void rb_erase_augmented(struct rb_node *node, struct rb_root *root,
-                        const struct rb_augment_callbacks *augment)
-{
-        __rb_erase(node, root, augment);
-}
-EXPORT_SYMBOL(rb_erase_augmented);
-
 /*
  * This function returns the first node (in sort order) of the tree.
  */
diff --git a/lib/rbtree_test.c b/lib/rbtree_test.c
index b20e99969b0f..268b23951fec 100644
--- a/lib/rbtree_test.c
+++ b/lib/rbtree_test.c
@@ -1,5 +1,5 @@
 #include <linux/module.h>
-#include <linux/rbtree.h>
+#include <linux/rbtree_augmented.h>
 #include <linux/random.h>
 #include <asm/timex.h>