kernel: add bl_list

Introduce a type of hlist that can support the use of the lowest bit in the
hlist_head. This will be subsequently used to implement per-bucket bit spinlock
for inode and dentry hashes, and may be useful in other cases such as network
hashes.

Reviewed-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Nick Piggin <npiggin@kernel.dk>

2 files changed, 271 insertions, 0 deletions

diff --git a/include/linux/list_bl.h b/include/linux/list_bl.h
new file mode 100644
index 000000000000..9ee97e7f2be4
--- /dev/null
+++ b/include/linux/list_bl.h
@@ -0,0 +1,144 @@
+#ifndef _LINUX_LIST_BL_H
+#define _LINUX_LIST_BL_H
+
+#include <linux/list.h>
+
+/*
+ * Special version of lists, where head of the list has a lock in the lowest
+ * bit. This is useful for scalable hash tables without increasing memory
+ * footprint overhead.
+ *
+ * For modification operations, the 0 bit of hlist_bl_head->first
+ * pointer must be set.
+ *
+ * With some small modifications, this can easily be adapted to store several
+ * arbitrary bits (not just a single lock bit), if the need arises to store
+ * some fast and compact auxiliary data.
+ */
+
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+#define LIST_BL_LOCKMASK        1UL
+#else
+#define LIST_BL_LOCKMASK        0UL
+#endif
+
+#ifdef CONFIG_DEBUG_LIST
+#define LIST_BL_BUG_ON(x) BUG_ON(x)
+#else
+#define LIST_BL_BUG_ON(x)
+#endif
+
+
+struct hlist_bl_head {
+        struct hlist_bl_node *first;
+};
+
+struct hlist_bl_node {
+        struct hlist_bl_node *next, **pprev;
+};
+#define INIT_HLIST_BL_HEAD(ptr) \
+        ((ptr)->first = NULL)
+
+static inline void INIT_HLIST_BL_NODE(struct hlist_bl_node *h)
+{
+        h->next = NULL;
+        h->pprev = NULL;
+}
+
+#define hlist_bl_entry(ptr, type, member) container_of(ptr,type,member)
+
+static inline int hlist_bl_unhashed(const struct hlist_bl_node *h)
+{
+        return !h->pprev;
+}
+
+static inline struct hlist_bl_node *hlist_bl_first(struct hlist_bl_head *h)
+{
+        return (struct hlist_bl_node *)
+                ((unsigned long)h->first & ~LIST_BL_LOCKMASK);
+}
+
+static inline void hlist_bl_set_first(struct hlist_bl_head *h,
+                                        struct hlist_bl_node *n)
+{
+        LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK);
+        LIST_BL_BUG_ON(!((unsigned long)h->first & LIST_BL_LOCKMASK));
+        h->first = (struct hlist_bl_node *)((unsigned long)n | LIST_BL_LOCKMASK);
+}
+
+static inline int hlist_bl_empty(const struct hlist_bl_head *h)
+{
+        return !((unsigned long)h->first & ~LIST_BL_LOCKMASK);
+}
+
+static inline void hlist_bl_add_head(struct hlist_bl_node *n,
+                                        struct hlist_bl_head *h)
+{
+        struct hlist_bl_node *first = hlist_bl_first(h);
+
+        n->next = first;
+        if (first)
+                first->pprev = &n->next;
+        n->pprev = &h->first;
+        hlist_bl_set_first(h, n);
+}
+
+static inline void __hlist_bl_del(struct hlist_bl_node *n)
+{
+        struct hlist_bl_node *next = n->next;
+        struct hlist_bl_node **pprev = n->pprev;
+
+        LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK);
+
+        /* pprev may be `first`, so be careful not to lose the lock bit */
+        *pprev = (struct hlist_bl_node *)
+                        ((unsigned long)next |
+                         ((unsigned long)*pprev & LIST_BL_LOCKMASK));
+        if (next)
+                next->pprev = pprev;
+}
+
+static inline void hlist_bl_del(struct hlist_bl_node *n)
+{
+        __hlist_bl_del(n);
+        n->next = LIST_POISON1;
+        n->pprev = LIST_POISON2;
+}
+
+static inline void hlist_bl_del_init(struct hlist_bl_node *n)
+{
+        if (!hlist_bl_unhashed(n)) {
+                __hlist_bl_del(n);
+                INIT_HLIST_BL_NODE(n);
+        }
+}
+
+/**
+ * hlist_bl_for_each_entry      - iterate over list of given type
+ * @tpos:       the type * to use as a loop cursor.
+ * @pos:        the &struct hlist_node to use as a loop cursor.
+ * @head:       the head for your list.
+ * @member:     the name of the hlist_node within the struct.
+ *
+ */
+#define hlist_bl_for_each_entry(tpos, pos, head, member)                \
+        for (pos = hlist_bl_first(head);                                \
+             pos &&                                                     \
+                ({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1;}); \
+             pos = pos->next)
+
+/**
+ * hlist_bl_for_each_entry_safe - iterate over list of given type safe against removal of list entry
+ * @tpos:       the type * to use as a loop cursor.
+ * @pos:        the &struct hlist_node to use as a loop cursor.
+ * @n:          another &struct hlist_node to use as temporary storage
+ * @head:       the head for your list.
+ * @member:     the name of the hlist_node within the struct.
+ */
+#define hlist_bl_for_each_entry_safe(tpos, pos, n, head, member)         \
+        for (pos = hlist_bl_first(head);                                 \
+             pos && ({ n = pos->next; 1; }) &&                           \
+                ({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1;}); \
+             pos = n)
+
+#endif
diff --git a/include/linux/rculist_bl.h b/include/linux/rculist_bl.h
new file mode 100644
index 000000000000..b872b493724d
--- /dev/null
+++ b/include/linux/rculist_bl.h
@@ -0,0 +1,127 @@
+#ifndef _LINUX_RCULIST_BL_H
+#define _LINUX_RCULIST_BL_H
+
+/*
+ * RCU-protected bl list version. See include/linux/list_bl.h.
+ */
+#include <linux/list_bl.h>
+#include <linux/rcupdate.h>
+
+static inline void hlist_bl_set_first_rcu(struct hlist_bl_head *h,
+                                        struct hlist_bl_node *n)
+{
+        LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK);
+        LIST_BL_BUG_ON(!((unsigned long)h->first & LIST_BL_LOCKMASK));
+        rcu_assign_pointer(h->first,
+                (struct hlist_bl_node *)((unsigned long)n | LIST_BL_LOCKMASK));
+}
+
+static inline struct hlist_bl_node *hlist_bl_first_rcu(struct hlist_bl_head *h)
+{
+        return (struct hlist_bl_node *)
+                ((unsigned long)rcu_dereference(h->first) & ~LIST_BL_LOCKMASK);
+}
+
+/**
+ * hlist_bl_del_init_rcu - deletes entry from hash list with re-initialization
+ * @n: the element to delete from the hash list.
+ *
+ * Note: hlist_bl_unhashed() on the node returns true after this. It is
+ * useful for RCU based read lockfree traversal if the writer side
+ * must know if the list entry is still hashed or already unhashed.
+ *
+ * In particular, it means that we can not poison the forward pointers
+ * that may still be used for walking the hash list and we can only
+ * zero the pprev pointer so list_unhashed() will return true after
+ * this.
+ *
+ * The caller must take whatever precautions are necessary (such as
+ * holding appropriate locks) to avoid racing with another
+ * list-mutation primitive, such as hlist_bl_add_head_rcu() or
+ * hlist_bl_del_rcu(), running on this same list.  However, it is
+ * perfectly legal to run concurrently with the _rcu list-traversal
+ * primitives, such as hlist_bl_for_each_entry_rcu().
+ */
+static inline void hlist_bl_del_init_rcu(struct hlist_bl_node *n)
+{
+        if (!hlist_bl_unhashed(n)) {
+                __hlist_bl_del(n);
+                n->pprev = NULL;
+        }
+}
+
+/**
+ * hlist_bl_del_rcu - deletes entry from hash list without re-initialization
+ * @n: the element to delete from the hash list.
+ *
+ * Note: hlist_bl_unhashed() on entry does not return true after this,
+ * the entry is in an undefined state. It is useful for RCU based
+ * lockfree traversal.
+ *
+ * In particular, it means that we can not poison the forward
+ * pointers that may still be used for walking the hash list.
+ *
+ * The caller must take whatever precautions are necessary
+ * (such as holding appropriate locks) to avoid racing
+ * with another list-mutation primitive, such as hlist_bl_add_head_rcu()
+ * or hlist_bl_del_rcu(), running on this same list.
+ * However, it is perfectly legal to run concurrently with
+ * the _rcu list-traversal primitives, such as
+ * hlist_bl_for_each_entry().
+ */
+static inline void hlist_bl_del_rcu(struct hlist_bl_node *n)
+{
+        __hlist_bl_del(n);
+        n->pprev = LIST_POISON2;
+}
+
+/**
+ * hlist_bl_add_head_rcu
+ * @n: the element to add to the hash list.
+ * @h: the list to add to.
+ *
+ * Description:
+ * Adds the specified element to the specified hlist_bl,
+ * while permitting racing traversals.
+ *
+ * The caller must take whatever precautions are necessary
+ * (such as holding appropriate locks) to avoid racing
+ * with another list-mutation primitive, such as hlist_bl_add_head_rcu()
+ * or hlist_bl_del_rcu(), running on this same list.
+ * However, it is perfectly legal to run concurrently with
+ * the _rcu list-traversal primitives, such as
+ * hlist_bl_for_each_entry_rcu(), used to prevent memory-consistency
+ * problems on Alpha CPUs.  Regardless of the type of CPU, the
+ * list-traversal primitive must be guarded by rcu_read_lock().
+ */
+static inline void hlist_bl_add_head_rcu(struct hlist_bl_node *n,
+                                        struct hlist_bl_head *h)
+{
+        struct hlist_bl_node *first;
+
+        /* don't need hlist_bl_first_rcu because we're under lock */
+        first = hlist_bl_first(h);
+
+        n->next = first;
+        if (first)
+                first->pprev = &n->next;
+        n->pprev = &h->first;
+
+        /* need _rcu because we can have concurrent lock free readers */
+        hlist_bl_set_first_rcu(h, n);
+}
+/**
+ * hlist_bl_for_each_entry_rcu - iterate over rcu list of given type
+ * @tpos:       the type * to use as a loop cursor.
+ * @pos:        the &struct hlist_bl_node to use as a loop cursor.
+ * @head:       the head for your list.
+ * @member:     the name of the hlist_bl_node within the struct.
+ *
+ */
+#define hlist_bl_for_each_entry_rcu(tpos, pos, head, member)            \
+        for (pos = hlist_bl_first_rcu(head);                            \
+                pos &&                                                  \
+                ({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1; }); \
+                pos = rcu_dereference_raw(pos->next))
+
+#endif