2 files changed, 31 insertions, 11 deletions

diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h
index eb7111039247..20f9c6af7473 100644
--- a/include/linux/rhashtable.h
+++ b/include/linux/rhashtable.h
@@ -75,8 +75,19 @@ struct bucket_table {
         struct rhash_head __rcu *buckets[] ____cacheline_aligned_in_smp;
 };
 
+/*
+ * NULLS_MARKER() expects a hash value with the low
+ * bits mostly likely to be significant, and it discards
+ * the msb.
+ * We git it an address, in which the bottom 2 bits are
+ * always 0, and the msb might be significant.
+ * So we shift the address down one bit to align with
+ * expectations and avoid losing a significant bit.
+ */
+#define RHT_NULLS_MARKER(ptr)   \
+        ((void *)NULLS_MARKER(((unsigned long) (ptr)) >> 1))
 #define INIT_RHT_NULLS_HEAD(ptr)        \
-        ((ptr) = (typeof(ptr)) NULLS_MARKER(0))
+        ((ptr) = RHT_NULLS_MARKER(&(ptr)))
 
 static inline bool rht_is_a_nulls(const struct rhash_head *ptr)
 {
@@ -471,6 +482,7 @@ static inline struct rhash_head *__rhashtable_lookup(
                 .ht = ht,
                 .key = key,
         };
+        struct rhash_head __rcu * const *head;
         struct bucket_table *tbl;
         struct rhash_head *he;
         unsigned int hash;
@@ -478,13 +490,19 @@ static inline struct rhash_head *__rhashtable_lookup(
         tbl = rht_dereference_rcu(ht->tbl, ht);
 restart:
         hash = rht_key_hashfn(ht, tbl, key, params);
-        rht_for_each_rcu(he, tbl, hash) {
-                if (params.obj_cmpfn ?
-                    params.obj_cmpfn(&arg, rht_obj(ht, he)) :
-                    rhashtable_compare(&arg, rht_obj(ht, he)))
-                        continue;
-                return he;
-        }
+        head = rht_bucket(tbl, hash);
+        do {
+                rht_for_each_rcu_continue(he, *head, tbl, hash) {
+                        if (params.obj_cmpfn ?
+                            params.obj_cmpfn(&arg, rht_obj(ht, he)) :
+                            rhashtable_compare(&arg, rht_obj(ht, he)))
+                                continue;
+                        return he;
+                }
+                /* An object might have been moved to a different hash chain,
+                 * while we walk along it - better check and retry.
+                 */
+        } while (he != RHT_NULLS_MARKER(head));
 
         /* Ensure we see any new tables. */
         smp_rmb();
diff --git a/lib/rhashtable.c b/lib/rhashtable.c
index 30526afa8343..852ffa5160f1 100644
--- a/lib/rhashtable.c
+++ b/lib/rhashtable.c
@@ -1179,8 +1179,7 @@ struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl,
                                             unsigned int hash)
 {
         const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *));
-        static struct rhash_head __rcu *rhnull =
-                (struct rhash_head __rcu *)NULLS_MARKER(0);
+        static struct rhash_head __rcu *rhnull;
         unsigned int index = hash & ((1 << tbl->nest) - 1);
         unsigned int size = tbl->size >> tbl->nest;
         unsigned int subhash = hash;
@@ -1198,8 +1197,11 @@ struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl,
                 subhash >>= shift;
         }
 
-        if (!ntbl)
+        if (!ntbl) {
+                if (!rhnull)
+                        INIT_RHT_NULLS_HEAD(rhnull);
                 return &rhnull;
+        }
 
         return &ntbl[subhash].bucket;