locking/lockdep, sched/core: Implement a better lock pinning scheme

The problem with the existing lock pinning is that each pin is of
value 1; this mean you can simply unpin if you know its pinned,
without having any extra information.

This scheme generates a random (16 bit) cookie for each pin and
requires this same cookie to unpin. This means you have to keep the
cookie in context.

No objsize difference for !LOCKDEP kernels.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

1 files changed, 62 insertions, 9 deletions

diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index ed9410936a22..d7f94f4c811d 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -45,6 +45,7 @@
 #include <linux/bitops.h>
 #include <linux/gfp.h>
 #include <linux/kmemcheck.h>
+#include <linux/random.h>
 
 #include <asm/sections.h>
 
@@ -3554,7 +3555,35 @@ static int __lock_is_held(struct lockdep_map *lock)
         return 0;
 }
 
-static void __lock_pin_lock(struct lockdep_map *lock)
+static struct pin_cookie __lock_pin_lock(struct lockdep_map *lock)
+{
+        struct pin_cookie cookie = NIL_COOKIE;
+        struct task_struct *curr = current;
+        int i;
+
+        if (unlikely(!debug_locks))
+                return cookie;
+
+        for (i = 0; i < curr->lockdep_depth; i++) {
+                struct held_lock *hlock = curr->held_locks + i;
+
+                if (match_held_lock(hlock, lock)) {
+                        /*
+                         * Grab 16bits of randomness; this is sufficient to not
+                         * be guessable and still allows some pin nesting in
+                         * our u32 pin_count.
+                         */
+                        cookie.val = 1 + (prandom_u32() >> 16);
+                        hlock->pin_count += cookie.val;
+                        return cookie;
+                }
+        }
+
+        WARN(1, "pinning an unheld lock\n");
+        return cookie;
+}
+
+static void __lock_repin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
 {
         struct task_struct *curr = current;
         int i;
@@ -3566,7 +3595,7 @@ static void __lock_pin_lock(struct lockdep_map *lock)
                 struct held_lock *hlock = curr->held_locks + i;
 
                 if (match_held_lock(hlock, lock)) {
-                        hlock->pin_count++;
+                        hlock->pin_count += cookie.val;
                         return;
                 }
         }
@@ -3574,7 +3603,7 @@ static void __lock_pin_lock(struct lockdep_map *lock)
         WARN(1, "pinning an unheld lock\n");
 }
 
-static void __lock_unpin_lock(struct lockdep_map *lock)
+static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
 {
         struct task_struct *curr = current;
         int i;
@@ -3589,7 +3618,11 @@ static void __lock_unpin_lock(struct lockdep_map *lock)
                         if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n"))
                                 return;
 
-                        hlock->pin_count--;
+                        hlock->pin_count -= cookie.val;
+
+                        if (WARN((int)hlock->pin_count < 0, "pin count corrupted\n"))
+                                hlock->pin_count = 0;
+
                         return;
                 }
         }
@@ -3720,24 +3753,44 @@ int lock_is_held(struct lockdep_map *lock)
 }
 EXPORT_SYMBOL_GPL(lock_is_held);
 
-void lock_pin_lock(struct lockdep_map *lock)
+struct pin_cookie lock_pin_lock(struct lockdep_map *lock)
 {
+        struct pin_cookie cookie = NIL_COOKIE;
         unsigned long flags;
 
         if (unlikely(current->lockdep_recursion))
-                return;
+                return cookie;
 
         raw_local_irq_save(flags);
         check_flags(flags);
 
         current->lockdep_recursion = 1;
-        __lock_pin_lock(lock);
+        cookie = __lock_pin_lock(lock);
         current->lockdep_recursion = 0;
         raw_local_irq_restore(flags);
+
+        return cookie;
 }
 EXPORT_SYMBOL_GPL(lock_pin_lock);
 
-void lock_unpin_lock(struct lockdep_map *lock)
+void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
+{
+        unsigned long flags;
+
+        if (unlikely(current->lockdep_recursion))
+                return;
+
+        raw_local_irq_save(flags);
+        check_flags(flags);
+
+        current->lockdep_recursion = 1;
+        __lock_repin_lock(lock, cookie);
+        current->lockdep_recursion = 0;
+        raw_local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(lock_repin_lock);
+
+void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
 {
         unsigned long flags;
 
@@ -3748,7 +3801,7 @@ void lock_unpin_lock(struct lockdep_map *lock)
         check_flags(flags);
 
         current->lockdep_recursion = 1;
-        __lock_unpin_lock(lock);
+        __lock_unpin_lock(lock, cookie);
         current->lockdep_recursion = 0;
         raw_local_irq_restore(flags);
 }