3 files changed, 426 insertions, 50 deletions

diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h
index b25d1b53df0d..9ccf0e286b2a 100644
--- a/include/linux/lockdep.h
+++ b/include/linux/lockdep.h
@@ -149,6 +149,12 @@ struct lock_list {
         struct lock_class               *class;
         struct stack_trace              trace;
         int                             distance;
+
+        /*
+         * The parent field is used to implement breadth-first search, and the
+         * bit 0 is reused to indicate if the lock has been accessed in BFS.
+         */
+        struct lock_list                *parent;
 };
 
 /*
@@ -208,10 +214,12 @@ struct held_lock {
          * interrupt context:
          */
         unsigned int irq_context:2; /* bit 0 - soft, bit 1 - hard */
-        unsigned int trylock:1;
+        unsigned int trylock:1;                                         /* 16 bits */
+
         unsigned int read:2;        /* see lock_acquire() comment */
         unsigned int check:2;       /* see lock_acquire() comment */
         unsigned int hardirqs_off:1;
+        unsigned int references:11;                                     /* 32 bits */
 };
 
 /*
@@ -291,6 +299,10 @@ extern void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 extern void lock_release(struct lockdep_map *lock, int nested,
                          unsigned long ip);
 
+#define lockdep_is_held(lock)   lock_is_held(&(lock)->dep_map)
+
+extern int lock_is_held(struct lockdep_map *lock);
+
 extern void lock_set_class(struct lockdep_map *lock, const char *name,
                            struct lock_class_key *key, unsigned int subclass,
                            unsigned long ip);
@@ -309,6 +321,8 @@ extern void lockdep_trace_alloc(gfp_t mask);
 
 #define lockdep_depth(tsk)      (debug_locks ? (tsk)->lockdep_depth : 0)
 
+#define lockdep_assert_held(l)  WARN_ON(debug_locks && !lockdep_is_held(l))
+
 #else /* !LOCKDEP */
 
 static inline void lockdep_off(void)
@@ -353,6 +367,8 @@ struct lock_class_key { };
 
 #define lockdep_depth(tsk)      (0)
 
+#define lockdep_assert_held(l)                  do { } while (0)
+
 #endif /* !LOCKDEP */
 
 #ifdef CONFIG_LOCK_STAT
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index 4be57ab03478..f0ca7a7a1757 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -143,15 +143,6 @@ static inline void smp_mb__after_lock(void) { smp_mb(); }
  */
 #define spin_unlock_wait(lock)  __raw_spin_unlock_wait(&(lock)->raw_lock)
 
-/*
- * Pull the _spin_*()/_read_*()/_write_*() functions/declarations:
- */
-#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
-# include <linux/spinlock_api_smp.h>
-#else
-# include <linux/spinlock_api_up.h>
-#endif
-
 #ifdef CONFIG_DEBUG_SPINLOCK
  extern void _raw_spin_lock(spinlock_t *lock);
 #define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
@@ -268,50 +259,16 @@ static inline void smp_mb__after_lock(void) { smp_mb(); }
 
 #define spin_lock_irq(lock)             _spin_lock_irq(lock)
 #define spin_lock_bh(lock)              _spin_lock_bh(lock)
-
 #define read_lock_irq(lock)             _read_lock_irq(lock)
 #define read_lock_bh(lock)              _read_lock_bh(lock)
-
 #define write_lock_irq(lock)            _write_lock_irq(lock)
 #define write_lock_bh(lock)             _write_lock_bh(lock)
-
-/*
- * We inline the unlock functions in the nondebug case:
- */
-#if defined(CONFIG_DEBUG_SPINLOCK) || defined(CONFIG_PREEMPT) || \
-        !defined(CONFIG_SMP)
-# define spin_unlock(lock)              _spin_unlock(lock)
-# define read_unlock(lock)              _read_unlock(lock)
-# define write_unlock(lock)             _write_unlock(lock)
-# define spin_unlock_irq(lock)          _spin_unlock_irq(lock)
-# define read_unlock_irq(lock)          _read_unlock_irq(lock)
-# define write_unlock_irq(lock)         _write_unlock_irq(lock)
-#else
-# define spin_unlock(lock) \
-    do {__raw_spin_unlock(&(lock)->raw_lock); __release(lock); } while (0)
-# define read_unlock(lock) \
-    do {__raw_read_unlock(&(lock)->raw_lock); __release(lock); } while (0)
-# define write_unlock(lock) \
-    do {__raw_write_unlock(&(lock)->raw_lock); __release(lock); } while (0)
-# define spin_unlock_irq(lock)                  \
-do {                                            \
-        __raw_spin_unlock(&(lock)->raw_lock);   \
-        __release(lock);                        \
-        local_irq_enable();                     \
-} while (0)
-# define read_unlock_irq(lock)                  \
-do {                                            \
-        __raw_read_unlock(&(lock)->raw_lock);   \
-        __release(lock);                        \
-        local_irq_enable();                     \
-} while (0)
-# define write_unlock_irq(lock)                 \
-do {                                            \
-        __raw_write_unlock(&(lock)->raw_lock);  \
-        __release(lock);                        \
-        local_irq_enable();                     \
-} while (0)
-#endif
+#define spin_unlock(lock)               _spin_unlock(lock)
+#define read_unlock(lock)               _read_unlock(lock)
+#define write_unlock(lock)              _write_unlock(lock)
+#define spin_unlock_irq(lock)           _spin_unlock_irq(lock)
+#define read_unlock_irq(lock)           _read_unlock_irq(lock)
+#define write_unlock_irq(lock)          _write_unlock_irq(lock)
 
 #define spin_unlock_irqrestore(lock, flags)             \
         do {                                            \
@@ -380,4 +337,13 @@ extern int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock);
  */
 #define spin_can_lock(lock)     (!spin_is_locked(lock))
 
+/*
+ * Pull the _spin_*()/_read_*()/_write_*() functions/declarations:
+ */
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+# include <linux/spinlock_api_smp.h>
+#else
+# include <linux/spinlock_api_up.h>
+#endif
+
 #endif /* __LINUX_SPINLOCK_H */
diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h
index d79845d034b5..7a7e18fc2415 100644
--- a/include/linux/spinlock_api_smp.h
+++ b/include/linux/spinlock_api_smp.h
@@ -60,4 +60,398 @@ void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
                                                         __releases(lock);
 
+/*
+ * We inline the unlock functions in the nondebug case:
+ */
+#if !defined(CONFIG_DEBUG_SPINLOCK) && !defined(CONFIG_PREEMPT)
+#define __always_inline__spin_unlock
+#define __always_inline__read_unlock
+#define __always_inline__write_unlock
+#define __always_inline__spin_unlock_irq
+#define __always_inline__read_unlock_irq
+#define __always_inline__write_unlock_irq
+#endif
+
+#ifndef CONFIG_DEBUG_SPINLOCK
+#ifndef CONFIG_GENERIC_LOCKBREAK
+
+#ifdef __always_inline__spin_lock
+#define _spin_lock(lock) __spin_lock(lock)
+#endif
+
+#ifdef __always_inline__read_lock
+#define _read_lock(lock) __read_lock(lock)
+#endif
+
+#ifdef __always_inline__write_lock
+#define _write_lock(lock) __write_lock(lock)
+#endif
+
+#ifdef __always_inline__spin_lock_bh
+#define _spin_lock_bh(lock) __spin_lock_bh(lock)
+#endif
+
+#ifdef __always_inline__read_lock_bh
+#define _read_lock_bh(lock) __read_lock_bh(lock)
+#endif
+
+#ifdef __always_inline__write_lock_bh
+#define _write_lock_bh(lock) __write_lock_bh(lock)
+#endif
+
+#ifdef __always_inline__spin_lock_irq
+#define _spin_lock_irq(lock) __spin_lock_irq(lock)
+#endif
+
+#ifdef __always_inline__read_lock_irq
+#define _read_lock_irq(lock) __read_lock_irq(lock)
+#endif
+
+#ifdef __always_inline__write_lock_irq
+#define _write_lock_irq(lock) __write_lock_irq(lock)
+#endif
+
+#ifdef __always_inline__spin_lock_irqsave
+#define _spin_lock_irqsave(lock) __spin_lock_irqsave(lock)
+#endif
+
+#ifdef __always_inline__read_lock_irqsave
+#define _read_lock_irqsave(lock) __read_lock_irqsave(lock)
+#endif
+
+#ifdef __always_inline__write_lock_irqsave
+#define _write_lock_irqsave(lock) __write_lock_irqsave(lock)
+#endif
+
+#endif /* !CONFIG_GENERIC_LOCKBREAK */
+
+#ifdef __always_inline__spin_trylock
+#define _spin_trylock(lock) __spin_trylock(lock)
+#endif
+
+#ifdef __always_inline__read_trylock
+#define _read_trylock(lock) __read_trylock(lock)
+#endif
+
+#ifdef __always_inline__write_trylock
+#define _write_trylock(lock) __write_trylock(lock)
+#endif
+
+#ifdef __always_inline__spin_trylock_bh
+#define _spin_trylock_bh(lock) __spin_trylock_bh(lock)
+#endif
+
+#ifdef __always_inline__spin_unlock
+#define _spin_unlock(lock) __spin_unlock(lock)
+#endif
+
+#ifdef __always_inline__read_unlock
+#define _read_unlock(lock) __read_unlock(lock)
+#endif
+
+#ifdef __always_inline__write_unlock
+#define _write_unlock(lock) __write_unlock(lock)
+#endif
+
+#ifdef __always_inline__spin_unlock_bh
+#define _spin_unlock_bh(lock) __spin_unlock_bh(lock)
+#endif
+
+#ifdef __always_inline__read_unlock_bh
+#define _read_unlock_bh(lock) __read_unlock_bh(lock)
+#endif
+
+#ifdef __always_inline__write_unlock_bh
+#define _write_unlock_bh(lock) __write_unlock_bh(lock)
+#endif
+
+#ifdef __always_inline__spin_unlock_irq
+#define _spin_unlock_irq(lock) __spin_unlock_irq(lock)
+#endif
+
+#ifdef __always_inline__read_unlock_irq
+#define _read_unlock_irq(lock) __read_unlock_irq(lock)
+#endif
+
+#ifdef __always_inline__write_unlock_irq
+#define _write_unlock_irq(lock) __write_unlock_irq(lock)
+#endif
+
+#ifdef __always_inline__spin_unlock_irqrestore
+#define _spin_unlock_irqrestore(lock, flags) __spin_unlock_irqrestore(lock, flags)
+#endif
+
+#ifdef __always_inline__read_unlock_irqrestore
+#define _read_unlock_irqrestore(lock, flags) __read_unlock_irqrestore(lock, flags)
+#endif
+
+#ifdef __always_inline__write_unlock_irqrestore
+#define _write_unlock_irqrestore(lock, flags) __write_unlock_irqrestore(lock, flags)
+#endif
+
+#endif /* CONFIG_DEBUG_SPINLOCK */
+
+static inline int __spin_trylock(spinlock_t *lock)
+{
+        preempt_disable();
+        if (_raw_spin_trylock(lock)) {
+                spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
+                return 1;
+        }
+        preempt_enable();
+        return 0;
+}
+
+static inline int __read_trylock(rwlock_t *lock)
+{
+        preempt_disable();
+        if (_raw_read_trylock(lock)) {
+                rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
+                return 1;
+        }
+        preempt_enable();
+        return 0;
+}
+
+static inline int __write_trylock(rwlock_t *lock)
+{
+        preempt_disable();
+        if (_raw_write_trylock(lock)) {
+                rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
+                return 1;
+        }
+        preempt_enable();
+        return 0;
+}
+
+/*
+ * If lockdep is enabled then we use the non-preemption spin-ops
+ * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
+ * not re-enabled during lock-acquire (which the preempt-spin-ops do):
+ */
+#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
+
+static inline void __read_lock(rwlock_t *lock)
+{
+        preempt_disable();
+        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
+        LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
+}
+
+static inline unsigned long __spin_lock_irqsave(spinlock_t *lock)
+{
+        unsigned long flags;
+
+        local_irq_save(flags);
+        preempt_disable();
+        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
+        /*
+         * On lockdep we dont want the hand-coded irq-enable of
+         * _raw_spin_lock_flags() code, because lockdep assumes
+         * that interrupts are not re-enabled during lock-acquire:
+         */
+#ifdef CONFIG_LOCKDEP
+        LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
+#else
+        _raw_spin_lock_flags(lock, &flags);
+#endif
+        return flags;
+}
+
+static inline void __spin_lock_irq(spinlock_t *lock)
+{
+        local_irq_disable();
+        preempt_disable();
+        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
+        LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
+}
+
+static inline void __spin_lock_bh(spinlock_t *lock)
+{
+        local_bh_disable();
+        preempt_disable();
+        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
+        LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
+}
+
+static inline unsigned long __read_lock_irqsave(rwlock_t *lock)
+{
+        unsigned long flags;
+
+        local_irq_save(flags);
+        preempt_disable();
+        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
+        LOCK_CONTENDED_FLAGS(lock, _raw_read_trylock, _raw_read_lock,
+                             _raw_read_lock_flags, &flags);
+        return flags;
+}
+
+static inline void __read_lock_irq(rwlock_t *lock)
+{
+        local_irq_disable();
+        preempt_disable();
+        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
+        LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
+}
+
+static inline void __read_lock_bh(rwlock_t *lock)
+{
+        local_bh_disable();
+        preempt_disable();
+        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
+        LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
+}
+
+static inline unsigned long __write_lock_irqsave(rwlock_t *lock)
+{
+        unsigned long flags;
+
+        local_irq_save(flags);
+        preempt_disable();
+        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
+        LOCK_CONTENDED_FLAGS(lock, _raw_write_trylock, _raw_write_lock,
+                             _raw_write_lock_flags, &flags);
+        return flags;
+}
+
+static inline void __write_lock_irq(rwlock_t *lock)
+{
+        local_irq_disable();
+        preempt_disable();
+        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
+        LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
+}
+
+static inline void __write_lock_bh(rwlock_t *lock)
+{
+        local_bh_disable();
+        preempt_disable();
+        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
+        LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
+}
+
+static inline void __spin_lock(spinlock_t *lock)
+{
+        preempt_disable();
+        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
+        LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
+}
+
+static inline void __write_lock(rwlock_t *lock)
+{
+        preempt_disable();
+        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
+        LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
+}
+
+#endif /* CONFIG_PREEMPT */
+
+static inline void __spin_unlock(spinlock_t *lock)
+{
+        spin_release(&lock->dep_map, 1, _RET_IP_);
+        _raw_spin_unlock(lock);
+        preempt_enable();
+}
+
+static inline void __write_unlock(rwlock_t *lock)
+{
+        rwlock_release(&lock->dep_map, 1, _RET_IP_);
+        _raw_write_unlock(lock);
+        preempt_enable();
+}
+
+static inline void __read_unlock(rwlock_t *lock)
+{
+        rwlock_release(&lock->dep_map, 1, _RET_IP_);
+        _raw_read_unlock(lock);
+        preempt_enable();
+}
+
+static inline void __spin_unlock_irqrestore(spinlock_t *lock,
+                                            unsigned long flags)
+{
+        spin_release(&lock->dep_map, 1, _RET_IP_);
+        _raw_spin_unlock(lock);
+        local_irq_restore(flags);
+        preempt_enable();
+}
+
+static inline void __spin_unlock_irq(spinlock_t *lock)
+{
+        spin_release(&lock->dep_map, 1, _RET_IP_);
+        _raw_spin_unlock(lock);
+        local_irq_enable();
+        preempt_enable();
+}
+
+static inline void __spin_unlock_bh(spinlock_t *lock)
+{
+        spin_release(&lock->dep_map, 1, _RET_IP_);
+        _raw_spin_unlock(lock);
+        preempt_enable_no_resched();
+        local_bh_enable_ip((unsigned long)__builtin_return_address(0));
+}
+
+static inline void __read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
+{
+        rwlock_release(&lock->dep_map, 1, _RET_IP_);
+        _raw_read_unlock(lock);
+        local_irq_restore(flags);
+        preempt_enable();
+}
+
+static inline void __read_unlock_irq(rwlock_t *lock)
+{
+        rwlock_release(&lock->dep_map, 1, _RET_IP_);
+        _raw_read_unlock(lock);
+        local_irq_enable();
+        preempt_enable();
+}
+
+static inline void __read_unlock_bh(rwlock_t *lock)
+{
+        rwlock_release(&lock->dep_map, 1, _RET_IP_);
+        _raw_read_unlock(lock);
+        preempt_enable_no_resched();
+        local_bh_enable_ip((unsigned long)__builtin_return_address(0));
+}
+
+static inline void __write_unlock_irqrestore(rwlock_t *lock,
+                                             unsigned long flags)
+{
+        rwlock_release(&lock->dep_map, 1, _RET_IP_);
+        _raw_write_unlock(lock);
+        local_irq_restore(flags);
+        preempt_enable();
+}
+
+static inline void __write_unlock_irq(rwlock_t *lock)
+{
+        rwlock_release(&lock->dep_map, 1, _RET_IP_);
+        _raw_write_unlock(lock);
+        local_irq_enable();
+        preempt_enable();
+}
+
+static inline void __write_unlock_bh(rwlock_t *lock)
+{
+        rwlock_release(&lock->dep_map, 1, _RET_IP_);
+        _raw_write_unlock(lock);
+        preempt_enable_no_resched();
+        local_bh_enable_ip((unsigned long)__builtin_return_address(0));
+}
+
+static inline int __spin_trylock_bh(spinlock_t *lock)
+{
+        local_bh_disable();
+        preempt_disable();
+        if (_raw_spin_trylock(lock)) {
+                spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
+                return 1;
+        }
+        preempt_enable_no_resched();
+        local_bh_enable_ip((unsigned long)__builtin_return_address(0));
+        return 0;
+}
+
 #endif /* __LINUX_SPINLOCK_API_SMP_H */