Merge branch 'core-locking-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'core-locking-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip:
  lockdep: Reduce stack_trace usage
  lockdep: No need to disable preemption in debug atomic ops
  lockdep: Actually _dec_ in debug_atomic_dec
  lockdep: Provide off case for redundant_hardirqs_on increment
  lockdep: Simplify debug atomic ops
  lockdep: Fix redundant_hardirqs_on incremented with irqs enabled
  lockstat: Make lockstat counting per cpu
  i8253: Convert i8253_lock to raw_spinlock

3 files changed, 123 insertions, 88 deletions

diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index 2594e1ce41cb..51080807dc8c 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -431,20 +431,7 @@ static struct stack_trace lockdep_init_trace = {
 /*
  * Various lockdep statistics:
  */
-atomic_t chain_lookup_hits;
-atomic_t chain_lookup_misses;
-atomic_t hardirqs_on_events;
-atomic_t hardirqs_off_events;
-atomic_t redundant_hardirqs_on;
-atomic_t redundant_hardirqs_off;
-atomic_t softirqs_on_events;
-atomic_t softirqs_off_events;
-atomic_t redundant_softirqs_on;
-atomic_t redundant_softirqs_off;
-atomic_t nr_unused_locks;
-atomic_t nr_cyclic_checks;
-atomic_t nr_find_usage_forwards_checks;
-atomic_t nr_find_usage_backwards_checks;
+DEFINE_PER_CPU(struct lockdep_stats, lockdep_stats);
 #endif
 
 /*
@@ -748,7 +735,7 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force)
                 return NULL;
         }
         class = lock_classes + nr_lock_classes++;
-        debug_atomic_inc(&nr_unused_locks);
+        debug_atomic_inc(nr_unused_locks);
         class->key = key;
         class->name = lock->name;
         class->subclass = subclass;
@@ -818,7 +805,8 @@ static struct lock_list *alloc_list_entry(void)
  * Add a new dependency to the head of the list:
  */
 static int add_lock_to_list(struct lock_class *class, struct lock_class *this,
-                            struct list_head *head, unsigned long ip, int distance)
+                            struct list_head *head, unsigned long ip,
+                            int distance, struct stack_trace *trace)
 {
         struct lock_list *entry;
         /*
@@ -829,11 +817,9 @@ static int add_lock_to_list(struct lock_class *class, struct lock_class *this,
         if (!entry)
                 return 0;
 
-        if (!save_trace(&entry->trace))
-                return 0;
-
         entry->class = this;
         entry->distance = distance;
+        entry->trace = *trace;
         /*
          * Since we never remove from the dependency list, the list can
          * be walked lockless by other CPUs, it's only allocation
@@ -1205,7 +1191,7 @@ check_noncircular(struct lock_list *root, struct lock_class *target,
 {
         int result;
 
-        debug_atomic_inc(&nr_cyclic_checks);
+        debug_atomic_inc(nr_cyclic_checks);
 
         result = __bfs_forwards(root, target, class_equal, target_entry);
 
@@ -1242,7 +1228,7 @@ find_usage_forwards(struct lock_list *root, enum lock_usage_bit bit,
 {
         int result;
 
-        debug_atomic_inc(&nr_find_usage_forwards_checks);
+        debug_atomic_inc(nr_find_usage_forwards_checks);
 
         result = __bfs_forwards(root, (void *)bit, usage_match, target_entry);
 
@@ -1265,7 +1251,7 @@ find_usage_backwards(struct lock_list *root, enum lock_usage_bit bit,
 {
         int result;
 
-        debug_atomic_inc(&nr_find_usage_backwards_checks);
+        debug_atomic_inc(nr_find_usage_backwards_checks);
 
         result = __bfs_backwards(root, (void *)bit, usage_match, target_entry);
 
@@ -1635,12 +1621,20 @@ check_deadlock(struct task_struct *curr, struct held_lock *next,
  */
 static int
 check_prev_add(struct task_struct *curr, struct held_lock *prev,
-               struct held_lock *next, int distance)
+               struct held_lock *next, int distance, int trylock_loop)
 {
         struct lock_list *entry;
         int ret;
         struct lock_list this;
         struct lock_list *uninitialized_var(target_entry);
+        /*
+         * Static variable, serialized by the graph_lock().
+         *
+         * We use this static variable to save the stack trace in case
+         * we call into this function multiple times due to encountering
+         * trylocks in the held lock stack.
+         */
+        static struct stack_trace trace;
 
         /*
          * Prove that the new <prev> -> <next> dependency would not
@@ -1688,20 +1682,23 @@ check_prev_add(struct task_struct *curr, struct held_lock *prev,
                 }
         }
 
+        if (!trylock_loop && !save_trace(&trace))
+                return 0;
+
         /*
          * Ok, all validations passed, add the new lock
          * to the previous lock's dependency list:
          */
         ret = add_lock_to_list(hlock_class(prev), hlock_class(next),
                                &hlock_class(prev)->locks_after,
-                               next->acquire_ip, distance);
+                               next->acquire_ip, distance, &trace);
 
         if (!ret)
                 return 0;
 
         ret = add_lock_to_list(hlock_class(next), hlock_class(prev),
                                &hlock_class(next)->locks_before,
-                               next->acquire_ip, distance);
+                               next->acquire_ip, distance, &trace);
         if (!ret)
                 return 0;
 
@@ -1731,6 +1728,7 @@ static int
 check_prevs_add(struct task_struct *curr, struct held_lock *next)
 {
         int depth = curr->lockdep_depth;
+        int trylock_loop = 0;
         struct held_lock *hlock;
 
         /*
@@ -1756,7 +1754,8 @@ check_prevs_add(struct task_struct *curr, struct held_lock *next)
                  * added:
                  */
                 if (hlock->read != 2) {
-                        if (!check_prev_add(curr, hlock, next, distance))
+                        if (!check_prev_add(curr, hlock, next,
+                                                distance, trylock_loop))
                                 return 0;
                         /*
                          * Stop after the first non-trylock entry,
@@ -1779,6 +1778,7 @@ check_prevs_add(struct task_struct *curr, struct held_lock *next)
                 if (curr->held_locks[depth].irq_context !=
                                 curr->held_locks[depth-1].irq_context)
                         break;
+                trylock_loop = 1;
         }
         return 1;
 out_bug:
@@ -1825,7 +1825,7 @@ static inline int lookup_chain_cache(struct task_struct *curr,
         list_for_each_entry(chain, hash_head, entry) {
                 if (chain->chain_key == chain_key) {
 cache_hit:
-                        debug_atomic_inc(&chain_lookup_hits);
+                        debug_atomic_inc(chain_lookup_hits);
                         if (very_verbose(class))
                                 printk("\nhash chain already cached, key: "
                                         "%016Lx tail class: [%p] %s\n",
@@ -1890,7 +1890,7 @@ cache_hit:
                 chain_hlocks[chain->base + j] = class - lock_classes;
         }
         list_add_tail_rcu(&chain->entry, hash_head);
-        debug_atomic_inc(&chain_lookup_misses);
+        debug_atomic_inc(chain_lookup_misses);
         inc_chains();
 
         return 1;
@@ -2311,7 +2311,12 @@ void trace_hardirqs_on_caller(unsigned long ip)
                 return;
 
         if (unlikely(curr->hardirqs_enabled)) {
-                debug_atomic_inc(&redundant_hardirqs_on);
+                /*
+                 * Neither irq nor preemption are disabled here
+                 * so this is racy by nature but loosing one hit
+                 * in a stat is not a big deal.
+                 */
+                __debug_atomic_inc(redundant_hardirqs_on);
                 return;
         }
         /* we'll do an OFF -> ON transition: */
@@ -2338,7 +2343,7 @@ void trace_hardirqs_on_caller(unsigned long ip)
 
         curr->hardirq_enable_ip = ip;
         curr->hardirq_enable_event = ++curr->irq_events;
-        debug_atomic_inc(&hardirqs_on_events);
+        debug_atomic_inc(hardirqs_on_events);
 }
 EXPORT_SYMBOL(trace_hardirqs_on_caller);
 
@@ -2370,9 +2375,9 @@ void trace_hardirqs_off_caller(unsigned long ip)
                 curr->hardirqs_enabled = 0;
                 curr->hardirq_disable_ip = ip;
                 curr->hardirq_disable_event = ++curr->irq_events;
-                debug_atomic_inc(&hardirqs_off_events);
+                debug_atomic_inc(hardirqs_off_events);
         } else
-                debug_atomic_inc(&redundant_hardirqs_off);
+                debug_atomic_inc(redundant_hardirqs_off);
 }
 EXPORT_SYMBOL(trace_hardirqs_off_caller);
 
@@ -2396,7 +2401,7 @@ void trace_softirqs_on(unsigned long ip)
                 return;
 
         if (curr->softirqs_enabled) {
-                debug_atomic_inc(&redundant_softirqs_on);
+                debug_atomic_inc(redundant_softirqs_on);
                 return;
         }
 
@@ -2406,7 +2411,7 @@ void trace_softirqs_on(unsigned long ip)
         curr->softirqs_enabled = 1;
         curr->softirq_enable_ip = ip;
         curr->softirq_enable_event = ++curr->irq_events;
-        debug_atomic_inc(&softirqs_on_events);
+        debug_atomic_inc(softirqs_on_events);
         /*
          * We are going to turn softirqs on, so set the
          * usage bit for all held locks, if hardirqs are
@@ -2436,10 +2441,10 @@ void trace_softirqs_off(unsigned long ip)
                 curr->softirqs_enabled = 0;
                 curr->softirq_disable_ip = ip;
                 curr->softirq_disable_event = ++curr->irq_events;
-                debug_atomic_inc(&softirqs_off_events);
+                debug_atomic_inc(softirqs_off_events);
                 DEBUG_LOCKS_WARN_ON(!softirq_count());
         } else
-                debug_atomic_inc(&redundant_softirqs_off);
+                debug_atomic_inc(redundant_softirqs_off);
 }
 
 static void __lockdep_trace_alloc(gfp_t gfp_mask, unsigned long flags)
@@ -2644,7 +2649,7 @@ static int mark_lock(struct task_struct *curr, struct held_lock *this,
                         return 0;
                 break;
         case LOCK_USED:
-                debug_atomic_dec(&nr_unused_locks);
+                debug_atomic_dec(nr_unused_locks);
                 break;
         default:
                 if (!debug_locks_off_graph_unlock())
@@ -2750,7 +2755,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
                 if (!class)
                         return 0;
         }
-        debug_atomic_inc((atomic_t *)&class->ops);
+        atomic_inc((atomic_t *)&class->ops);
         if (very_verbose(class)) {
                 printk("\nacquire class [%p] %s", class->key, class->name);
                 if (class->name_version > 1)
diff --git a/kernel/lockdep_internals.h b/kernel/lockdep_internals.h
index a2ee95ad1313..4f560cfedc8f 100644
--- a/kernel/lockdep_internals.h
+++ b/kernel/lockdep_internals.h
@@ -110,30 +110,60 @@ lockdep_count_backward_deps(struct lock_class *class)
 #endif
 
 #ifdef CONFIG_DEBUG_LOCKDEP
+
+#include <asm/local.h>
 /*
- * Various lockdep statistics:
+ * Various lockdep statistics.
+ * We want them per cpu as they are often accessed in fast path
+ * and we want to avoid too much cache bouncing.
  */
-extern atomic_t chain_lookup_hits;
-extern atomic_t chain_lookup_misses;
-extern atomic_t hardirqs_on_events;
-extern atomic_t hardirqs_off_events;
-extern atomic_t redundant_hardirqs_on;
-extern atomic_t redundant_hardirqs_off;
-extern atomic_t softirqs_on_events;
-extern atomic_t softirqs_off_events;
-extern atomic_t redundant_softirqs_on;
-extern atomic_t redundant_softirqs_off;
-extern atomic_t nr_unused_locks;
-extern atomic_t nr_cyclic_checks;
-extern atomic_t nr_cyclic_check_recursions;
-extern atomic_t nr_find_usage_forwards_checks;
-extern atomic_t nr_find_usage_forwards_recursions;
-extern atomic_t nr_find_usage_backwards_checks;
-extern atomic_t nr_find_usage_backwards_recursions;
-# define debug_atomic_inc(ptr)          atomic_inc(ptr)
-# define debug_atomic_dec(ptr)          atomic_dec(ptr)
-# define debug_atomic_read(ptr)         atomic_read(ptr)
+struct lockdep_stats {
+        int     chain_lookup_hits;
+        int     chain_lookup_misses;
+        int     hardirqs_on_events;
+        int     hardirqs_off_events;
+        int     redundant_hardirqs_on;
+        int     redundant_hardirqs_off;
+        int     softirqs_on_events;
+        int     softirqs_off_events;
+        int     redundant_softirqs_on;
+        int     redundant_softirqs_off;
+        int     nr_unused_locks;
+        int     nr_cyclic_checks;
+        int     nr_cyclic_check_recursions;
+        int     nr_find_usage_forwards_checks;
+        int     nr_find_usage_forwards_recursions;
+        int     nr_find_usage_backwards_checks;
+        int     nr_find_usage_backwards_recursions;
+};
+
+DECLARE_PER_CPU(struct lockdep_stats, lockdep_stats);
+
+#define __debug_atomic_inc(ptr)                                 \
+        this_cpu_inc(lockdep_stats.ptr);
+
+#define debug_atomic_inc(ptr)                   {               \
+        WARN_ON_ONCE(!irqs_disabled());                         \
+        __this_cpu_inc(lockdep_stats.ptr);                      \
+}
+
+#define debug_atomic_dec(ptr)                   {               \
+        WARN_ON_ONCE(!irqs_disabled());                         \
+        __this_cpu_dec(lockdep_stats.ptr);                      \
+}
+
+#define debug_atomic_read(ptr)          ({                              \
+        struct lockdep_stats *__cpu_lockdep_stats;                      \
+        unsigned long long __total = 0;                                 \
+        int __cpu;                                                      \
+        for_each_possible_cpu(__cpu) {                                  \
+                __cpu_lockdep_stats = &per_cpu(lockdep_stats, __cpu);   \
+                __total += __cpu_lockdep_stats->ptr;                    \
+        }                                                               \
+        __total;                                                        \
+})
 #else
+# define __debug_atomic_inc(ptr)        do { } while (0)
 # define debug_atomic_inc(ptr)          do { } while (0)
 # define debug_atomic_dec(ptr)          do { } while (0)
 # define debug_atomic_read(ptr)         0
diff --git a/kernel/lockdep_proc.c b/kernel/lockdep_proc.c
index d4aba4f3584c..59b76c8ce9d7 100644
--- a/kernel/lockdep_proc.c
+++ b/kernel/lockdep_proc.c
@@ -184,34 +184,34 @@ static const struct file_operations proc_lockdep_chains_operations = {
 static void lockdep_stats_debug_show(struct seq_file *m)
 {
 #ifdef CONFIG_DEBUG_LOCKDEP
-        unsigned int hi1 = debug_atomic_read(&hardirqs_on_events),
-                     hi2 = debug_atomic_read(&hardirqs_off_events),
-                     hr1 = debug_atomic_read(&redundant_hardirqs_on),
-                     hr2 = debug_atomic_read(&redundant_hardirqs_off),
-                     si1 = debug_atomic_read(&softirqs_on_events),
-                     si2 = debug_atomic_read(&softirqs_off_events),
-                     sr1 = debug_atomic_read(&redundant_softirqs_on),
-                     sr2 = debug_atomic_read(&redundant_softirqs_off);
-
-        seq_printf(m, " chain lookup misses:           %11u\n",
-                debug_atomic_read(&chain_lookup_misses));
-        seq_printf(m, " chain lookup hits:             %11u\n",
-                debug_atomic_read(&chain_lookup_hits));
-        seq_printf(m, " cyclic checks:                 %11u\n",
-                debug_atomic_read(&nr_cyclic_checks));
-        seq_printf(m, " find-mask forwards checks:     %11u\n",
-                debug_atomic_read(&nr_find_usage_forwards_checks));
-        seq_printf(m, " find-mask backwards checks:    %11u\n",
-                debug_atomic_read(&nr_find_usage_backwards_checks));
-
-        seq_printf(m, " hardirq on events:             %11u\n", hi1);
-        seq_printf(m, " hardirq off events:            %11u\n", hi2);
-        seq_printf(m, " redundant hardirq ons:         %11u\n", hr1);
-        seq_printf(m, " redundant hardirq offs:        %11u\n", hr2);
-        seq_printf(m, " softirq on events:             %11u\n", si1);
-        seq_printf(m, " softirq off events:            %11u\n", si2);
-        seq_printf(m, " redundant softirq ons:         %11u\n", sr1);
-        seq_printf(m, " redundant softirq offs:        %11u\n", sr2);
+        unsigned long long hi1 = debug_atomic_read(hardirqs_on_events),
+                           hi2 = debug_atomic_read(hardirqs_off_events),
+                           hr1 = debug_atomic_read(redundant_hardirqs_on),
+                           hr2 = debug_atomic_read(redundant_hardirqs_off),
+                           si1 = debug_atomic_read(softirqs_on_events),
+                           si2 = debug_atomic_read(softirqs_off_events),
+                           sr1 = debug_atomic_read(redundant_softirqs_on),
+                           sr2 = debug_atomic_read(redundant_softirqs_off);
+
+        seq_printf(m, " chain lookup misses:           %11llu\n",
+                debug_atomic_read(chain_lookup_misses));
+        seq_printf(m, " chain lookup hits:             %11llu\n",
+                debug_atomic_read(chain_lookup_hits));
+        seq_printf(m, " cyclic checks:                 %11llu\n",
+                debug_atomic_read(nr_cyclic_checks));
+        seq_printf(m, " find-mask forwards checks:     %11llu\n",
+                debug_atomic_read(nr_find_usage_forwards_checks));
+        seq_printf(m, " find-mask backwards checks:    %11llu\n",
+                debug_atomic_read(nr_find_usage_backwards_checks));
+
+        seq_printf(m, " hardirq on events:             %11llu\n", hi1);
+        seq_printf(m, " hardirq off events:            %11llu\n", hi2);
+        seq_printf(m, " redundant hardirq ons:         %11llu\n", hr1);
+        seq_printf(m, " redundant hardirq offs:        %11llu\n", hr2);
+        seq_printf(m, " softirq on events:             %11llu\n", si1);
+        seq_printf(m, " softirq off events:            %11llu\n", si2);
+        seq_printf(m, " redundant softirq ons:         %11llu\n", sr1);
+        seq_printf(m, " redundant softirq offs:        %11llu\n", sr2);
 #endif
 }
 
@@ -263,7 +263,7 @@ static int lockdep_stats_show(struct seq_file *m, void *v)
 #endif
         }
 #ifdef CONFIG_DEBUG_LOCKDEP
-        DEBUG_LOCKS_WARN_ON(debug_atomic_read(&nr_unused_locks) != nr_unused);
+        DEBUG_LOCKS_WARN_ON(debug_atomic_read(nr_unused_locks) != nr_unused);
 #endif
         seq_printf(m, " lock-classes:                  %11lu [max: %lu]\n",
                         nr_lock_classes, MAX_LOCKDEP_KEYS);