perf: Convert perf_event to local_t

Since now all modification to event->count (and ->prev_count
and ->period_left) are local to a cpu, change then to local64_t so we
avoid the LOCK'ed ops.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

1 files changed, 9 insertions, 9 deletions

diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index 79e199843db6..2d0d29069275 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -296,10 +296,10 @@ x86_perf_event_update(struct perf_event *event)
          * count to the generic event atomically:
          */
 again:
-        prev_raw_count = atomic64_read(&hwc->prev_count);
+        prev_raw_count = local64_read(&hwc->prev_count);
         rdmsrl(hwc->event_base + idx, new_raw_count);
 
-        if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
+        if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
                                         new_raw_count) != prev_raw_count)
                 goto again;
 
@@ -314,8 +314,8 @@ again:
         delta = (new_raw_count << shift) - (prev_raw_count << shift);
         delta >>= shift;
 
-        atomic64_add(delta, &event->count);
-        atomic64_sub(delta, &hwc->period_left);
+        local64_add(delta, &event->count);
+        local64_sub(delta, &hwc->period_left);
 
         return new_raw_count;
 }
@@ -439,7 +439,7 @@ static int x86_setup_perfctr(struct perf_event *event)
         if (!hwc->sample_period) {
                 hwc->sample_period = x86_pmu.max_period;
                 hwc->last_period = hwc->sample_period;
-                atomic64_set(&hwc->period_left, hwc->sample_period);
+                local64_set(&hwc->period_left, hwc->sample_period);
         } else {
                 /*
                  * If we have a PMU initialized but no APIC
@@ -886,7 +886,7 @@ static int
 x86_perf_event_set_period(struct perf_event *event)
 {
         struct hw_perf_event *hwc = &event->hw;
-        s64 left = atomic64_read(&hwc->period_left);
+        s64 left = local64_read(&hwc->period_left);
         s64 period = hwc->sample_period;
         int ret = 0, idx = hwc->idx;
 
@@ -898,14 +898,14 @@ x86_perf_event_set_period(struct perf_event *event)
          */
         if (unlikely(left <= -period)) {
                 left = period;
-                atomic64_set(&hwc->period_left, left);
+                local64_set(&hwc->period_left, left);
                 hwc->last_period = period;
                 ret = 1;
         }
 
         if (unlikely(left <= 0)) {
                 left += period;
-                atomic64_set(&hwc->period_left, left);
+                local64_set(&hwc->period_left, left);
                 hwc->last_period = period;
                 ret = 1;
         }
@@ -924,7 +924,7 @@ x86_perf_event_set_period(struct perf_event *event)
          * The hw event starts counting from this event offset,
          * mark it to be able to extra future deltas:
          */
-        atomic64_set(&hwc->prev_count, (u64)-left);
+        local64_set(&hwc->prev_count, (u64)-left);
 
         wrmsrl(hwc->event_base + idx, (u64)(-left) & x86_pmu.cntval_mask);