diff options
Diffstat (limited to 'kernel/perf_event.c')
| -rw-r--r-- | kernel/perf_event.c | 30 |
1 files changed, 15 insertions, 15 deletions
diff --git a/kernel/perf_event.c b/kernel/perf_event.c index 8cf737da3ec4..1f98c78c3343 100644 --- a/kernel/perf_event.c +++ b/kernel/perf_event.c | |||
| @@ -2916,7 +2916,7 @@ static void perf_output_get_handle(struct perf_output_handle *handle) | |||
| 2916 | struct perf_mmap_data *data = handle->data; | 2916 | struct perf_mmap_data *data = handle->data; |
| 2917 | 2917 | ||
| 2918 | preempt_disable(); | 2918 | preempt_disable(); |
| 2919 | atomic_inc(&data->nest); | 2919 | local_inc(&data->nest); |
| 2920 | } | 2920 | } |
| 2921 | 2921 | ||
| 2922 | static void perf_output_put_handle(struct perf_output_handle *handle) | 2922 | static void perf_output_put_handle(struct perf_output_handle *handle) |
| @@ -2925,13 +2925,13 @@ static void perf_output_put_handle(struct perf_output_handle *handle) | |||
| 2925 | unsigned long head; | 2925 | unsigned long head; |
| 2926 | 2926 | ||
| 2927 | again: | 2927 | again: |
| 2928 | head = atomic_long_read(&data->head); | 2928 | head = local_read(&data->head); |
| 2929 | 2929 | ||
| 2930 | /* | 2930 | /* |
| 2931 | * IRQ/NMI can happen here, which means we can miss a head update. | 2931 | * IRQ/NMI can happen here, which means we can miss a head update. |
| 2932 | */ | 2932 | */ |
| 2933 | 2933 | ||
| 2934 | if (!atomic_dec_and_test(&data->nest)) | 2934 | if (!local_dec_and_test(&data->nest)) |
| 2935 | return; | 2935 | return; |
| 2936 | 2936 | ||
| 2937 | /* | 2937 | /* |
| @@ -2945,12 +2945,12 @@ again: | |||
| 2945 | * Now check if we missed an update, rely on the (compiler) | 2945 | * Now check if we missed an update, rely on the (compiler) |
| 2946 | * barrier in atomic_dec_and_test() to re-read data->head. | 2946 | * barrier in atomic_dec_and_test() to re-read data->head. |
| 2947 | */ | 2947 | */ |
| 2948 | if (unlikely(head != atomic_long_read(&data->head))) { | 2948 | if (unlikely(head != local_read(&data->head))) { |
| 2949 | atomic_inc(&data->nest); | 2949 | local_inc(&data->nest); |
| 2950 | goto again; | 2950 | goto again; |
| 2951 | } | 2951 | } |
| 2952 | 2952 | ||
| 2953 | if (atomic_xchg(&data->wakeup, 0)) | 2953 | if (local_xchg(&data->wakeup, 0)) |
| 2954 | perf_output_wakeup(handle); | 2954 | perf_output_wakeup(handle); |
| 2955 | 2955 | ||
| 2956 | preempt_enable(); | 2956 | preempt_enable(); |
| @@ -3031,7 +3031,7 @@ int perf_output_begin(struct perf_output_handle *handle, | |||
| 3031 | if (!data->nr_pages) | 3031 | if (!data->nr_pages) |
| 3032 | goto out; | 3032 | goto out; |
| 3033 | 3033 | ||
| 3034 | have_lost = atomic_read(&data->lost); | 3034 | have_lost = local_read(&data->lost); |
| 3035 | if (have_lost) | 3035 | if (have_lost) |
| 3036 | size += sizeof(lost_event); | 3036 | size += sizeof(lost_event); |
| 3037 | 3037 | ||
| @@ -3045,24 +3045,24 @@ int perf_output_begin(struct perf_output_handle *handle, | |||
| 3045 | */ | 3045 | */ |
| 3046 | tail = ACCESS_ONCE(data->user_page->data_tail); | 3046 | tail = ACCESS_ONCE(data->user_page->data_tail); |
| 3047 | smp_rmb(); | 3047 | smp_rmb(); |
| 3048 | offset = head = atomic_long_read(&data->head); | 3048 | offset = head = local_read(&data->head); |
| 3049 | head += size; | 3049 | head += size; |
| 3050 | if (unlikely(!perf_output_space(data, tail, offset, head))) | 3050 | if (unlikely(!perf_output_space(data, tail, offset, head))) |
| 3051 | goto fail; | 3051 | goto fail; |
| 3052 | } while (atomic_long_cmpxchg(&data->head, offset, head) != offset); | 3052 | } while (local_cmpxchg(&data->head, offset, head) != offset); |
| 3053 | 3053 | ||
| 3054 | handle->offset = offset; | 3054 | handle->offset = offset; |
| 3055 | handle->head = head; | 3055 | handle->head = head; |
| 3056 | 3056 | ||
| 3057 | if (head - tail > data->watermark) | 3057 | if (head - tail > data->watermark) |
| 3058 | atomic_inc(&data->wakeup); | 3058 | local_inc(&data->wakeup); |
| 3059 | 3059 | ||
| 3060 | if (have_lost) { | 3060 | if (have_lost) { |
| 3061 | lost_event.header.type = PERF_RECORD_LOST; | 3061 | lost_event.header.type = PERF_RECORD_LOST; |
| 3062 | lost_event.header.misc = 0; | 3062 | lost_event.header.misc = 0; |
| 3063 | lost_event.header.size = sizeof(lost_event); | 3063 | lost_event.header.size = sizeof(lost_event); |
| 3064 | lost_event.id = event->id; | 3064 | lost_event.id = event->id; |
| 3065 | lost_event.lost = atomic_xchg(&data->lost, 0); | 3065 | lost_event.lost = local_xchg(&data->lost, 0); |
| 3066 | 3066 | ||
| 3067 | perf_output_put(handle, lost_event); | 3067 | perf_output_put(handle, lost_event); |
| 3068 | } | 3068 | } |
| @@ -3070,7 +3070,7 @@ int perf_output_begin(struct perf_output_handle *handle, | |||
| 3070 | return 0; | 3070 | return 0; |
| 3071 | 3071 | ||
| 3072 | fail: | 3072 | fail: |
| 3073 | atomic_inc(&data->lost); | 3073 | local_inc(&data->lost); |
| 3074 | perf_output_put_handle(handle); | 3074 | perf_output_put_handle(handle); |
| 3075 | out: | 3075 | out: |
| 3076 | rcu_read_unlock(); | 3076 | rcu_read_unlock(); |
| @@ -3086,10 +3086,10 @@ void perf_output_end(struct perf_output_handle *handle) | |||
| 3086 | int wakeup_events = event->attr.wakeup_events; | 3086 | int wakeup_events = event->attr.wakeup_events; |
| 3087 | 3087 | ||
| 3088 | if (handle->sample && wakeup_events) { | 3088 | if (handle->sample && wakeup_events) { |
| 3089 | int events = atomic_inc_return(&data->events); | 3089 | int events = local_inc_return(&data->events); |
| 3090 | if (events >= wakeup_events) { | 3090 | if (events >= wakeup_events) { |
| 3091 | atomic_sub(wakeup_events, &data->events); | 3091 | local_sub(wakeup_events, &data->events); |
| 3092 | atomic_inc(&data->wakeup); | 3092 | local_inc(&data->wakeup); |
| 3093 | } | 3093 | } |
| 3094 | } | 3094 | } |
| 3095 | 3095 | ||
