diff options
Diffstat (limited to 'kernel/perf_event.c')
| -rw-r--r-- | kernel/perf_event.c | 233 |
1 files changed, 120 insertions, 113 deletions
diff --git a/kernel/perf_event.c b/kernel/perf_event.c index 40a996ec39fa..603c0d8b5df1 100644 --- a/kernel/perf_event.c +++ b/kernel/perf_event.c | |||
| @@ -36,7 +36,7 @@ | |||
| 36 | /* | 36 | /* |
| 37 | * Each CPU has a list of per CPU events: | 37 | * Each CPU has a list of per CPU events: |
| 38 | */ | 38 | */ |
| 39 | DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context); | 39 | static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context); |
| 40 | 40 | ||
| 41 | int perf_max_events __read_mostly = 1; | 41 | int perf_max_events __read_mostly = 1; |
| 42 | static int perf_reserved_percpu __read_mostly; | 42 | static int perf_reserved_percpu __read_mostly; |
| @@ -203,14 +203,14 @@ perf_lock_task_context(struct task_struct *task, unsigned long *flags) | |||
| 203 | * if so. If we locked the right context, then it | 203 | * if so. If we locked the right context, then it |
| 204 | * can't get swapped on us any more. | 204 | * can't get swapped on us any more. |
| 205 | */ | 205 | */ |
| 206 | spin_lock_irqsave(&ctx->lock, *flags); | 206 | raw_spin_lock_irqsave(&ctx->lock, *flags); |
| 207 | if (ctx != rcu_dereference(task->perf_event_ctxp)) { | 207 | if (ctx != rcu_dereference(task->perf_event_ctxp)) { |
| 208 | spin_unlock_irqrestore(&ctx->lock, *flags); | 208 | raw_spin_unlock_irqrestore(&ctx->lock, *flags); |
| 209 | goto retry; | 209 | goto retry; |
| 210 | } | 210 | } |
| 211 | 211 | ||
| 212 | if (!atomic_inc_not_zero(&ctx->refcount)) { | 212 | if (!atomic_inc_not_zero(&ctx->refcount)) { |
| 213 | spin_unlock_irqrestore(&ctx->lock, *flags); | 213 | raw_spin_unlock_irqrestore(&ctx->lock, *flags); |
| 214 | ctx = NULL; | 214 | ctx = NULL; |
| 215 | } | 215 | } |
| 216 | } | 216 | } |
| @@ -231,7 +231,7 @@ static struct perf_event_context *perf_pin_task_context(struct task_struct *task | |||
| 231 | ctx = perf_lock_task_context(task, &flags); | 231 | ctx = perf_lock_task_context(task, &flags); |
| 232 | if (ctx) { | 232 | if (ctx) { |
| 233 | ++ctx->pin_count; | 233 | ++ctx->pin_count; |
| 234 | spin_unlock_irqrestore(&ctx->lock, flags); | 234 | raw_spin_unlock_irqrestore(&ctx->lock, flags); |
| 235 | } | 235 | } |
| 236 | return ctx; | 236 | return ctx; |
| 237 | } | 237 | } |
| @@ -240,9 +240,9 @@ static void perf_unpin_context(struct perf_event_context *ctx) | |||
| 240 | { | 240 | { |
| 241 | unsigned long flags; | 241 | unsigned long flags; |
| 242 | 242 | ||
| 243 | spin_lock_irqsave(&ctx->lock, flags); | 243 | raw_spin_lock_irqsave(&ctx->lock, flags); |
| 244 | --ctx->pin_count; | 244 | --ctx->pin_count; |
| 245 | spin_unlock_irqrestore(&ctx->lock, flags); | 245 | raw_spin_unlock_irqrestore(&ctx->lock, flags); |
| 246 | put_ctx(ctx); | 246 | put_ctx(ctx); |
| 247 | } | 247 | } |
| 248 | 248 | ||
| @@ -427,7 +427,7 @@ static void __perf_event_remove_from_context(void *info) | |||
| 427 | if (ctx->task && cpuctx->task_ctx != ctx) | 427 | if (ctx->task && cpuctx->task_ctx != ctx) |
| 428 | return; | 428 | return; |
| 429 | 429 | ||
| 430 | spin_lock(&ctx->lock); | 430 | raw_spin_lock(&ctx->lock); |
| 431 | /* | 431 | /* |
| 432 | * Protect the list operation against NMI by disabling the | 432 | * Protect the list operation against NMI by disabling the |
| 433 | * events on a global level. | 433 | * events on a global level. |
| @@ -449,7 +449,7 @@ static void __perf_event_remove_from_context(void *info) | |||
| 449 | } | 449 | } |
| 450 | 450 | ||
| 451 | perf_enable(); | 451 | perf_enable(); |
| 452 | spin_unlock(&ctx->lock); | 452 | raw_spin_unlock(&ctx->lock); |
| 453 | } | 453 | } |
| 454 | 454 | ||
| 455 | 455 | ||
| @@ -488,12 +488,12 @@ retry: | |||
| 488 | task_oncpu_function_call(task, __perf_event_remove_from_context, | 488 | task_oncpu_function_call(task, __perf_event_remove_from_context, |
| 489 | event); | 489 | event); |
| 490 | 490 | ||
| 491 | spin_lock_irq(&ctx->lock); | 491 | raw_spin_lock_irq(&ctx->lock); |
| 492 | /* | 492 | /* |
| 493 | * If the context is active we need to retry the smp call. | 493 | * If the context is active we need to retry the smp call. |
| 494 | */ | 494 | */ |
| 495 | if (ctx->nr_active && !list_empty(&event->group_entry)) { | 495 | if (ctx->nr_active && !list_empty(&event->group_entry)) { |
| 496 | spin_unlock_irq(&ctx->lock); | 496 | raw_spin_unlock_irq(&ctx->lock); |
| 497 | goto retry; | 497 | goto retry; |
| 498 | } | 498 | } |
| 499 | 499 | ||
| @@ -504,7 +504,7 @@ retry: | |||
| 504 | */ | 504 | */ |
| 505 | if (!list_empty(&event->group_entry)) | 505 | if (!list_empty(&event->group_entry)) |
| 506 | list_del_event(event, ctx); | 506 | list_del_event(event, ctx); |
| 507 | spin_unlock_irq(&ctx->lock); | 507 | raw_spin_unlock_irq(&ctx->lock); |
| 508 | } | 508 | } |
| 509 | 509 | ||
| 510 | /* | 510 | /* |
| @@ -535,7 +535,7 @@ static void __perf_event_disable(void *info) | |||
| 535 | if (ctx->task && cpuctx->task_ctx != ctx) | 535 | if (ctx->task && cpuctx->task_ctx != ctx) |
| 536 | return; | 536 | return; |
| 537 | 537 | ||
| 538 | spin_lock(&ctx->lock); | 538 | raw_spin_lock(&ctx->lock); |
| 539 | 539 | ||
| 540 | /* | 540 | /* |
| 541 | * If the event is on, turn it off. | 541 | * If the event is on, turn it off. |
| @@ -551,7 +551,7 @@ static void __perf_event_disable(void *info) | |||
| 551 | event->state = PERF_EVENT_STATE_OFF; | 551 | event->state = PERF_EVENT_STATE_OFF; |
| 552 | } | 552 | } |
| 553 | 553 | ||
| 554 | spin_unlock(&ctx->lock); | 554 | raw_spin_unlock(&ctx->lock); |
| 555 | } | 555 | } |
| 556 | 556 | ||
| 557 | /* | 557 | /* |
| @@ -567,7 +567,7 @@ static void __perf_event_disable(void *info) | |||
| 567 | * is the current context on this CPU and preemption is disabled, | 567 | * is the current context on this CPU and preemption is disabled, |
| 568 | * hence we can't get into perf_event_task_sched_out for this context. | 568 | * hence we can't get into perf_event_task_sched_out for this context. |
| 569 | */ | 569 | */ |
| 570 | static void perf_event_disable(struct perf_event *event) | 570 | void perf_event_disable(struct perf_event *event) |
| 571 | { | 571 | { |
| 572 | struct perf_event_context *ctx = event->ctx; | 572 | struct perf_event_context *ctx = event->ctx; |
| 573 | struct task_struct *task = ctx->task; | 573 | struct task_struct *task = ctx->task; |
| @@ -584,12 +584,12 @@ static void perf_event_disable(struct perf_event *event) | |||
| 584 | retry: | 584 | retry: |
| 585 | task_oncpu_function_call(task, __perf_event_disable, event); | 585 | task_oncpu_function_call(task, __perf_event_disable, event); |
| 586 | 586 | ||
| 587 | spin_lock_irq(&ctx->lock); | 587 | raw_spin_lock_irq(&ctx->lock); |
| 588 | /* | 588 | /* |
| 589 | * If the event is still active, we need to retry the cross-call. | 589 | * If the event is still active, we need to retry the cross-call. |
| 590 | */ | 590 | */ |
| 591 | if (event->state == PERF_EVENT_STATE_ACTIVE) { | 591 | if (event->state == PERF_EVENT_STATE_ACTIVE) { |
| 592 | spin_unlock_irq(&ctx->lock); | 592 | raw_spin_unlock_irq(&ctx->lock); |
| 593 | goto retry; | 593 | goto retry; |
| 594 | } | 594 | } |
| 595 | 595 | ||
| @@ -602,7 +602,7 @@ static void perf_event_disable(struct perf_event *event) | |||
| 602 | event->state = PERF_EVENT_STATE_OFF; | 602 | event->state = PERF_EVENT_STATE_OFF; |
| 603 | } | 603 | } |
| 604 | 604 | ||
| 605 | spin_unlock_irq(&ctx->lock); | 605 | raw_spin_unlock_irq(&ctx->lock); |
| 606 | } | 606 | } |
| 607 | 607 | ||
| 608 | static int | 608 | static int |
| @@ -770,7 +770,7 @@ static void __perf_install_in_context(void *info) | |||
| 770 | cpuctx->task_ctx = ctx; | 770 | cpuctx->task_ctx = ctx; |
| 771 | } | 771 | } |
| 772 | 772 | ||
| 773 | spin_lock(&ctx->lock); | 773 | raw_spin_lock(&ctx->lock); |
| 774 | ctx->is_active = 1; | 774 | ctx->is_active = 1; |
| 775 | update_context_time(ctx); | 775 | update_context_time(ctx); |
| 776 | 776 | ||
| @@ -782,6 +782,9 @@ static void __perf_install_in_context(void *info) | |||
| 782 | 782 | ||
| 783 | add_event_to_ctx(event, ctx); | 783 | add_event_to_ctx(event, ctx); |
| 784 | 784 | ||
| 785 | if (event->cpu != -1 && event->cpu != smp_processor_id()) | ||
| 786 | goto unlock; | ||
| 787 | |||
| 785 | /* | 788 | /* |
| 786 | * Don't put the event on if it is disabled or if | 789 | * Don't put the event on if it is disabled or if |
| 787 | * it is in a group and the group isn't on. | 790 | * it is in a group and the group isn't on. |
| @@ -820,7 +823,7 @@ static void __perf_install_in_context(void *info) | |||
| 820 | unlock: | 823 | unlock: |
| 821 | perf_enable(); | 824 | perf_enable(); |
| 822 | 825 | ||
| 823 | spin_unlock(&ctx->lock); | 826 | raw_spin_unlock(&ctx->lock); |
| 824 | } | 827 | } |
| 825 | 828 | ||
| 826 | /* | 829 | /* |
| @@ -856,12 +859,12 @@ retry: | |||
| 856 | task_oncpu_function_call(task, __perf_install_in_context, | 859 | task_oncpu_function_call(task, __perf_install_in_context, |
| 857 | event); | 860 | event); |
| 858 | 861 | ||
| 859 | spin_lock_irq(&ctx->lock); | 862 | raw_spin_lock_irq(&ctx->lock); |
| 860 | /* | 863 | /* |
| 861 | * we need to retry the smp call. | 864 | * we need to retry the smp call. |
| 862 | */ | 865 | */ |
| 863 | if (ctx->is_active && list_empty(&event->group_entry)) { | 866 | if (ctx->is_active && list_empty(&event->group_entry)) { |
| 864 | spin_unlock_irq(&ctx->lock); | 867 | raw_spin_unlock_irq(&ctx->lock); |
| 865 | goto retry; | 868 | goto retry; |
| 866 | } | 869 | } |
| 867 | 870 | ||
| @@ -872,7 +875,7 @@ retry: | |||
| 872 | */ | 875 | */ |
| 873 | if (list_empty(&event->group_entry)) | 876 | if (list_empty(&event->group_entry)) |
| 874 | add_event_to_ctx(event, ctx); | 877 | add_event_to_ctx(event, ctx); |
| 875 | spin_unlock_irq(&ctx->lock); | 878 | raw_spin_unlock_irq(&ctx->lock); |
| 876 | } | 879 | } |
| 877 | 880 | ||
| 878 | /* | 881 | /* |
| @@ -917,7 +920,7 @@ static void __perf_event_enable(void *info) | |||
| 917 | cpuctx->task_ctx = ctx; | 920 | cpuctx->task_ctx = ctx; |
| 918 | } | 921 | } |
| 919 | 922 | ||
| 920 | spin_lock(&ctx->lock); | 923 | raw_spin_lock(&ctx->lock); |
| 921 | ctx->is_active = 1; | 924 | ctx->is_active = 1; |
| 922 | update_context_time(ctx); | 925 | update_context_time(ctx); |
| 923 | 926 | ||
| @@ -925,6 +928,9 @@ static void __perf_event_enable(void *info) | |||
| 925 | goto unlock; | 928 | goto unlock; |
| 926 | __perf_event_mark_enabled(event, ctx); | 929 | __perf_event_mark_enabled(event, ctx); |
| 927 | 930 | ||
| 931 | if (event->cpu != -1 && event->cpu != smp_processor_id()) | ||
| 932 | goto unlock; | ||
| 933 | |||
| 928 | /* | 934 | /* |
| 929 | * If the event is in a group and isn't the group leader, | 935 | * If the event is in a group and isn't the group leader, |
| 930 | * then don't put it on unless the group is on. | 936 | * then don't put it on unless the group is on. |
| @@ -959,7 +965,7 @@ static void __perf_event_enable(void *info) | |||
| 959 | } | 965 | } |
| 960 | 966 | ||
| 961 | unlock: | 967 | unlock: |
| 962 | spin_unlock(&ctx->lock); | 968 | raw_spin_unlock(&ctx->lock); |
| 963 | } | 969 | } |
| 964 | 970 | ||
| 965 | /* | 971 | /* |
| @@ -971,7 +977,7 @@ static void __perf_event_enable(void *info) | |||
| 971 | * perf_event_for_each_child or perf_event_for_each as described | 977 | * perf_event_for_each_child or perf_event_for_each as described |
| 972 | * for perf_event_disable. | 978 | * for perf_event_disable. |
| 973 | */ | 979 | */ |
| 974 | static void perf_event_enable(struct perf_event *event) | 980 | void perf_event_enable(struct perf_event *event) |
| 975 | { | 981 | { |
| 976 | struct perf_event_context *ctx = event->ctx; | 982 | struct perf_event_context *ctx = event->ctx; |
| 977 | struct task_struct *task = ctx->task; | 983 | struct task_struct *task = ctx->task; |
| @@ -985,7 +991,7 @@ static void perf_event_enable(struct perf_event *event) | |||
| 985 | return; | 991 | return; |
| 986 | } | 992 | } |
| 987 | 993 | ||
| 988 | spin_lock_irq(&ctx->lock); | 994 | raw_spin_lock_irq(&ctx->lock); |
| 989 | if (event->state >= PERF_EVENT_STATE_INACTIVE) | 995 | if (event->state >= PERF_EVENT_STATE_INACTIVE) |
| 990 | goto out; | 996 | goto out; |
| 991 | 997 | ||
| @@ -1000,10 +1006,10 @@ static void perf_event_enable(struct perf_event *event) | |||
| 1000 | event->state = PERF_EVENT_STATE_OFF; | 1006 | event->state = PERF_EVENT_STATE_OFF; |
| 1001 | 1007 | ||
| 1002 | retry: | 1008 | retry: |
| 1003 | spin_unlock_irq(&ctx->lock); | 1009 | raw_spin_unlock_irq(&ctx->lock); |
| 1004 | task_oncpu_function_call(task, __perf_event_enable, event); | 1010 | task_oncpu_function_call(task, __perf_event_enable, event); |
| 1005 | 1011 | ||
| 1006 | spin_lock_irq(&ctx->lock); | 1012 | raw_spin_lock_irq(&ctx->lock); |
| 1007 | 1013 | ||
| 1008 | /* | 1014 | /* |
| 1009 | * If the context is active and the event is still off, | 1015 | * If the context is active and the event is still off, |
| @@ -1020,7 +1026,7 @@ static void perf_event_enable(struct perf_event *event) | |||
| 1020 | __perf_event_mark_enabled(event, ctx); | 1026 | __perf_event_mark_enabled(event, ctx); |
| 1021 | 1027 | ||
| 1022 | out: | 1028 | out: |
| 1023 | spin_unlock_irq(&ctx->lock); | 1029 | raw_spin_unlock_irq(&ctx->lock); |
| 1024 | } | 1030 | } |
| 1025 | 1031 | ||
| 1026 | static int perf_event_refresh(struct perf_event *event, int refresh) | 1032 | static int perf_event_refresh(struct perf_event *event, int refresh) |
| @@ -1042,7 +1048,7 @@ void __perf_event_sched_out(struct perf_event_context *ctx, | |||
| 1042 | { | 1048 | { |
| 1043 | struct perf_event *event; | 1049 | struct perf_event *event; |
| 1044 | 1050 | ||
| 1045 | spin_lock(&ctx->lock); | 1051 | raw_spin_lock(&ctx->lock); |
| 1046 | ctx->is_active = 0; | 1052 | ctx->is_active = 0; |
| 1047 | if (likely(!ctx->nr_events)) | 1053 | if (likely(!ctx->nr_events)) |
| 1048 | goto out; | 1054 | goto out; |
| @@ -1055,7 +1061,7 @@ void __perf_event_sched_out(struct perf_event_context *ctx, | |||
| 1055 | } | 1061 | } |
| 1056 | perf_enable(); | 1062 | perf_enable(); |
| 1057 | out: | 1063 | out: |
| 1058 | spin_unlock(&ctx->lock); | 1064 | raw_spin_unlock(&ctx->lock); |
| 1059 | } | 1065 | } |
| 1060 | 1066 | ||
| 1061 | /* | 1067 | /* |
| @@ -1193,8 +1199,8 @@ void perf_event_task_sched_out(struct task_struct *task, | |||
| 1193 | * order we take the locks because no other cpu could | 1199 | * order we take the locks because no other cpu could |
| 1194 | * be trying to lock both of these tasks. | 1200 | * be trying to lock both of these tasks. |
| 1195 | */ | 1201 | */ |
| 1196 | spin_lock(&ctx->lock); | 1202 | raw_spin_lock(&ctx->lock); |
| 1197 | spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); | 1203 | raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); |
| 1198 | if (context_equiv(ctx, next_ctx)) { | 1204 | if (context_equiv(ctx, next_ctx)) { |
| 1199 | /* | 1205 | /* |
| 1200 | * XXX do we need a memory barrier of sorts | 1206 | * XXX do we need a memory barrier of sorts |
| @@ -1208,8 +1214,8 @@ void perf_event_task_sched_out(struct task_struct *task, | |||
| 1208 | 1214 | ||
| 1209 | perf_event_sync_stat(ctx, next_ctx); | 1215 | perf_event_sync_stat(ctx, next_ctx); |
| 1210 | } | 1216 | } |
| 1211 | spin_unlock(&next_ctx->lock); | 1217 | raw_spin_unlock(&next_ctx->lock); |
| 1212 | spin_unlock(&ctx->lock); | 1218 | raw_spin_unlock(&ctx->lock); |
| 1213 | } | 1219 | } |
| 1214 | rcu_read_unlock(); | 1220 | rcu_read_unlock(); |
| 1215 | 1221 | ||
| @@ -1251,7 +1257,7 @@ __perf_event_sched_in(struct perf_event_context *ctx, | |||
| 1251 | struct perf_event *event; | 1257 | struct perf_event *event; |
| 1252 | int can_add_hw = 1; | 1258 | int can_add_hw = 1; |
| 1253 | 1259 | ||
| 1254 | spin_lock(&ctx->lock); | 1260 | raw_spin_lock(&ctx->lock); |
| 1255 | ctx->is_active = 1; | 1261 | ctx->is_active = 1; |
| 1256 | if (likely(!ctx->nr_events)) | 1262 | if (likely(!ctx->nr_events)) |
| 1257 | goto out; | 1263 | goto out; |
| @@ -1306,7 +1312,7 @@ __perf_event_sched_in(struct perf_event_context *ctx, | |||
| 1306 | } | 1312 | } |
| 1307 | perf_enable(); | 1313 | perf_enable(); |
| 1308 | out: | 1314 | out: |
| 1309 | spin_unlock(&ctx->lock); | 1315 | raw_spin_unlock(&ctx->lock); |
| 1310 | } | 1316 | } |
| 1311 | 1317 | ||
| 1312 | /* | 1318 | /* |
| @@ -1370,11 +1376,14 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx) | |||
| 1370 | struct hw_perf_event *hwc; | 1376 | struct hw_perf_event *hwc; |
| 1371 | u64 interrupts, freq; | 1377 | u64 interrupts, freq; |
| 1372 | 1378 | ||
| 1373 | spin_lock(&ctx->lock); | 1379 | raw_spin_lock(&ctx->lock); |
| 1374 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { | 1380 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { |
| 1375 | if (event->state != PERF_EVENT_STATE_ACTIVE) | 1381 | if (event->state != PERF_EVENT_STATE_ACTIVE) |
| 1376 | continue; | 1382 | continue; |
| 1377 | 1383 | ||
| 1384 | if (event->cpu != -1 && event->cpu != smp_processor_id()) | ||
| 1385 | continue; | ||
| 1386 | |||
| 1378 | hwc = &event->hw; | 1387 | hwc = &event->hw; |
| 1379 | 1388 | ||
| 1380 | interrupts = hwc->interrupts; | 1389 | interrupts = hwc->interrupts; |
| @@ -1425,7 +1434,7 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx) | |||
| 1425 | perf_enable(); | 1434 | perf_enable(); |
| 1426 | } | 1435 | } |
| 1427 | } | 1436 | } |
| 1428 | spin_unlock(&ctx->lock); | 1437 | raw_spin_unlock(&ctx->lock); |
| 1429 | } | 1438 | } |
| 1430 | 1439 | ||
| 1431 | /* | 1440 | /* |
| @@ -1438,7 +1447,7 @@ static void rotate_ctx(struct perf_event_context *ctx) | |||
| 1438 | if (!ctx->nr_events) | 1447 | if (!ctx->nr_events) |
| 1439 | return; | 1448 | return; |
| 1440 | 1449 | ||
| 1441 | spin_lock(&ctx->lock); | 1450 | raw_spin_lock(&ctx->lock); |
| 1442 | /* | 1451 | /* |
| 1443 | * Rotate the first entry last (works just fine for group events too): | 1452 | * Rotate the first entry last (works just fine for group events too): |
| 1444 | */ | 1453 | */ |
| @@ -1449,7 +1458,7 @@ static void rotate_ctx(struct perf_event_context *ctx) | |||
| 1449 | } | 1458 | } |
| 1450 | perf_enable(); | 1459 | perf_enable(); |
| 1451 | 1460 | ||
| 1452 | spin_unlock(&ctx->lock); | 1461 | raw_spin_unlock(&ctx->lock); |
| 1453 | } | 1462 | } |
| 1454 | 1463 | ||
| 1455 | void perf_event_task_tick(struct task_struct *curr, int cpu) | 1464 | void perf_event_task_tick(struct task_struct *curr, int cpu) |
| @@ -1498,7 +1507,7 @@ static void perf_event_enable_on_exec(struct task_struct *task) | |||
| 1498 | 1507 | ||
| 1499 | __perf_event_task_sched_out(ctx); | 1508 | __perf_event_task_sched_out(ctx); |
| 1500 | 1509 | ||
| 1501 | spin_lock(&ctx->lock); | 1510 | raw_spin_lock(&ctx->lock); |
| 1502 | 1511 | ||
| 1503 | list_for_each_entry(event, &ctx->group_list, group_entry) { | 1512 | list_for_each_entry(event, &ctx->group_list, group_entry) { |
| 1504 | if (!event->attr.enable_on_exec) | 1513 | if (!event->attr.enable_on_exec) |
| @@ -1516,7 +1525,7 @@ static void perf_event_enable_on_exec(struct task_struct *task) | |||
| 1516 | if (enabled) | 1525 | if (enabled) |
| 1517 | unclone_ctx(ctx); | 1526 | unclone_ctx(ctx); |
| 1518 | 1527 | ||
| 1519 | spin_unlock(&ctx->lock); | 1528 | raw_spin_unlock(&ctx->lock); |
| 1520 | 1529 | ||
| 1521 | perf_event_task_sched_in(task, smp_processor_id()); | 1530 | perf_event_task_sched_in(task, smp_processor_id()); |
| 1522 | out: | 1531 | out: |
| @@ -1542,10 +1551,10 @@ static void __perf_event_read(void *info) | |||
| 1542 | if (ctx->task && cpuctx->task_ctx != ctx) | 1551 | if (ctx->task && cpuctx->task_ctx != ctx) |
| 1543 | return; | 1552 | return; |
| 1544 | 1553 | ||
| 1545 | spin_lock(&ctx->lock); | 1554 | raw_spin_lock(&ctx->lock); |
| 1546 | update_context_time(ctx); | 1555 | update_context_time(ctx); |
| 1547 | update_event_times(event); | 1556 | update_event_times(event); |
| 1548 | spin_unlock(&ctx->lock); | 1557 | raw_spin_unlock(&ctx->lock); |
| 1549 | 1558 | ||
| 1550 | event->pmu->read(event); | 1559 | event->pmu->read(event); |
| 1551 | } | 1560 | } |
| @@ -1563,10 +1572,10 @@ static u64 perf_event_read(struct perf_event *event) | |||
| 1563 | struct perf_event_context *ctx = event->ctx; | 1572 | struct perf_event_context *ctx = event->ctx; |
| 1564 | unsigned long flags; | 1573 | unsigned long flags; |
| 1565 | 1574 | ||
| 1566 | spin_lock_irqsave(&ctx->lock, flags); | 1575 | raw_spin_lock_irqsave(&ctx->lock, flags); |
| 1567 | update_context_time(ctx); | 1576 | update_context_time(ctx); |
| 1568 | update_event_times(event); | 1577 | update_event_times(event); |
| 1569 | spin_unlock_irqrestore(&ctx->lock, flags); | 1578 | raw_spin_unlock_irqrestore(&ctx->lock, flags); |
| 1570 | } | 1579 | } |
| 1571 | 1580 | ||
| 1572 | return atomic64_read(&event->count); | 1581 | return atomic64_read(&event->count); |
| @@ -1579,8 +1588,7 @@ static void | |||
| 1579 | __perf_event_init_context(struct perf_event_context *ctx, | 1588 | __perf_event_init_context(struct perf_event_context *ctx, |
| 1580 | struct task_struct *task) | 1589 | struct task_struct *task) |
| 1581 | { | 1590 | { |
| 1582 | memset(ctx, 0, sizeof(*ctx)); | 1591 | raw_spin_lock_init(&ctx->lock); |
| 1583 | spin_lock_init(&ctx->lock); | ||
| 1584 | mutex_init(&ctx->mutex); | 1592 | mutex_init(&ctx->mutex); |
| 1585 | INIT_LIST_HEAD(&ctx->group_list); | 1593 | INIT_LIST_HEAD(&ctx->group_list); |
| 1586 | INIT_LIST_HEAD(&ctx->event_list); | 1594 | INIT_LIST_HEAD(&ctx->event_list); |
| @@ -1596,15 +1604,12 @@ static struct perf_event_context *find_get_context(pid_t pid, int cpu) | |||
| 1596 | unsigned long flags; | 1604 | unsigned long flags; |
| 1597 | int err; | 1605 | int err; |
| 1598 | 1606 | ||
| 1599 | /* | 1607 | if (pid == -1 && cpu != -1) { |
| 1600 | * If cpu is not a wildcard then this is a percpu event: | ||
| 1601 | */ | ||
| 1602 | if (cpu != -1) { | ||
| 1603 | /* Must be root to operate on a CPU event: */ | 1608 | /* Must be root to operate on a CPU event: */ |
| 1604 | if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN)) | 1609 | if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN)) |
| 1605 | return ERR_PTR(-EACCES); | 1610 | return ERR_PTR(-EACCES); |
| 1606 | 1611 | ||
| 1607 | if (cpu < 0 || cpu > num_possible_cpus()) | 1612 | if (cpu < 0 || cpu >= nr_cpumask_bits) |
| 1608 | return ERR_PTR(-EINVAL); | 1613 | return ERR_PTR(-EINVAL); |
| 1609 | 1614 | ||
| 1610 | /* | 1615 | /* |
| @@ -1612,7 +1617,7 @@ static struct perf_event_context *find_get_context(pid_t pid, int cpu) | |||
| 1612 | * offline CPU and activate it when the CPU comes up, but | 1617 | * offline CPU and activate it when the CPU comes up, but |
| 1613 | * that's for later. | 1618 | * that's for later. |
| 1614 | */ | 1619 | */ |
| 1615 | if (!cpu_isset(cpu, cpu_online_map)) | 1620 | if (!cpu_online(cpu)) |
| 1616 | return ERR_PTR(-ENODEV); | 1621 | return ERR_PTR(-ENODEV); |
| 1617 | 1622 | ||
| 1618 | cpuctx = &per_cpu(perf_cpu_context, cpu); | 1623 | cpuctx = &per_cpu(perf_cpu_context, cpu); |
| @@ -1650,11 +1655,11 @@ static struct perf_event_context *find_get_context(pid_t pid, int cpu) | |||
| 1650 | ctx = perf_lock_task_context(task, &flags); | 1655 | ctx = perf_lock_task_context(task, &flags); |
| 1651 | if (ctx) { | 1656 | if (ctx) { |
| 1652 | unclone_ctx(ctx); | 1657 | unclone_ctx(ctx); |
| 1653 | spin_unlock_irqrestore(&ctx->lock, flags); | 1658 | raw_spin_unlock_irqrestore(&ctx->lock, flags); |
| 1654 | } | 1659 | } |
| 1655 | 1660 | ||
| 1656 | if (!ctx) { | 1661 | if (!ctx) { |
| 1657 | ctx = kmalloc(sizeof(struct perf_event_context), GFP_KERNEL); | 1662 | ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL); |
| 1658 | err = -ENOMEM; | 1663 | err = -ENOMEM; |
| 1659 | if (!ctx) | 1664 | if (!ctx) |
| 1660 | goto errout; | 1665 | goto errout; |
| @@ -1988,7 +1993,7 @@ static int perf_event_period(struct perf_event *event, u64 __user *arg) | |||
| 1988 | if (!value) | 1993 | if (!value) |
| 1989 | return -EINVAL; | 1994 | return -EINVAL; |
| 1990 | 1995 | ||
| 1991 | spin_lock_irq(&ctx->lock); | 1996 | raw_spin_lock_irq(&ctx->lock); |
| 1992 | if (event->attr.freq) { | 1997 | if (event->attr.freq) { |
| 1993 | if (value > sysctl_perf_event_sample_rate) { | 1998 | if (value > sysctl_perf_event_sample_rate) { |
| 1994 | ret = -EINVAL; | 1999 | ret = -EINVAL; |
| @@ -2001,7 +2006,7 @@ static int perf_event_period(struct perf_event *event, u64 __user *arg) | |||
| 2001 | event->hw.sample_period = value; | 2006 | event->hw.sample_period = value; |
| 2002 | } | 2007 | } |
| 2003 | unlock: | 2008 | unlock: |
| 2004 | spin_unlock_irq(&ctx->lock); | 2009 | raw_spin_unlock_irq(&ctx->lock); |
| 2005 | 2010 | ||
| 2006 | return ret; | 2011 | return ret; |
| 2007 | } | 2012 | } |
| @@ -3263,6 +3268,9 @@ static void perf_event_task_output(struct perf_event *event, | |||
| 3263 | 3268 | ||
| 3264 | static int perf_event_task_match(struct perf_event *event) | 3269 | static int perf_event_task_match(struct perf_event *event) |
| 3265 | { | 3270 | { |
| 3271 | if (event->cpu != -1 && event->cpu != smp_processor_id()) | ||
| 3272 | return 0; | ||
| 3273 | |||
| 3266 | if (event->attr.comm || event->attr.mmap || event->attr.task) | 3274 | if (event->attr.comm || event->attr.mmap || event->attr.task) |
| 3267 | return 1; | 3275 | return 1; |
| 3268 | 3276 | ||
| @@ -3288,12 +3296,11 @@ static void perf_event_task_event(struct perf_task_event *task_event) | |||
| 3288 | rcu_read_lock(); | 3296 | rcu_read_lock(); |
| 3289 | cpuctx = &get_cpu_var(perf_cpu_context); | 3297 | cpuctx = &get_cpu_var(perf_cpu_context); |
| 3290 | perf_event_task_ctx(&cpuctx->ctx, task_event); | 3298 | perf_event_task_ctx(&cpuctx->ctx, task_event); |
| 3291 | put_cpu_var(perf_cpu_context); | ||
| 3292 | |||
| 3293 | if (!ctx) | 3299 | if (!ctx) |
| 3294 | ctx = rcu_dereference(task_event->task->perf_event_ctxp); | 3300 | ctx = rcu_dereference(task_event->task->perf_event_ctxp); |
| 3295 | if (ctx) | 3301 | if (ctx) |
| 3296 | perf_event_task_ctx(ctx, task_event); | 3302 | perf_event_task_ctx(ctx, task_event); |
| 3303 | put_cpu_var(perf_cpu_context); | ||
| 3297 | rcu_read_unlock(); | 3304 | rcu_read_unlock(); |
| 3298 | } | 3305 | } |
| 3299 | 3306 | ||
| @@ -3370,6 +3377,9 @@ static void perf_event_comm_output(struct perf_event *event, | |||
| 3370 | 3377 | ||
| 3371 | static int perf_event_comm_match(struct perf_event *event) | 3378 | static int perf_event_comm_match(struct perf_event *event) |
| 3372 | { | 3379 | { |
| 3380 | if (event->cpu != -1 && event->cpu != smp_processor_id()) | ||
| 3381 | return 0; | ||
| 3382 | |||
| 3373 | if (event->attr.comm) | 3383 | if (event->attr.comm) |
| 3374 | return 1; | 3384 | return 1; |
| 3375 | 3385 | ||
| @@ -3406,15 +3416,10 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event) | |||
| 3406 | rcu_read_lock(); | 3416 | rcu_read_lock(); |
| 3407 | cpuctx = &get_cpu_var(perf_cpu_context); | 3417 | cpuctx = &get_cpu_var(perf_cpu_context); |
| 3408 | perf_event_comm_ctx(&cpuctx->ctx, comm_event); | 3418 | perf_event_comm_ctx(&cpuctx->ctx, comm_event); |
| 3409 | put_cpu_var(perf_cpu_context); | ||
| 3410 | |||
| 3411 | /* | ||
| 3412 | * doesn't really matter which of the child contexts the | ||
| 3413 | * events ends up in. | ||
| 3414 | */ | ||
| 3415 | ctx = rcu_dereference(current->perf_event_ctxp); | 3419 | ctx = rcu_dereference(current->perf_event_ctxp); |
| 3416 | if (ctx) | 3420 | if (ctx) |
| 3417 | perf_event_comm_ctx(ctx, comm_event); | 3421 | perf_event_comm_ctx(ctx, comm_event); |
| 3422 | put_cpu_var(perf_cpu_context); | ||
| 3418 | rcu_read_unlock(); | 3423 | rcu_read_unlock(); |
| 3419 | } | 3424 | } |
| 3420 | 3425 | ||
| @@ -3489,6 +3494,9 @@ static void perf_event_mmap_output(struct perf_event *event, | |||
| 3489 | static int perf_event_mmap_match(struct perf_event *event, | 3494 | static int perf_event_mmap_match(struct perf_event *event, |
| 3490 | struct perf_mmap_event *mmap_event) | 3495 | struct perf_mmap_event *mmap_event) |
| 3491 | { | 3496 | { |
| 3497 | if (event->cpu != -1 && event->cpu != smp_processor_id()) | ||
| 3498 | return 0; | ||
| 3499 | |||
| 3492 | if (event->attr.mmap) | 3500 | if (event->attr.mmap) |
| 3493 | return 1; | 3501 | return 1; |
| 3494 | 3502 | ||
| @@ -3562,15 +3570,10 @@ got_name: | |||
| 3562 | rcu_read_lock(); | 3570 | rcu_read_lock(); |
| 3563 | cpuctx = &get_cpu_var(perf_cpu_context); | 3571 | cpuctx = &get_cpu_var(perf_cpu_context); |
| 3564 | perf_event_mmap_ctx(&cpuctx->ctx, mmap_event); | 3572 | perf_event_mmap_ctx(&cpuctx->ctx, mmap_event); |
| 3565 | put_cpu_var(perf_cpu_context); | ||
| 3566 | |||
| 3567 | /* | ||
| 3568 | * doesn't really matter which of the child contexts the | ||
| 3569 | * events ends up in. | ||
| 3570 | */ | ||
| 3571 | ctx = rcu_dereference(current->perf_event_ctxp); | 3573 | ctx = rcu_dereference(current->perf_event_ctxp); |
| 3572 | if (ctx) | 3574 | if (ctx) |
| 3573 | perf_event_mmap_ctx(ctx, mmap_event); | 3575 | perf_event_mmap_ctx(ctx, mmap_event); |
| 3576 | put_cpu_var(perf_cpu_context); | ||
| 3574 | rcu_read_unlock(); | 3577 | rcu_read_unlock(); |
| 3575 | 3578 | ||
| 3576 | kfree(buf); | 3579 | kfree(buf); |
| @@ -3861,6 +3864,9 @@ static int perf_swevent_match(struct perf_event *event, | |||
| 3861 | struct perf_sample_data *data, | 3864 | struct perf_sample_data *data, |
| 3862 | struct pt_regs *regs) | 3865 | struct pt_regs *regs) |
| 3863 | { | 3866 | { |
| 3867 | if (event->cpu != -1 && event->cpu != smp_processor_id()) | ||
| 3868 | return 0; | ||
| 3869 | |||
| 3864 | if (!perf_swevent_is_counting(event)) | 3870 | if (!perf_swevent_is_counting(event)) |
| 3865 | return 0; | 3871 | return 0; |
| 3866 | 3872 | ||
| @@ -4011,6 +4017,7 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) | |||
| 4011 | event->pmu->read(event); | 4017 | event->pmu->read(event); |
| 4012 | 4018 | ||
| 4013 | data.addr = 0; | 4019 | data.addr = 0; |
| 4020 | data.raw = NULL; | ||
| 4014 | data.period = event->hw.last_period; | 4021 | data.period = event->hw.last_period; |
| 4015 | regs = get_irq_regs(); | 4022 | regs = get_irq_regs(); |
| 4016 | /* | 4023 | /* |
| @@ -4080,8 +4087,7 @@ static void cpu_clock_perf_event_update(struct perf_event *event) | |||
| 4080 | u64 now; | 4087 | u64 now; |
| 4081 | 4088 | ||
| 4082 | now = cpu_clock(cpu); | 4089 | now = cpu_clock(cpu); |
| 4083 | prev = atomic64_read(&event->hw.prev_count); | 4090 | prev = atomic64_xchg(&event->hw.prev_count, now); |
| 4084 | atomic64_set(&event->hw.prev_count, now); | ||
| 4085 | atomic64_add(now - prev, &event->count); | 4091 | atomic64_add(now - prev, &event->count); |
| 4086 | } | 4092 | } |
| 4087 | 4093 | ||
| @@ -4286,15 +4292,8 @@ static void bp_perf_event_destroy(struct perf_event *event) | |||
| 4286 | static const struct pmu *bp_perf_event_init(struct perf_event *bp) | 4292 | static const struct pmu *bp_perf_event_init(struct perf_event *bp) |
| 4287 | { | 4293 | { |
| 4288 | int err; | 4294 | int err; |
| 4289 | /* | 4295 | |
| 4290 | * The breakpoint is already filled if we haven't created the counter | 4296 | err = register_perf_hw_breakpoint(bp); |
| 4291 | * through perf syscall | ||
| 4292 | * FIXME: manage to get trigerred to NULL if it comes from syscalls | ||
| 4293 | */ | ||
| 4294 | if (!bp->callback) | ||
| 4295 | err = register_perf_hw_breakpoint(bp); | ||
| 4296 | else | ||
| 4297 | err = __register_perf_hw_breakpoint(bp); | ||
| 4298 | if (err) | 4297 | if (err) |
| 4299 | return ERR_PTR(err); | 4298 | return ERR_PTR(err); |
| 4300 | 4299 | ||
| @@ -4308,6 +4307,7 @@ void perf_bp_event(struct perf_event *bp, void *data) | |||
| 4308 | struct perf_sample_data sample; | 4307 | struct perf_sample_data sample; |
| 4309 | struct pt_regs *regs = data; | 4308 | struct pt_regs *regs = data; |
| 4310 | 4309 | ||
| 4310 | sample.raw = NULL; | ||
| 4311 | sample.addr = bp->attr.bp_addr; | 4311 | sample.addr = bp->attr.bp_addr; |
| 4312 | 4312 | ||
| 4313 | if (!perf_exclude_event(bp, regs)) | 4313 | if (!perf_exclude_event(bp, regs)) |
| @@ -4390,7 +4390,7 @@ perf_event_alloc(struct perf_event_attr *attr, | |||
| 4390 | struct perf_event_context *ctx, | 4390 | struct perf_event_context *ctx, |
| 4391 | struct perf_event *group_leader, | 4391 | struct perf_event *group_leader, |
| 4392 | struct perf_event *parent_event, | 4392 | struct perf_event *parent_event, |
| 4393 | perf_callback_t callback, | 4393 | perf_overflow_handler_t overflow_handler, |
| 4394 | gfp_t gfpflags) | 4394 | gfp_t gfpflags) |
| 4395 | { | 4395 | { |
| 4396 | const struct pmu *pmu; | 4396 | const struct pmu *pmu; |
| @@ -4433,10 +4433,10 @@ perf_event_alloc(struct perf_event_attr *attr, | |||
| 4433 | 4433 | ||
| 4434 | event->state = PERF_EVENT_STATE_INACTIVE; | 4434 | event->state = PERF_EVENT_STATE_INACTIVE; |
| 4435 | 4435 | ||
| 4436 | if (!callback && parent_event) | 4436 | if (!overflow_handler && parent_event) |
| 4437 | callback = parent_event->callback; | 4437 | overflow_handler = parent_event->overflow_handler; |
| 4438 | 4438 | ||
| 4439 | event->callback = callback; | 4439 | event->overflow_handler = overflow_handler; |
| 4440 | 4440 | ||
| 4441 | if (attr->disabled) | 4441 | if (attr->disabled) |
| 4442 | event->state = PERF_EVENT_STATE_OFF; | 4442 | event->state = PERF_EVENT_STATE_OFF; |
| @@ -4571,7 +4571,7 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, | |||
| 4571 | if (attr->type >= PERF_TYPE_MAX) | 4571 | if (attr->type >= PERF_TYPE_MAX) |
| 4572 | return -EINVAL; | 4572 | return -EINVAL; |
| 4573 | 4573 | ||
| 4574 | if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3) | 4574 | if (attr->__reserved_1 || attr->__reserved_2) |
| 4575 | return -EINVAL; | 4575 | return -EINVAL; |
| 4576 | 4576 | ||
| 4577 | if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) | 4577 | if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) |
| @@ -4724,7 +4724,7 @@ SYSCALL_DEFINE5(perf_event_open, | |||
| 4724 | if (IS_ERR(event)) | 4724 | if (IS_ERR(event)) |
| 4725 | goto err_put_context; | 4725 | goto err_put_context; |
| 4726 | 4726 | ||
| 4727 | err = anon_inode_getfd("[perf_event]", &perf_fops, event, 0); | 4727 | err = anon_inode_getfd("[perf_event]", &perf_fops, event, O_RDWR); |
| 4728 | if (err < 0) | 4728 | if (err < 0) |
| 4729 | goto err_free_put_context; | 4729 | goto err_free_put_context; |
| 4730 | 4730 | ||
| @@ -4776,7 +4776,8 @@ err_put_context: | |||
| 4776 | */ | 4776 | */ |
| 4777 | struct perf_event * | 4777 | struct perf_event * |
| 4778 | perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, | 4778 | perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, |
| 4779 | pid_t pid, perf_callback_t callback) | 4779 | pid_t pid, |
| 4780 | perf_overflow_handler_t overflow_handler) | ||
| 4780 | { | 4781 | { |
| 4781 | struct perf_event *event; | 4782 | struct perf_event *event; |
| 4782 | struct perf_event_context *ctx; | 4783 | struct perf_event_context *ctx; |
| @@ -4793,7 +4794,7 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, | |||
| 4793 | } | 4794 | } |
| 4794 | 4795 | ||
| 4795 | event = perf_event_alloc(attr, cpu, ctx, NULL, | 4796 | event = perf_event_alloc(attr, cpu, ctx, NULL, |
| 4796 | NULL, callback, GFP_KERNEL); | 4797 | NULL, overflow_handler, GFP_KERNEL); |
| 4797 | if (IS_ERR(event)) { | 4798 | if (IS_ERR(event)) { |
| 4798 | err = PTR_ERR(event); | 4799 | err = PTR_ERR(event); |
| 4799 | goto err_put_context; | 4800 | goto err_put_context; |
| @@ -4998,7 +4999,7 @@ void perf_event_exit_task(struct task_struct *child) | |||
| 4998 | * reading child->perf_event_ctxp, we wait until it has | 4999 | * reading child->perf_event_ctxp, we wait until it has |
| 4999 | * incremented the context's refcount before we do put_ctx below. | 5000 | * incremented the context's refcount before we do put_ctx below. |
| 5000 | */ | 5001 | */ |
| 5001 | spin_lock(&child_ctx->lock); | 5002 | raw_spin_lock(&child_ctx->lock); |
| 5002 | child->perf_event_ctxp = NULL; | 5003 | child->perf_event_ctxp = NULL; |
| 5003 | /* | 5004 | /* |
| 5004 | * If this context is a clone; unclone it so it can't get | 5005 | * If this context is a clone; unclone it so it can't get |
| @@ -5007,7 +5008,7 @@ void perf_event_exit_task(struct task_struct *child) | |||
| 5007 | */ | 5008 | */ |
| 5008 | unclone_ctx(child_ctx); | 5009 | unclone_ctx(child_ctx); |
| 5009 | update_context_time(child_ctx); | 5010 | update_context_time(child_ctx); |
| 5010 | spin_unlock_irqrestore(&child_ctx->lock, flags); | 5011 | raw_spin_unlock_irqrestore(&child_ctx->lock, flags); |
| 5011 | 5012 | ||
| 5012 | /* | 5013 | /* |
| 5013 | * Report the task dead after unscheduling the events so that we | 5014 | * Report the task dead after unscheduling the events so that we |
| @@ -5090,7 +5091,7 @@ again: | |||
| 5090 | */ | 5091 | */ |
| 5091 | int perf_event_init_task(struct task_struct *child) | 5092 | int perf_event_init_task(struct task_struct *child) |
| 5092 | { | 5093 | { |
| 5093 | struct perf_event_context *child_ctx, *parent_ctx; | 5094 | struct perf_event_context *child_ctx = NULL, *parent_ctx; |
| 5094 | struct perf_event_context *cloned_ctx; | 5095 | struct perf_event_context *cloned_ctx; |
| 5095 | struct perf_event *event; | 5096 | struct perf_event *event; |
| 5096 | struct task_struct *parent = current; | 5097 | struct task_struct *parent = current; |
| @@ -5106,20 +5107,6 @@ int perf_event_init_task(struct task_struct *child) | |||
| 5106 | return 0; | 5107 | return 0; |
| 5107 | 5108 | ||
| 5108 | /* | 5109 | /* |
| 5109 | * This is executed from the parent task context, so inherit | ||
| 5110 | * events that have been marked for cloning. | ||
| 5111 | * First allocate and initialize a context for the child. | ||
| 5112 | */ | ||
| 5113 | |||
| 5114 | child_ctx = kmalloc(sizeof(struct perf_event_context), GFP_KERNEL); | ||
| 5115 | if (!child_ctx) | ||
| 5116 | return -ENOMEM; | ||
| 5117 | |||
| 5118 | __perf_event_init_context(child_ctx, child); | ||
| 5119 | child->perf_event_ctxp = child_ctx; | ||
| 5120 | get_task_struct(child); | ||
| 5121 | |||
| 5122 | /* | ||
| 5123 | * If the parent's context is a clone, pin it so it won't get | 5110 | * If the parent's context is a clone, pin it so it won't get |
| 5124 | * swapped under us. | 5111 | * swapped under us. |
| 5125 | */ | 5112 | */ |
| @@ -5149,6 +5136,26 @@ int perf_event_init_task(struct task_struct *child) | |||
| 5149 | continue; | 5136 | continue; |
| 5150 | } | 5137 | } |
| 5151 | 5138 | ||
| 5139 | if (!child->perf_event_ctxp) { | ||
| 5140 | /* | ||
| 5141 | * This is executed from the parent task context, so | ||
| 5142 | * inherit events that have been marked for cloning. | ||
| 5143 | * First allocate and initialize a context for the | ||
| 5144 | * child. | ||
| 5145 | */ | ||
| 5146 | |||
| 5147 | child_ctx = kzalloc(sizeof(struct perf_event_context), | ||
| 5148 | GFP_KERNEL); | ||
| 5149 | if (!child_ctx) { | ||
| 5150 | ret = -ENOMEM; | ||
| 5151 | break; | ||
| 5152 | } | ||
| 5153 | |||
| 5154 | __perf_event_init_context(child_ctx, child); | ||
| 5155 | child->perf_event_ctxp = child_ctx; | ||
| 5156 | get_task_struct(child); | ||
| 5157 | } | ||
| 5158 | |||
| 5152 | ret = inherit_group(event, parent, parent_ctx, | 5159 | ret = inherit_group(event, parent, parent_ctx, |
| 5153 | child, child_ctx); | 5160 | child, child_ctx); |
| 5154 | if (ret) { | 5161 | if (ret) { |
| @@ -5157,7 +5164,7 @@ int perf_event_init_task(struct task_struct *child) | |||
| 5157 | } | 5164 | } |
| 5158 | } | 5165 | } |
| 5159 | 5166 | ||
| 5160 | if (inherited_all) { | 5167 | if (child_ctx && inherited_all) { |
| 5161 | /* | 5168 | /* |
| 5162 | * Mark the child context as a clone of the parent | 5169 | * Mark the child context as a clone of the parent |
| 5163 | * context, or of whatever the parent is a clone of. | 5170 | * context, or of whatever the parent is a clone of. |
| @@ -5291,11 +5298,11 @@ perf_set_reserve_percpu(struct sysdev_class *class, | |||
| 5291 | perf_reserved_percpu = val; | 5298 | perf_reserved_percpu = val; |
| 5292 | for_each_online_cpu(cpu) { | 5299 | for_each_online_cpu(cpu) { |
| 5293 | cpuctx = &per_cpu(perf_cpu_context, cpu); | 5300 | cpuctx = &per_cpu(perf_cpu_context, cpu); |
| 5294 | spin_lock_irq(&cpuctx->ctx.lock); | 5301 | raw_spin_lock_irq(&cpuctx->ctx.lock); |
| 5295 | mpt = min(perf_max_events - cpuctx->ctx.nr_events, | 5302 | mpt = min(perf_max_events - cpuctx->ctx.nr_events, |
| 5296 | perf_max_events - perf_reserved_percpu); | 5303 | perf_max_events - perf_reserved_percpu); |
| 5297 | cpuctx->max_pertask = mpt; | 5304 | cpuctx->max_pertask = mpt; |
| 5298 | spin_unlock_irq(&cpuctx->ctx.lock); | 5305 | raw_spin_unlock_irq(&cpuctx->ctx.lock); |
| 5299 | } | 5306 | } |
| 5300 | spin_unlock(&perf_resource_lock); | 5307 | spin_unlock(&perf_resource_lock); |
| 5301 | 5308 | ||
