aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/perf_event.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/perf_event.c')
-rw-r--r--kernel/perf_event.c233
1 files changed, 120 insertions, 113 deletions
diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index 40a996ec39fa..603c0d8b5df1 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -36,7 +36,7 @@
36/* 36/*
37 * Each CPU has a list of per CPU events: 37 * Each CPU has a list of per CPU events:
38 */ 38 */
39DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context); 39static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
40 40
41int perf_max_events __read_mostly = 1; 41int perf_max_events __read_mostly = 1;
42static int perf_reserved_percpu __read_mostly; 42static int perf_reserved_percpu __read_mostly;
@@ -203,14 +203,14 @@ perf_lock_task_context(struct task_struct *task, unsigned long *flags)
203 * if so. If we locked the right context, then it 203 * if so. If we locked the right context, then it
204 * can't get swapped on us any more. 204 * can't get swapped on us any more.
205 */ 205 */
206 spin_lock_irqsave(&ctx->lock, *flags); 206 raw_spin_lock_irqsave(&ctx->lock, *flags);
207 if (ctx != rcu_dereference(task->perf_event_ctxp)) { 207 if (ctx != rcu_dereference(task->perf_event_ctxp)) {
208 spin_unlock_irqrestore(&ctx->lock, *flags); 208 raw_spin_unlock_irqrestore(&ctx->lock, *flags);
209 goto retry; 209 goto retry;
210 } 210 }
211 211
212 if (!atomic_inc_not_zero(&ctx->refcount)) { 212 if (!atomic_inc_not_zero(&ctx->refcount)) {
213 spin_unlock_irqrestore(&ctx->lock, *flags); 213 raw_spin_unlock_irqrestore(&ctx->lock, *flags);
214 ctx = NULL; 214 ctx = NULL;
215 } 215 }
216 } 216 }
@@ -231,7 +231,7 @@ static struct perf_event_context *perf_pin_task_context(struct task_struct *task
231 ctx = perf_lock_task_context(task, &flags); 231 ctx = perf_lock_task_context(task, &flags);
232 if (ctx) { 232 if (ctx) {
233 ++ctx->pin_count; 233 ++ctx->pin_count;
234 spin_unlock_irqrestore(&ctx->lock, flags); 234 raw_spin_unlock_irqrestore(&ctx->lock, flags);
235 } 235 }
236 return ctx; 236 return ctx;
237} 237}
@@ -240,9 +240,9 @@ static void perf_unpin_context(struct perf_event_context *ctx)
240{ 240{
241 unsigned long flags; 241 unsigned long flags;
242 242
243 spin_lock_irqsave(&ctx->lock, flags); 243 raw_spin_lock_irqsave(&ctx->lock, flags);
244 --ctx->pin_count; 244 --ctx->pin_count;
245 spin_unlock_irqrestore(&ctx->lock, flags); 245 raw_spin_unlock_irqrestore(&ctx->lock, flags);
246 put_ctx(ctx); 246 put_ctx(ctx);
247} 247}
248 248
@@ -427,7 +427,7 @@ static void __perf_event_remove_from_context(void *info)
427 if (ctx->task && cpuctx->task_ctx != ctx) 427 if (ctx->task && cpuctx->task_ctx != ctx)
428 return; 428 return;
429 429
430 spin_lock(&ctx->lock); 430 raw_spin_lock(&ctx->lock);
431 /* 431 /*
432 * Protect the list operation against NMI by disabling the 432 * Protect the list operation against NMI by disabling the
433 * events on a global level. 433 * events on a global level.
@@ -449,7 +449,7 @@ static void __perf_event_remove_from_context(void *info)
449 } 449 }
450 450
451 perf_enable(); 451 perf_enable();
452 spin_unlock(&ctx->lock); 452 raw_spin_unlock(&ctx->lock);
453} 453}
454 454
455 455
@@ -488,12 +488,12 @@ retry:
488 task_oncpu_function_call(task, __perf_event_remove_from_context, 488 task_oncpu_function_call(task, __perf_event_remove_from_context,
489 event); 489 event);
490 490
491 spin_lock_irq(&ctx->lock); 491 raw_spin_lock_irq(&ctx->lock);
492 /* 492 /*
493 * If the context is active we need to retry the smp call. 493 * If the context is active we need to retry the smp call.
494 */ 494 */
495 if (ctx->nr_active && !list_empty(&event->group_entry)) { 495 if (ctx->nr_active && !list_empty(&event->group_entry)) {
496 spin_unlock_irq(&ctx->lock); 496 raw_spin_unlock_irq(&ctx->lock);
497 goto retry; 497 goto retry;
498 } 498 }
499 499
@@ -504,7 +504,7 @@ retry:
504 */ 504 */
505 if (!list_empty(&event->group_entry)) 505 if (!list_empty(&event->group_entry))
506 list_del_event(event, ctx); 506 list_del_event(event, ctx);
507 spin_unlock_irq(&ctx->lock); 507 raw_spin_unlock_irq(&ctx->lock);
508} 508}
509 509
510/* 510/*
@@ -535,7 +535,7 @@ static void __perf_event_disable(void *info)
535 if (ctx->task && cpuctx->task_ctx != ctx) 535 if (ctx->task && cpuctx->task_ctx != ctx)
536 return; 536 return;
537 537
538 spin_lock(&ctx->lock); 538 raw_spin_lock(&ctx->lock);
539 539
540 /* 540 /*
541 * If the event is on, turn it off. 541 * If the event is on, turn it off.
@@ -551,7 +551,7 @@ static void __perf_event_disable(void *info)
551 event->state = PERF_EVENT_STATE_OFF; 551 event->state = PERF_EVENT_STATE_OFF;
552 } 552 }
553 553
554 spin_unlock(&ctx->lock); 554 raw_spin_unlock(&ctx->lock);
555} 555}
556 556
557/* 557/*
@@ -567,7 +567,7 @@ static void __perf_event_disable(void *info)
567 * is the current context on this CPU and preemption is disabled, 567 * is the current context on this CPU and preemption is disabled,
568 * hence we can't get into perf_event_task_sched_out for this context. 568 * hence we can't get into perf_event_task_sched_out for this context.
569 */ 569 */
570static void perf_event_disable(struct perf_event *event) 570void perf_event_disable(struct perf_event *event)
571{ 571{
572 struct perf_event_context *ctx = event->ctx; 572 struct perf_event_context *ctx = event->ctx;
573 struct task_struct *task = ctx->task; 573 struct task_struct *task = ctx->task;
@@ -584,12 +584,12 @@ static void perf_event_disable(struct perf_event *event)
584 retry: 584 retry:
585 task_oncpu_function_call(task, __perf_event_disable, event); 585 task_oncpu_function_call(task, __perf_event_disable, event);
586 586
587 spin_lock_irq(&ctx->lock); 587 raw_spin_lock_irq(&ctx->lock);
588 /* 588 /*
589 * If the event is still active, we need to retry the cross-call. 589 * If the event is still active, we need to retry the cross-call.
590 */ 590 */
591 if (event->state == PERF_EVENT_STATE_ACTIVE) { 591 if (event->state == PERF_EVENT_STATE_ACTIVE) {
592 spin_unlock_irq(&ctx->lock); 592 raw_spin_unlock_irq(&ctx->lock);
593 goto retry; 593 goto retry;
594 } 594 }
595 595
@@ -602,7 +602,7 @@ static void perf_event_disable(struct perf_event *event)
602 event->state = PERF_EVENT_STATE_OFF; 602 event->state = PERF_EVENT_STATE_OFF;
603 } 603 }
604 604
605 spin_unlock_irq(&ctx->lock); 605 raw_spin_unlock_irq(&ctx->lock);
606} 606}
607 607
608static int 608static int
@@ -770,7 +770,7 @@ static void __perf_install_in_context(void *info)
770 cpuctx->task_ctx = ctx; 770 cpuctx->task_ctx = ctx;
771 } 771 }
772 772
773 spin_lock(&ctx->lock); 773 raw_spin_lock(&ctx->lock);
774 ctx->is_active = 1; 774 ctx->is_active = 1;
775 update_context_time(ctx); 775 update_context_time(ctx);
776 776
@@ -782,6 +782,9 @@ static void __perf_install_in_context(void *info)
782 782
783 add_event_to_ctx(event, ctx); 783 add_event_to_ctx(event, ctx);
784 784
785 if (event->cpu != -1 && event->cpu != smp_processor_id())
786 goto unlock;
787
785 /* 788 /*
786 * Don't put the event on if it is disabled or if 789 * Don't put the event on if it is disabled or if
787 * it is in a group and the group isn't on. 790 * it is in a group and the group isn't on.
@@ -820,7 +823,7 @@ static void __perf_install_in_context(void *info)
820 unlock: 823 unlock:
821 perf_enable(); 824 perf_enable();
822 825
823 spin_unlock(&ctx->lock); 826 raw_spin_unlock(&ctx->lock);
824} 827}
825 828
826/* 829/*
@@ -856,12 +859,12 @@ retry:
856 task_oncpu_function_call(task, __perf_install_in_context, 859 task_oncpu_function_call(task, __perf_install_in_context,
857 event); 860 event);
858 861
859 spin_lock_irq(&ctx->lock); 862 raw_spin_lock_irq(&ctx->lock);
860 /* 863 /*
861 * we need to retry the smp call. 864 * we need to retry the smp call.
862 */ 865 */
863 if (ctx->is_active && list_empty(&event->group_entry)) { 866 if (ctx->is_active && list_empty(&event->group_entry)) {
864 spin_unlock_irq(&ctx->lock); 867 raw_spin_unlock_irq(&ctx->lock);
865 goto retry; 868 goto retry;
866 } 869 }
867 870
@@ -872,7 +875,7 @@ retry:
872 */ 875 */
873 if (list_empty(&event->group_entry)) 876 if (list_empty(&event->group_entry))
874 add_event_to_ctx(event, ctx); 877 add_event_to_ctx(event, ctx);
875 spin_unlock_irq(&ctx->lock); 878 raw_spin_unlock_irq(&ctx->lock);
876} 879}
877 880
878/* 881/*
@@ -917,7 +920,7 @@ static void __perf_event_enable(void *info)
917 cpuctx->task_ctx = ctx; 920 cpuctx->task_ctx = ctx;
918 } 921 }
919 922
920 spin_lock(&ctx->lock); 923 raw_spin_lock(&ctx->lock);
921 ctx->is_active = 1; 924 ctx->is_active = 1;
922 update_context_time(ctx); 925 update_context_time(ctx);
923 926
@@ -925,6 +928,9 @@ static void __perf_event_enable(void *info)
925 goto unlock; 928 goto unlock;
926 __perf_event_mark_enabled(event, ctx); 929 __perf_event_mark_enabled(event, ctx);
927 930
931 if (event->cpu != -1 && event->cpu != smp_processor_id())
932 goto unlock;
933
928 /* 934 /*
929 * If the event is in a group and isn't the group leader, 935 * If the event is in a group and isn't the group leader,
930 * then don't put it on unless the group is on. 936 * then don't put it on unless the group is on.
@@ -959,7 +965,7 @@ static void __perf_event_enable(void *info)
959 } 965 }
960 966
961 unlock: 967 unlock:
962 spin_unlock(&ctx->lock); 968 raw_spin_unlock(&ctx->lock);
963} 969}
964 970
965/* 971/*
@@ -971,7 +977,7 @@ static void __perf_event_enable(void *info)
971 * perf_event_for_each_child or perf_event_for_each as described 977 * perf_event_for_each_child or perf_event_for_each as described
972 * for perf_event_disable. 978 * for perf_event_disable.
973 */ 979 */
974static void perf_event_enable(struct perf_event *event) 980void perf_event_enable(struct perf_event *event)
975{ 981{
976 struct perf_event_context *ctx = event->ctx; 982 struct perf_event_context *ctx = event->ctx;
977 struct task_struct *task = ctx->task; 983 struct task_struct *task = ctx->task;
@@ -985,7 +991,7 @@ static void perf_event_enable(struct perf_event *event)
985 return; 991 return;
986 } 992 }
987 993
988 spin_lock_irq(&ctx->lock); 994 raw_spin_lock_irq(&ctx->lock);
989 if (event->state >= PERF_EVENT_STATE_INACTIVE) 995 if (event->state >= PERF_EVENT_STATE_INACTIVE)
990 goto out; 996 goto out;
991 997
@@ -1000,10 +1006,10 @@ static void perf_event_enable(struct perf_event *event)
1000 event->state = PERF_EVENT_STATE_OFF; 1006 event->state = PERF_EVENT_STATE_OFF;
1001 1007
1002 retry: 1008 retry:
1003 spin_unlock_irq(&ctx->lock); 1009 raw_spin_unlock_irq(&ctx->lock);
1004 task_oncpu_function_call(task, __perf_event_enable, event); 1010 task_oncpu_function_call(task, __perf_event_enable, event);
1005 1011
1006 spin_lock_irq(&ctx->lock); 1012 raw_spin_lock_irq(&ctx->lock);
1007 1013
1008 /* 1014 /*
1009 * If the context is active and the event is still off, 1015 * If the context is active and the event is still off,
@@ -1020,7 +1026,7 @@ static void perf_event_enable(struct perf_event *event)
1020 __perf_event_mark_enabled(event, ctx); 1026 __perf_event_mark_enabled(event, ctx);
1021 1027
1022 out: 1028 out:
1023 spin_unlock_irq(&ctx->lock); 1029 raw_spin_unlock_irq(&ctx->lock);
1024} 1030}
1025 1031
1026static int perf_event_refresh(struct perf_event *event, int refresh) 1032static int perf_event_refresh(struct perf_event *event, int refresh)
@@ -1042,7 +1048,7 @@ void __perf_event_sched_out(struct perf_event_context *ctx,
1042{ 1048{
1043 struct perf_event *event; 1049 struct perf_event *event;
1044 1050
1045 spin_lock(&ctx->lock); 1051 raw_spin_lock(&ctx->lock);
1046 ctx->is_active = 0; 1052 ctx->is_active = 0;
1047 if (likely(!ctx->nr_events)) 1053 if (likely(!ctx->nr_events))
1048 goto out; 1054 goto out;
@@ -1055,7 +1061,7 @@ void __perf_event_sched_out(struct perf_event_context *ctx,
1055 } 1061 }
1056 perf_enable(); 1062 perf_enable();
1057 out: 1063 out:
1058 spin_unlock(&ctx->lock); 1064 raw_spin_unlock(&ctx->lock);
1059} 1065}
1060 1066
1061/* 1067/*
@@ -1193,8 +1199,8 @@ void perf_event_task_sched_out(struct task_struct *task,
1193 * order we take the locks because no other cpu could 1199 * order we take the locks because no other cpu could
1194 * be trying to lock both of these tasks. 1200 * be trying to lock both of these tasks.
1195 */ 1201 */
1196 spin_lock(&ctx->lock); 1202 raw_spin_lock(&ctx->lock);
1197 spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); 1203 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1198 if (context_equiv(ctx, next_ctx)) { 1204 if (context_equiv(ctx, next_ctx)) {
1199 /* 1205 /*
1200 * XXX do we need a memory barrier of sorts 1206 * XXX do we need a memory barrier of sorts
@@ -1208,8 +1214,8 @@ void perf_event_task_sched_out(struct task_struct *task,
1208 1214
1209 perf_event_sync_stat(ctx, next_ctx); 1215 perf_event_sync_stat(ctx, next_ctx);
1210 } 1216 }
1211 spin_unlock(&next_ctx->lock); 1217 raw_spin_unlock(&next_ctx->lock);
1212 spin_unlock(&ctx->lock); 1218 raw_spin_unlock(&ctx->lock);
1213 } 1219 }
1214 rcu_read_unlock(); 1220 rcu_read_unlock();
1215 1221
@@ -1251,7 +1257,7 @@ __perf_event_sched_in(struct perf_event_context *ctx,
1251 struct perf_event *event; 1257 struct perf_event *event;
1252 int can_add_hw = 1; 1258 int can_add_hw = 1;
1253 1259
1254 spin_lock(&ctx->lock); 1260 raw_spin_lock(&ctx->lock);
1255 ctx->is_active = 1; 1261 ctx->is_active = 1;
1256 if (likely(!ctx->nr_events)) 1262 if (likely(!ctx->nr_events))
1257 goto out; 1263 goto out;
@@ -1306,7 +1312,7 @@ __perf_event_sched_in(struct perf_event_context *ctx,
1306 } 1312 }
1307 perf_enable(); 1313 perf_enable();
1308 out: 1314 out:
1309 spin_unlock(&ctx->lock); 1315 raw_spin_unlock(&ctx->lock);
1310} 1316}
1311 1317
1312/* 1318/*
@@ -1370,11 +1376,14 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
1370 struct hw_perf_event *hwc; 1376 struct hw_perf_event *hwc;
1371 u64 interrupts, freq; 1377 u64 interrupts, freq;
1372 1378
1373 spin_lock(&ctx->lock); 1379 raw_spin_lock(&ctx->lock);
1374 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 1380 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
1375 if (event->state != PERF_EVENT_STATE_ACTIVE) 1381 if (event->state != PERF_EVENT_STATE_ACTIVE)
1376 continue; 1382 continue;
1377 1383
1384 if (event->cpu != -1 && event->cpu != smp_processor_id())
1385 continue;
1386
1378 hwc = &event->hw; 1387 hwc = &event->hw;
1379 1388
1380 interrupts = hwc->interrupts; 1389 interrupts = hwc->interrupts;
@@ -1425,7 +1434,7 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
1425 perf_enable(); 1434 perf_enable();
1426 } 1435 }
1427 } 1436 }
1428 spin_unlock(&ctx->lock); 1437 raw_spin_unlock(&ctx->lock);
1429} 1438}
1430 1439
1431/* 1440/*
@@ -1438,7 +1447,7 @@ static void rotate_ctx(struct perf_event_context *ctx)
1438 if (!ctx->nr_events) 1447 if (!ctx->nr_events)
1439 return; 1448 return;
1440 1449
1441 spin_lock(&ctx->lock); 1450 raw_spin_lock(&ctx->lock);
1442 /* 1451 /*
1443 * Rotate the first entry last (works just fine for group events too): 1452 * Rotate the first entry last (works just fine for group events too):
1444 */ 1453 */
@@ -1449,7 +1458,7 @@ static void rotate_ctx(struct perf_event_context *ctx)
1449 } 1458 }
1450 perf_enable(); 1459 perf_enable();
1451 1460
1452 spin_unlock(&ctx->lock); 1461 raw_spin_unlock(&ctx->lock);
1453} 1462}
1454 1463
1455void perf_event_task_tick(struct task_struct *curr, int cpu) 1464void perf_event_task_tick(struct task_struct *curr, int cpu)
@@ -1498,7 +1507,7 @@ static void perf_event_enable_on_exec(struct task_struct *task)
1498 1507
1499 __perf_event_task_sched_out(ctx); 1508 __perf_event_task_sched_out(ctx);
1500 1509
1501 spin_lock(&ctx->lock); 1510 raw_spin_lock(&ctx->lock);
1502 1511
1503 list_for_each_entry(event, &ctx->group_list, group_entry) { 1512 list_for_each_entry(event, &ctx->group_list, group_entry) {
1504 if (!event->attr.enable_on_exec) 1513 if (!event->attr.enable_on_exec)
@@ -1516,7 +1525,7 @@ static void perf_event_enable_on_exec(struct task_struct *task)
1516 if (enabled) 1525 if (enabled)
1517 unclone_ctx(ctx); 1526 unclone_ctx(ctx);
1518 1527
1519 spin_unlock(&ctx->lock); 1528 raw_spin_unlock(&ctx->lock);
1520 1529
1521 perf_event_task_sched_in(task, smp_processor_id()); 1530 perf_event_task_sched_in(task, smp_processor_id());
1522 out: 1531 out:
@@ -1542,10 +1551,10 @@ static void __perf_event_read(void *info)
1542 if (ctx->task && cpuctx->task_ctx != ctx) 1551 if (ctx->task && cpuctx->task_ctx != ctx)
1543 return; 1552 return;
1544 1553
1545 spin_lock(&ctx->lock); 1554 raw_spin_lock(&ctx->lock);
1546 update_context_time(ctx); 1555 update_context_time(ctx);
1547 update_event_times(event); 1556 update_event_times(event);
1548 spin_unlock(&ctx->lock); 1557 raw_spin_unlock(&ctx->lock);
1549 1558
1550 event->pmu->read(event); 1559 event->pmu->read(event);
1551} 1560}
@@ -1563,10 +1572,10 @@ static u64 perf_event_read(struct perf_event *event)
1563 struct perf_event_context *ctx = event->ctx; 1572 struct perf_event_context *ctx = event->ctx;
1564 unsigned long flags; 1573 unsigned long flags;
1565 1574
1566 spin_lock_irqsave(&ctx->lock, flags); 1575 raw_spin_lock_irqsave(&ctx->lock, flags);
1567 update_context_time(ctx); 1576 update_context_time(ctx);
1568 update_event_times(event); 1577 update_event_times(event);
1569 spin_unlock_irqrestore(&ctx->lock, flags); 1578 raw_spin_unlock_irqrestore(&ctx->lock, flags);
1570 } 1579 }
1571 1580
1572 return atomic64_read(&event->count); 1581 return atomic64_read(&event->count);
@@ -1579,8 +1588,7 @@ static void
1579__perf_event_init_context(struct perf_event_context *ctx, 1588__perf_event_init_context(struct perf_event_context *ctx,
1580 struct task_struct *task) 1589 struct task_struct *task)
1581{ 1590{
1582 memset(ctx, 0, sizeof(*ctx)); 1591 raw_spin_lock_init(&ctx->lock);
1583 spin_lock_init(&ctx->lock);
1584 mutex_init(&ctx->mutex); 1592 mutex_init(&ctx->mutex);
1585 INIT_LIST_HEAD(&ctx->group_list); 1593 INIT_LIST_HEAD(&ctx->group_list);
1586 INIT_LIST_HEAD(&ctx->event_list); 1594 INIT_LIST_HEAD(&ctx->event_list);
@@ -1596,15 +1604,12 @@ static struct perf_event_context *find_get_context(pid_t pid, int cpu)
1596 unsigned long flags; 1604 unsigned long flags;
1597 int err; 1605 int err;
1598 1606
1599 /* 1607 if (pid == -1 && cpu != -1) {
1600 * If cpu is not a wildcard then this is a percpu event:
1601 */
1602 if (cpu != -1) {
1603 /* Must be root to operate on a CPU event: */ 1608 /* Must be root to operate on a CPU event: */
1604 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN)) 1609 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
1605 return ERR_PTR(-EACCES); 1610 return ERR_PTR(-EACCES);
1606 1611
1607 if (cpu < 0 || cpu > num_possible_cpus()) 1612 if (cpu < 0 || cpu >= nr_cpumask_bits)
1608 return ERR_PTR(-EINVAL); 1613 return ERR_PTR(-EINVAL);
1609 1614
1610 /* 1615 /*
@@ -1612,7 +1617,7 @@ static struct perf_event_context *find_get_context(pid_t pid, int cpu)
1612 * offline CPU and activate it when the CPU comes up, but 1617 * offline CPU and activate it when the CPU comes up, but
1613 * that's for later. 1618 * that's for later.
1614 */ 1619 */
1615 if (!cpu_isset(cpu, cpu_online_map)) 1620 if (!cpu_online(cpu))
1616 return ERR_PTR(-ENODEV); 1621 return ERR_PTR(-ENODEV);
1617 1622
1618 cpuctx = &per_cpu(perf_cpu_context, cpu); 1623 cpuctx = &per_cpu(perf_cpu_context, cpu);
@@ -1650,11 +1655,11 @@ static struct perf_event_context *find_get_context(pid_t pid, int cpu)
1650 ctx = perf_lock_task_context(task, &flags); 1655 ctx = perf_lock_task_context(task, &flags);
1651 if (ctx) { 1656 if (ctx) {
1652 unclone_ctx(ctx); 1657 unclone_ctx(ctx);
1653 spin_unlock_irqrestore(&ctx->lock, flags); 1658 raw_spin_unlock_irqrestore(&ctx->lock, flags);
1654 } 1659 }
1655 1660
1656 if (!ctx) { 1661 if (!ctx) {
1657 ctx = kmalloc(sizeof(struct perf_event_context), GFP_KERNEL); 1662 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
1658 err = -ENOMEM; 1663 err = -ENOMEM;
1659 if (!ctx) 1664 if (!ctx)
1660 goto errout; 1665 goto errout;
@@ -1988,7 +1993,7 @@ static int perf_event_period(struct perf_event *event, u64 __user *arg)
1988 if (!value) 1993 if (!value)
1989 return -EINVAL; 1994 return -EINVAL;
1990 1995
1991 spin_lock_irq(&ctx->lock); 1996 raw_spin_lock_irq(&ctx->lock);
1992 if (event->attr.freq) { 1997 if (event->attr.freq) {
1993 if (value > sysctl_perf_event_sample_rate) { 1998 if (value > sysctl_perf_event_sample_rate) {
1994 ret = -EINVAL; 1999 ret = -EINVAL;
@@ -2001,7 +2006,7 @@ static int perf_event_period(struct perf_event *event, u64 __user *arg)
2001 event->hw.sample_period = value; 2006 event->hw.sample_period = value;
2002 } 2007 }
2003unlock: 2008unlock:
2004 spin_unlock_irq(&ctx->lock); 2009 raw_spin_unlock_irq(&ctx->lock);
2005 2010
2006 return ret; 2011 return ret;
2007} 2012}
@@ -3263,6 +3268,9 @@ static void perf_event_task_output(struct perf_event *event,
3263 3268
3264static int perf_event_task_match(struct perf_event *event) 3269static int perf_event_task_match(struct perf_event *event)
3265{ 3270{
3271 if (event->cpu != -1 && event->cpu != smp_processor_id())
3272 return 0;
3273
3266 if (event->attr.comm || event->attr.mmap || event->attr.task) 3274 if (event->attr.comm || event->attr.mmap || event->attr.task)
3267 return 1; 3275 return 1;
3268 3276
@@ -3288,12 +3296,11 @@ static void perf_event_task_event(struct perf_task_event *task_event)
3288 rcu_read_lock(); 3296 rcu_read_lock();
3289 cpuctx = &get_cpu_var(perf_cpu_context); 3297 cpuctx = &get_cpu_var(perf_cpu_context);
3290 perf_event_task_ctx(&cpuctx->ctx, task_event); 3298 perf_event_task_ctx(&cpuctx->ctx, task_event);
3291 put_cpu_var(perf_cpu_context);
3292
3293 if (!ctx) 3299 if (!ctx)
3294 ctx = rcu_dereference(task_event->task->perf_event_ctxp); 3300 ctx = rcu_dereference(task_event->task->perf_event_ctxp);
3295 if (ctx) 3301 if (ctx)
3296 perf_event_task_ctx(ctx, task_event); 3302 perf_event_task_ctx(ctx, task_event);
3303 put_cpu_var(perf_cpu_context);
3297 rcu_read_unlock(); 3304 rcu_read_unlock();
3298} 3305}
3299 3306
@@ -3370,6 +3377,9 @@ static void perf_event_comm_output(struct perf_event *event,
3370 3377
3371static int perf_event_comm_match(struct perf_event *event) 3378static int perf_event_comm_match(struct perf_event *event)
3372{ 3379{
3380 if (event->cpu != -1 && event->cpu != smp_processor_id())
3381 return 0;
3382
3373 if (event->attr.comm) 3383 if (event->attr.comm)
3374 return 1; 3384 return 1;
3375 3385
@@ -3406,15 +3416,10 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event)
3406 rcu_read_lock(); 3416 rcu_read_lock();
3407 cpuctx = &get_cpu_var(perf_cpu_context); 3417 cpuctx = &get_cpu_var(perf_cpu_context);
3408 perf_event_comm_ctx(&cpuctx->ctx, comm_event); 3418 perf_event_comm_ctx(&cpuctx->ctx, comm_event);
3409 put_cpu_var(perf_cpu_context);
3410
3411 /*
3412 * doesn't really matter which of the child contexts the
3413 * events ends up in.
3414 */
3415 ctx = rcu_dereference(current->perf_event_ctxp); 3419 ctx = rcu_dereference(current->perf_event_ctxp);
3416 if (ctx) 3420 if (ctx)
3417 perf_event_comm_ctx(ctx, comm_event); 3421 perf_event_comm_ctx(ctx, comm_event);
3422 put_cpu_var(perf_cpu_context);
3418 rcu_read_unlock(); 3423 rcu_read_unlock();
3419} 3424}
3420 3425
@@ -3489,6 +3494,9 @@ static void perf_event_mmap_output(struct perf_event *event,
3489static int perf_event_mmap_match(struct perf_event *event, 3494static int perf_event_mmap_match(struct perf_event *event,
3490 struct perf_mmap_event *mmap_event) 3495 struct perf_mmap_event *mmap_event)
3491{ 3496{
3497 if (event->cpu != -1 && event->cpu != smp_processor_id())
3498 return 0;
3499
3492 if (event->attr.mmap) 3500 if (event->attr.mmap)
3493 return 1; 3501 return 1;
3494 3502
@@ -3562,15 +3570,10 @@ got_name:
3562 rcu_read_lock(); 3570 rcu_read_lock();
3563 cpuctx = &get_cpu_var(perf_cpu_context); 3571 cpuctx = &get_cpu_var(perf_cpu_context);
3564 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event); 3572 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event);
3565 put_cpu_var(perf_cpu_context);
3566
3567 /*
3568 * doesn't really matter which of the child contexts the
3569 * events ends up in.
3570 */
3571 ctx = rcu_dereference(current->perf_event_ctxp); 3573 ctx = rcu_dereference(current->perf_event_ctxp);
3572 if (ctx) 3574 if (ctx)
3573 perf_event_mmap_ctx(ctx, mmap_event); 3575 perf_event_mmap_ctx(ctx, mmap_event);
3576 put_cpu_var(perf_cpu_context);
3574 rcu_read_unlock(); 3577 rcu_read_unlock();
3575 3578
3576 kfree(buf); 3579 kfree(buf);
@@ -3861,6 +3864,9 @@ static int perf_swevent_match(struct perf_event *event,
3861 struct perf_sample_data *data, 3864 struct perf_sample_data *data,
3862 struct pt_regs *regs) 3865 struct pt_regs *regs)
3863{ 3866{
3867 if (event->cpu != -1 && event->cpu != smp_processor_id())
3868 return 0;
3869
3864 if (!perf_swevent_is_counting(event)) 3870 if (!perf_swevent_is_counting(event))
3865 return 0; 3871 return 0;
3866 3872
@@ -4011,6 +4017,7 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
4011 event->pmu->read(event); 4017 event->pmu->read(event);
4012 4018
4013 data.addr = 0; 4019 data.addr = 0;
4020 data.raw = NULL;
4014 data.period = event->hw.last_period; 4021 data.period = event->hw.last_period;
4015 regs = get_irq_regs(); 4022 regs = get_irq_regs();
4016 /* 4023 /*
@@ -4080,8 +4087,7 @@ static void cpu_clock_perf_event_update(struct perf_event *event)
4080 u64 now; 4087 u64 now;
4081 4088
4082 now = cpu_clock(cpu); 4089 now = cpu_clock(cpu);
4083 prev = atomic64_read(&event->hw.prev_count); 4090 prev = atomic64_xchg(&event->hw.prev_count, now);
4084 atomic64_set(&event->hw.prev_count, now);
4085 atomic64_add(now - prev, &event->count); 4091 atomic64_add(now - prev, &event->count);
4086} 4092}
4087 4093
@@ -4286,15 +4292,8 @@ static void bp_perf_event_destroy(struct perf_event *event)
4286static const struct pmu *bp_perf_event_init(struct perf_event *bp) 4292static const struct pmu *bp_perf_event_init(struct perf_event *bp)
4287{ 4293{
4288 int err; 4294 int err;
4289 /* 4295
4290 * The breakpoint is already filled if we haven't created the counter 4296 err = register_perf_hw_breakpoint(bp);
4291 * through perf syscall
4292 * FIXME: manage to get trigerred to NULL if it comes from syscalls
4293 */
4294 if (!bp->callback)
4295 err = register_perf_hw_breakpoint(bp);
4296 else
4297 err = __register_perf_hw_breakpoint(bp);
4298 if (err) 4297 if (err)
4299 return ERR_PTR(err); 4298 return ERR_PTR(err);
4300 4299
@@ -4308,6 +4307,7 @@ void perf_bp_event(struct perf_event *bp, void *data)
4308 struct perf_sample_data sample; 4307 struct perf_sample_data sample;
4309 struct pt_regs *regs = data; 4308 struct pt_regs *regs = data;
4310 4309
4310 sample.raw = NULL;
4311 sample.addr = bp->attr.bp_addr; 4311 sample.addr = bp->attr.bp_addr;
4312 4312
4313 if (!perf_exclude_event(bp, regs)) 4313 if (!perf_exclude_event(bp, regs))
@@ -4390,7 +4390,7 @@ perf_event_alloc(struct perf_event_attr *attr,
4390 struct perf_event_context *ctx, 4390 struct perf_event_context *ctx,
4391 struct perf_event *group_leader, 4391 struct perf_event *group_leader,
4392 struct perf_event *parent_event, 4392 struct perf_event *parent_event,
4393 perf_callback_t callback, 4393 perf_overflow_handler_t overflow_handler,
4394 gfp_t gfpflags) 4394 gfp_t gfpflags)
4395{ 4395{
4396 const struct pmu *pmu; 4396 const struct pmu *pmu;
@@ -4433,10 +4433,10 @@ perf_event_alloc(struct perf_event_attr *attr,
4433 4433
4434 event->state = PERF_EVENT_STATE_INACTIVE; 4434 event->state = PERF_EVENT_STATE_INACTIVE;
4435 4435
4436 if (!callback && parent_event) 4436 if (!overflow_handler && parent_event)
4437 callback = parent_event->callback; 4437 overflow_handler = parent_event->overflow_handler;
4438 4438
4439 event->callback = callback; 4439 event->overflow_handler = overflow_handler;
4440 4440
4441 if (attr->disabled) 4441 if (attr->disabled)
4442 event->state = PERF_EVENT_STATE_OFF; 4442 event->state = PERF_EVENT_STATE_OFF;
@@ -4571,7 +4571,7 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr,
4571 if (attr->type >= PERF_TYPE_MAX) 4571 if (attr->type >= PERF_TYPE_MAX)
4572 return -EINVAL; 4572 return -EINVAL;
4573 4573
4574 if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3) 4574 if (attr->__reserved_1 || attr->__reserved_2)
4575 return -EINVAL; 4575 return -EINVAL;
4576 4576
4577 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) 4577 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
@@ -4724,7 +4724,7 @@ SYSCALL_DEFINE5(perf_event_open,
4724 if (IS_ERR(event)) 4724 if (IS_ERR(event))
4725 goto err_put_context; 4725 goto err_put_context;
4726 4726
4727 err = anon_inode_getfd("[perf_event]", &perf_fops, event, 0); 4727 err = anon_inode_getfd("[perf_event]", &perf_fops, event, O_RDWR);
4728 if (err < 0) 4728 if (err < 0)
4729 goto err_free_put_context; 4729 goto err_free_put_context;
4730 4730
@@ -4776,7 +4776,8 @@ err_put_context:
4776 */ 4776 */
4777struct perf_event * 4777struct perf_event *
4778perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, 4778perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
4779 pid_t pid, perf_callback_t callback) 4779 pid_t pid,
4780 perf_overflow_handler_t overflow_handler)
4780{ 4781{
4781 struct perf_event *event; 4782 struct perf_event *event;
4782 struct perf_event_context *ctx; 4783 struct perf_event_context *ctx;
@@ -4793,7 +4794,7 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
4793 } 4794 }
4794 4795
4795 event = perf_event_alloc(attr, cpu, ctx, NULL, 4796 event = perf_event_alloc(attr, cpu, ctx, NULL,
4796 NULL, callback, GFP_KERNEL); 4797 NULL, overflow_handler, GFP_KERNEL);
4797 if (IS_ERR(event)) { 4798 if (IS_ERR(event)) {
4798 err = PTR_ERR(event); 4799 err = PTR_ERR(event);
4799 goto err_put_context; 4800 goto err_put_context;
@@ -4998,7 +4999,7 @@ void perf_event_exit_task(struct task_struct *child)
4998 * reading child->perf_event_ctxp, we wait until it has 4999 * reading child->perf_event_ctxp, we wait until it has
4999 * incremented the context's refcount before we do put_ctx below. 5000 * incremented the context's refcount before we do put_ctx below.
5000 */ 5001 */
5001 spin_lock(&child_ctx->lock); 5002 raw_spin_lock(&child_ctx->lock);
5002 child->perf_event_ctxp = NULL; 5003 child->perf_event_ctxp = NULL;
5003 /* 5004 /*
5004 * If this context is a clone; unclone it so it can't get 5005 * If this context is a clone; unclone it so it can't get
@@ -5007,7 +5008,7 @@ void perf_event_exit_task(struct task_struct *child)
5007 */ 5008 */
5008 unclone_ctx(child_ctx); 5009 unclone_ctx(child_ctx);
5009 update_context_time(child_ctx); 5010 update_context_time(child_ctx);
5010 spin_unlock_irqrestore(&child_ctx->lock, flags); 5011 raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
5011 5012
5012 /* 5013 /*
5013 * Report the task dead after unscheduling the events so that we 5014 * Report the task dead after unscheduling the events so that we
@@ -5090,7 +5091,7 @@ again:
5090 */ 5091 */
5091int perf_event_init_task(struct task_struct *child) 5092int perf_event_init_task(struct task_struct *child)
5092{ 5093{
5093 struct perf_event_context *child_ctx, *parent_ctx; 5094 struct perf_event_context *child_ctx = NULL, *parent_ctx;
5094 struct perf_event_context *cloned_ctx; 5095 struct perf_event_context *cloned_ctx;
5095 struct perf_event *event; 5096 struct perf_event *event;
5096 struct task_struct *parent = current; 5097 struct task_struct *parent = current;
@@ -5106,20 +5107,6 @@ int perf_event_init_task(struct task_struct *child)
5106 return 0; 5107 return 0;
5107 5108
5108 /* 5109 /*
5109 * This is executed from the parent task context, so inherit
5110 * events that have been marked for cloning.
5111 * First allocate and initialize a context for the child.
5112 */
5113
5114 child_ctx = kmalloc(sizeof(struct perf_event_context), GFP_KERNEL);
5115 if (!child_ctx)
5116 return -ENOMEM;
5117
5118 __perf_event_init_context(child_ctx, child);
5119 child->perf_event_ctxp = child_ctx;
5120 get_task_struct(child);
5121
5122 /*
5123 * If the parent's context is a clone, pin it so it won't get 5110 * If the parent's context is a clone, pin it so it won't get
5124 * swapped under us. 5111 * swapped under us.
5125 */ 5112 */
@@ -5149,6 +5136,26 @@ int perf_event_init_task(struct task_struct *child)
5149 continue; 5136 continue;
5150 } 5137 }
5151 5138
5139 if (!child->perf_event_ctxp) {
5140 /*
5141 * This is executed from the parent task context, so
5142 * inherit events that have been marked for cloning.
5143 * First allocate and initialize a context for the
5144 * child.
5145 */
5146
5147 child_ctx = kzalloc(sizeof(struct perf_event_context),
5148 GFP_KERNEL);
5149 if (!child_ctx) {
5150 ret = -ENOMEM;
5151 break;
5152 }
5153
5154 __perf_event_init_context(child_ctx, child);
5155 child->perf_event_ctxp = child_ctx;
5156 get_task_struct(child);
5157 }
5158
5152 ret = inherit_group(event, parent, parent_ctx, 5159 ret = inherit_group(event, parent, parent_ctx,
5153 child, child_ctx); 5160 child, child_ctx);
5154 if (ret) { 5161 if (ret) {
@@ -5157,7 +5164,7 @@ int perf_event_init_task(struct task_struct *child)
5157 } 5164 }
5158 } 5165 }
5159 5166
5160 if (inherited_all) { 5167 if (child_ctx && inherited_all) {
5161 /* 5168 /*
5162 * Mark the child context as a clone of the parent 5169 * Mark the child context as a clone of the parent
5163 * context, or of whatever the parent is a clone of. 5170 * context, or of whatever the parent is a clone of.
@@ -5291,11 +5298,11 @@ perf_set_reserve_percpu(struct sysdev_class *class,
5291 perf_reserved_percpu = val; 5298 perf_reserved_percpu = val;
5292 for_each_online_cpu(cpu) { 5299 for_each_online_cpu(cpu) {
5293 cpuctx = &per_cpu(perf_cpu_context, cpu); 5300 cpuctx = &per_cpu(perf_cpu_context, cpu);
5294 spin_lock_irq(&cpuctx->ctx.lock); 5301 raw_spin_lock_irq(&cpuctx->ctx.lock);
5295 mpt = min(perf_max_events - cpuctx->ctx.nr_events, 5302 mpt = min(perf_max_events - cpuctx->ctx.nr_events,
5296 perf_max_events - perf_reserved_percpu); 5303 perf_max_events - perf_reserved_percpu);
5297 cpuctx->max_pertask = mpt; 5304 cpuctx->max_pertask = mpt;
5298 spin_unlock_irq(&cpuctx->ctx.lock); 5305 raw_spin_unlock_irq(&cpuctx->ctx.lock);
5299 } 5306 }
5300 spin_unlock(&perf_resource_lock); 5307 spin_unlock(&perf_resource_lock);
5301 5308