aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/Makefile1
-rw-r--r--kernel/exit.c13
-rw-r--r--kernel/fork.c1
-rw-r--r--kernel/mutex.c2
-rw-r--r--kernel/perf_counter.c3302
-rw-r--r--kernel/sched.c44
-rw-r--r--kernel/sys.c7
-rw-r--r--kernel/sys_ni.c3
-rw-r--r--kernel/sysctl.c11
-rw-r--r--kernel/timer.c3
10 files changed, 3379 insertions, 8 deletions
diff --git a/kernel/Makefile b/kernel/Makefile
index 42423665660a..e914ca992d70 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -95,6 +95,7 @@ obj-$(CONFIG_FUNCTION_TRACER) += trace/
95obj-$(CONFIG_TRACING) += trace/ 95obj-$(CONFIG_TRACING) += trace/
96obj-$(CONFIG_SMP) += sched_cpupri.o 96obj-$(CONFIG_SMP) += sched_cpupri.o
97obj-$(CONFIG_SLOW_WORK) += slow-work.o 97obj-$(CONFIG_SLOW_WORK) += slow-work.o
98obj-$(CONFIG_PERF_COUNTERS) += perf_counter.o
98 99
99ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) 100ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y)
100# According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is 101# According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is
diff --git a/kernel/exit.c b/kernel/exit.c
index abf9cf3b95c6..4741376c8dec 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -158,6 +158,9 @@ static void delayed_put_task_struct(struct rcu_head *rhp)
158{ 158{
159 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu); 159 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
160 160
161#ifdef CONFIG_PERF_COUNTERS
162 WARN_ON_ONCE(!list_empty(&tsk->perf_counter_ctx.counter_list));
163#endif
161 trace_sched_process_free(tsk); 164 trace_sched_process_free(tsk);
162 put_task_struct(tsk); 165 put_task_struct(tsk);
163} 166}
@@ -981,10 +984,6 @@ NORET_TYPE void do_exit(long code)
981 tsk->mempolicy = NULL; 984 tsk->mempolicy = NULL;
982#endif 985#endif
983#ifdef CONFIG_FUTEX 986#ifdef CONFIG_FUTEX
984 /*
985 * This must happen late, after the PID is not
986 * hashed anymore:
987 */
988 if (unlikely(!list_empty(&tsk->pi_state_list))) 987 if (unlikely(!list_empty(&tsk->pi_state_list)))
989 exit_pi_state_list(tsk); 988 exit_pi_state_list(tsk);
990 if (unlikely(current->pi_state_cache)) 989 if (unlikely(current->pi_state_cache))
@@ -1251,6 +1250,12 @@ static int wait_task_zombie(struct task_struct *p, int options,
1251 */ 1250 */
1252 read_unlock(&tasklist_lock); 1251 read_unlock(&tasklist_lock);
1253 1252
1253 /*
1254 * Flush inherited counters to the parent - before the parent
1255 * gets woken up by child-exit notifications.
1256 */
1257 perf_counter_exit_task(p);
1258
1254 retval = ru ? getrusage(p, RUSAGE_BOTH, ru) : 0; 1259 retval = ru ? getrusage(p, RUSAGE_BOTH, ru) : 0;
1255 status = (p->signal->flags & SIGNAL_GROUP_EXIT) 1260 status = (p->signal->flags & SIGNAL_GROUP_EXIT)
1256 ? p->signal->group_exit_code : p->exit_code; 1261 ? p->signal->group_exit_code : p->exit_code;
diff --git a/kernel/fork.c b/kernel/fork.c
index b9e2edd00726..d32fef4d38e5 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -983,6 +983,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
983 goto fork_out; 983 goto fork_out;
984 984
985 rt_mutex_init_task(p); 985 rt_mutex_init_task(p);
986 perf_counter_init_task(p);
986 987
987#ifdef CONFIG_PROVE_LOCKING 988#ifdef CONFIG_PROVE_LOCKING
988 DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled); 989 DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
diff --git a/kernel/mutex.c b/kernel/mutex.c
index 507cf2b5e9f1..f415e80a9119 100644
--- a/kernel/mutex.c
+++ b/kernel/mutex.c
@@ -89,7 +89,7 @@ __mutex_lock_slowpath(atomic_t *lock_count);
89 * 89 *
90 * This function is similar to (but not equivalent to) down(). 90 * This function is similar to (but not equivalent to) down().
91 */ 91 */
92void inline __sched mutex_lock(struct mutex *lock) 92void __sched mutex_lock(struct mutex *lock)
93{ 93{
94 might_sleep(); 94 might_sleep();
95 /* 95 /*
diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c
new file mode 100644
index 000000000000..09396098dd0d
--- /dev/null
+++ b/kernel/perf_counter.c
@@ -0,0 +1,3302 @@
1/*
2 * Performance counter core code
3 *
4 * Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
6 *
7 *
8 * For licensing details see kernel-base/COPYING
9 */
10
11#include <linux/fs.h>
12#include <linux/mm.h>
13#include <linux/cpu.h>
14#include <linux/smp.h>
15#include <linux/file.h>
16#include <linux/poll.h>
17#include <linux/sysfs.h>
18#include <linux/ptrace.h>
19#include <linux/percpu.h>
20#include <linux/vmstat.h>
21#include <linux/hardirq.h>
22#include <linux/rculist.h>
23#include <linux/uaccess.h>
24#include <linux/syscalls.h>
25#include <linux/anon_inodes.h>
26#include <linux/kernel_stat.h>
27#include <linux/perf_counter.h>
28#include <linux/dcache.h>
29
30#include <asm/irq_regs.h>
31
32/*
33 * Each CPU has a list of per CPU counters:
34 */
35DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
36
37int perf_max_counters __read_mostly = 1;
38static int perf_reserved_percpu __read_mostly;
39static int perf_overcommit __read_mostly = 1;
40
41static atomic_t nr_mmap_tracking __read_mostly;
42static atomic_t nr_munmap_tracking __read_mostly;
43static atomic_t nr_comm_tracking __read_mostly;
44
45int sysctl_perf_counter_priv __read_mostly; /* do we need to be privileged */
46
47/*
48 * Mutex for (sysadmin-configurable) counter reservations:
49 */
50static DEFINE_MUTEX(perf_resource_mutex);
51
52/*
53 * Architecture provided APIs - weak aliases:
54 */
55extern __weak const struct hw_perf_counter_ops *
56hw_perf_counter_init(struct perf_counter *counter)
57{
58 return NULL;
59}
60
61u64 __weak hw_perf_save_disable(void) { return 0; }
62void __weak hw_perf_restore(u64 ctrl) { barrier(); }
63void __weak hw_perf_counter_setup(int cpu) { barrier(); }
64int __weak hw_perf_group_sched_in(struct perf_counter *group_leader,
65 struct perf_cpu_context *cpuctx,
66 struct perf_counter_context *ctx, int cpu)
67{
68 return 0;
69}
70
71void __weak perf_counter_print_debug(void) { }
72
73static void
74list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
75{
76 struct perf_counter *group_leader = counter->group_leader;
77
78 /*
79 * Depending on whether it is a standalone or sibling counter,
80 * add it straight to the context's counter list, or to the group
81 * leader's sibling list:
82 */
83 if (counter->group_leader == counter)
84 list_add_tail(&counter->list_entry, &ctx->counter_list);
85 else {
86 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
87 group_leader->nr_siblings++;
88 }
89
90 list_add_rcu(&counter->event_entry, &ctx->event_list);
91}
92
93static void
94list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
95{
96 struct perf_counter *sibling, *tmp;
97
98 list_del_init(&counter->list_entry);
99 list_del_rcu(&counter->event_entry);
100
101 if (counter->group_leader != counter)
102 counter->group_leader->nr_siblings--;
103
104 /*
105 * If this was a group counter with sibling counters then
106 * upgrade the siblings to singleton counters by adding them
107 * to the context list directly:
108 */
109 list_for_each_entry_safe(sibling, tmp,
110 &counter->sibling_list, list_entry) {
111
112 list_move_tail(&sibling->list_entry, &ctx->counter_list);
113 sibling->group_leader = sibling;
114 }
115}
116
117static void
118counter_sched_out(struct perf_counter *counter,
119 struct perf_cpu_context *cpuctx,
120 struct perf_counter_context *ctx)
121{
122 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
123 return;
124
125 counter->state = PERF_COUNTER_STATE_INACTIVE;
126 counter->tstamp_stopped = ctx->time;
127 counter->hw_ops->disable(counter);
128 counter->oncpu = -1;
129
130 if (!is_software_counter(counter))
131 cpuctx->active_oncpu--;
132 ctx->nr_active--;
133 if (counter->hw_event.exclusive || !cpuctx->active_oncpu)
134 cpuctx->exclusive = 0;
135}
136
137static void
138group_sched_out(struct perf_counter *group_counter,
139 struct perf_cpu_context *cpuctx,
140 struct perf_counter_context *ctx)
141{
142 struct perf_counter *counter;
143
144 if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
145 return;
146
147 counter_sched_out(group_counter, cpuctx, ctx);
148
149 /*
150 * Schedule out siblings (if any):
151 */
152 list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
153 counter_sched_out(counter, cpuctx, ctx);
154
155 if (group_counter->hw_event.exclusive)
156 cpuctx->exclusive = 0;
157}
158
159/*
160 * Cross CPU call to remove a performance counter
161 *
162 * We disable the counter on the hardware level first. After that we
163 * remove it from the context list.
164 */
165static void __perf_counter_remove_from_context(void *info)
166{
167 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
168 struct perf_counter *counter = info;
169 struct perf_counter_context *ctx = counter->ctx;
170 unsigned long flags;
171 u64 perf_flags;
172
173 /*
174 * If this is a task context, we need to check whether it is
175 * the current task context of this cpu. If not it has been
176 * scheduled out before the smp call arrived.
177 */
178 if (ctx->task && cpuctx->task_ctx != ctx)
179 return;
180
181 spin_lock_irqsave(&ctx->lock, flags);
182
183 counter_sched_out(counter, cpuctx, ctx);
184
185 counter->task = NULL;
186 ctx->nr_counters--;
187
188 /*
189 * Protect the list operation against NMI by disabling the
190 * counters on a global level. NOP for non NMI based counters.
191 */
192 perf_flags = hw_perf_save_disable();
193 list_del_counter(counter, ctx);
194 hw_perf_restore(perf_flags);
195
196 if (!ctx->task) {
197 /*
198 * Allow more per task counters with respect to the
199 * reservation:
200 */
201 cpuctx->max_pertask =
202 min(perf_max_counters - ctx->nr_counters,
203 perf_max_counters - perf_reserved_percpu);
204 }
205
206 spin_unlock_irqrestore(&ctx->lock, flags);
207}
208
209
210/*
211 * Remove the counter from a task's (or a CPU's) list of counters.
212 *
213 * Must be called with counter->mutex and ctx->mutex held.
214 *
215 * CPU counters are removed with a smp call. For task counters we only
216 * call when the task is on a CPU.
217 */
218static void perf_counter_remove_from_context(struct perf_counter *counter)
219{
220 struct perf_counter_context *ctx = counter->ctx;
221 struct task_struct *task = ctx->task;
222
223 if (!task) {
224 /*
225 * Per cpu counters are removed via an smp call and
226 * the removal is always sucessful.
227 */
228 smp_call_function_single(counter->cpu,
229 __perf_counter_remove_from_context,
230 counter, 1);
231 return;
232 }
233
234retry:
235 task_oncpu_function_call(task, __perf_counter_remove_from_context,
236 counter);
237
238 spin_lock_irq(&ctx->lock);
239 /*
240 * If the context is active we need to retry the smp call.
241 */
242 if (ctx->nr_active && !list_empty(&counter->list_entry)) {
243 spin_unlock_irq(&ctx->lock);
244 goto retry;
245 }
246
247 /*
248 * The lock prevents that this context is scheduled in so we
249 * can remove the counter safely, if the call above did not
250 * succeed.
251 */
252 if (!list_empty(&counter->list_entry)) {
253 ctx->nr_counters--;
254 list_del_counter(counter, ctx);
255 counter->task = NULL;
256 }
257 spin_unlock_irq(&ctx->lock);
258}
259
260static inline u64 perf_clock(void)
261{
262 return cpu_clock(smp_processor_id());
263}
264
265/*
266 * Update the record of the current time in a context.
267 */
268static void update_context_time(struct perf_counter_context *ctx)
269{
270 u64 now = perf_clock();
271
272 ctx->time += now - ctx->timestamp;
273 ctx->timestamp = now;
274}
275
276/*
277 * Update the total_time_enabled and total_time_running fields for a counter.
278 */
279static void update_counter_times(struct perf_counter *counter)
280{
281 struct perf_counter_context *ctx = counter->ctx;
282 u64 run_end;
283
284 if (counter->state < PERF_COUNTER_STATE_INACTIVE)
285 return;
286
287 counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
288
289 if (counter->state == PERF_COUNTER_STATE_INACTIVE)
290 run_end = counter->tstamp_stopped;
291 else
292 run_end = ctx->time;
293
294 counter->total_time_running = run_end - counter->tstamp_running;
295}
296
297/*
298 * Update total_time_enabled and total_time_running for all counters in a group.
299 */
300static void update_group_times(struct perf_counter *leader)
301{
302 struct perf_counter *counter;
303
304 update_counter_times(leader);
305 list_for_each_entry(counter, &leader->sibling_list, list_entry)
306 update_counter_times(counter);
307}
308
309/*
310 * Cross CPU call to disable a performance counter
311 */
312static void __perf_counter_disable(void *info)
313{
314 struct perf_counter *counter = info;
315 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
316 struct perf_counter_context *ctx = counter->ctx;
317 unsigned long flags;
318
319 /*
320 * If this is a per-task counter, need to check whether this
321 * counter's task is the current task on this cpu.
322 */
323 if (ctx->task && cpuctx->task_ctx != ctx)
324 return;
325
326 spin_lock_irqsave(&ctx->lock, flags);
327
328 /*
329 * If the counter is on, turn it off.
330 * If it is in error state, leave it in error state.
331 */
332 if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
333 update_context_time(ctx);
334 update_counter_times(counter);
335 if (counter == counter->group_leader)
336 group_sched_out(counter, cpuctx, ctx);
337 else
338 counter_sched_out(counter, cpuctx, ctx);
339 counter->state = PERF_COUNTER_STATE_OFF;
340 }
341
342 spin_unlock_irqrestore(&ctx->lock, flags);
343}
344
345/*
346 * Disable a counter.
347 */
348static void perf_counter_disable(struct perf_counter *counter)
349{
350 struct perf_counter_context *ctx = counter->ctx;
351 struct task_struct *task = ctx->task;
352
353 if (!task) {
354 /*
355 * Disable the counter on the cpu that it's on
356 */
357 smp_call_function_single(counter->cpu, __perf_counter_disable,
358 counter, 1);
359 return;
360 }
361
362 retry:
363 task_oncpu_function_call(task, __perf_counter_disable, counter);
364
365 spin_lock_irq(&ctx->lock);
366 /*
367 * If the counter is still active, we need to retry the cross-call.
368 */
369 if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
370 spin_unlock_irq(&ctx->lock);
371 goto retry;
372 }
373
374 /*
375 * Since we have the lock this context can't be scheduled
376 * in, so we can change the state safely.
377 */
378 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
379 update_counter_times(counter);
380 counter->state = PERF_COUNTER_STATE_OFF;
381 }
382
383 spin_unlock_irq(&ctx->lock);
384}
385
386/*
387 * Disable a counter and all its children.
388 */
389static void perf_counter_disable_family(struct perf_counter *counter)
390{
391 struct perf_counter *child;
392
393 perf_counter_disable(counter);
394
395 /*
396 * Lock the mutex to protect the list of children
397 */
398 mutex_lock(&counter->mutex);
399 list_for_each_entry(child, &counter->child_list, child_list)
400 perf_counter_disable(child);
401 mutex_unlock(&counter->mutex);
402}
403
404static int
405counter_sched_in(struct perf_counter *counter,
406 struct perf_cpu_context *cpuctx,
407 struct perf_counter_context *ctx,
408 int cpu)
409{
410 if (counter->state <= PERF_COUNTER_STATE_OFF)
411 return 0;
412
413 counter->state = PERF_COUNTER_STATE_ACTIVE;
414 counter->oncpu = cpu; /* TODO: put 'cpu' into cpuctx->cpu */
415 /*
416 * The new state must be visible before we turn it on in the hardware:
417 */
418 smp_wmb();
419
420 if (counter->hw_ops->enable(counter)) {
421 counter->state = PERF_COUNTER_STATE_INACTIVE;
422 counter->oncpu = -1;
423 return -EAGAIN;
424 }
425
426 counter->tstamp_running += ctx->time - counter->tstamp_stopped;
427
428 if (!is_software_counter(counter))
429 cpuctx->active_oncpu++;
430 ctx->nr_active++;
431
432 if (counter->hw_event.exclusive)
433 cpuctx->exclusive = 1;
434
435 return 0;
436}
437
438/*
439 * Return 1 for a group consisting entirely of software counters,
440 * 0 if the group contains any hardware counters.
441 */
442static int is_software_only_group(struct perf_counter *leader)
443{
444 struct perf_counter *counter;
445
446 if (!is_software_counter(leader))
447 return 0;
448
449 list_for_each_entry(counter, &leader->sibling_list, list_entry)
450 if (!is_software_counter(counter))
451 return 0;
452
453 return 1;
454}
455
456/*
457 * Work out whether we can put this counter group on the CPU now.
458 */
459static int group_can_go_on(struct perf_counter *counter,
460 struct perf_cpu_context *cpuctx,
461 int can_add_hw)
462{
463 /*
464 * Groups consisting entirely of software counters can always go on.
465 */
466 if (is_software_only_group(counter))
467 return 1;
468 /*
469 * If an exclusive group is already on, no other hardware
470 * counters can go on.
471 */
472 if (cpuctx->exclusive)
473 return 0;
474 /*
475 * If this group is exclusive and there are already
476 * counters on the CPU, it can't go on.
477 */
478 if (counter->hw_event.exclusive && cpuctx->active_oncpu)
479 return 0;
480 /*
481 * Otherwise, try to add it if all previous groups were able
482 * to go on.
483 */
484 return can_add_hw;
485}
486
487static void add_counter_to_ctx(struct perf_counter *counter,
488 struct perf_counter_context *ctx)
489{
490 list_add_counter(counter, ctx);
491 ctx->nr_counters++;
492 counter->prev_state = PERF_COUNTER_STATE_OFF;
493 counter->tstamp_enabled = ctx->time;
494 counter->tstamp_running = ctx->time;
495 counter->tstamp_stopped = ctx->time;
496}
497
498/*
499 * Cross CPU call to install and enable a performance counter
500 */
501static void __perf_install_in_context(void *info)
502{
503 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
504 struct perf_counter *counter = info;
505 struct perf_counter_context *ctx = counter->ctx;
506 struct perf_counter *leader = counter->group_leader;
507 int cpu = smp_processor_id();
508 unsigned long flags;
509 u64 perf_flags;
510 int err;
511
512 /*
513 * If this is a task context, we need to check whether it is
514 * the current task context of this cpu. If not it has been
515 * scheduled out before the smp call arrived.
516 */
517 if (ctx->task && cpuctx->task_ctx != ctx)
518 return;
519
520 spin_lock_irqsave(&ctx->lock, flags);
521 update_context_time(ctx);
522
523 /*
524 * Protect the list operation against NMI by disabling the
525 * counters on a global level. NOP for non NMI based counters.
526 */
527 perf_flags = hw_perf_save_disable();
528
529 add_counter_to_ctx(counter, ctx);
530
531 /*
532 * Don't put the counter on if it is disabled or if
533 * it is in a group and the group isn't on.
534 */
535 if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
536 (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
537 goto unlock;
538
539 /*
540 * An exclusive counter can't go on if there are already active
541 * hardware counters, and no hardware counter can go on if there
542 * is already an exclusive counter on.
543 */
544 if (!group_can_go_on(counter, cpuctx, 1))
545 err = -EEXIST;
546 else
547 err = counter_sched_in(counter, cpuctx, ctx, cpu);
548
549 if (err) {
550 /*
551 * This counter couldn't go on. If it is in a group
552 * then we have to pull the whole group off.
553 * If the counter group is pinned then put it in error state.
554 */
555 if (leader != counter)
556 group_sched_out(leader, cpuctx, ctx);
557 if (leader->hw_event.pinned) {
558 update_group_times(leader);
559 leader->state = PERF_COUNTER_STATE_ERROR;
560 }
561 }
562
563 if (!err && !ctx->task && cpuctx->max_pertask)
564 cpuctx->max_pertask--;
565
566 unlock:
567 hw_perf_restore(perf_flags);
568
569 spin_unlock_irqrestore(&ctx->lock, flags);
570}
571
572/*
573 * Attach a performance counter to a context
574 *
575 * First we add the counter to the list with the hardware enable bit
576 * in counter->hw_config cleared.
577 *
578 * If the counter is attached to a task which is on a CPU we use a smp
579 * call to enable it in the task context. The task might have been
580 * scheduled away, but we check this in the smp call again.
581 *
582 * Must be called with ctx->mutex held.
583 */
584static void
585perf_install_in_context(struct perf_counter_context *ctx,
586 struct perf_counter *counter,
587 int cpu)
588{
589 struct task_struct *task = ctx->task;
590
591 if (!task) {
592 /*
593 * Per cpu counters are installed via an smp call and
594 * the install is always sucessful.
595 */
596 smp_call_function_single(cpu, __perf_install_in_context,
597 counter, 1);
598 return;
599 }
600
601 counter->task = task;
602retry:
603 task_oncpu_function_call(task, __perf_install_in_context,
604 counter);
605
606 spin_lock_irq(&ctx->lock);
607 /*
608 * we need to retry the smp call.
609 */
610 if (ctx->is_active && list_empty(&counter->list_entry)) {
611 spin_unlock_irq(&ctx->lock);
612 goto retry;
613 }
614
615 /*
616 * The lock prevents that this context is scheduled in so we
617 * can add the counter safely, if it the call above did not
618 * succeed.
619 */
620 if (list_empty(&counter->list_entry))
621 add_counter_to_ctx(counter, ctx);
622 spin_unlock_irq(&ctx->lock);
623}
624
625/*
626 * Cross CPU call to enable a performance counter
627 */
628static void __perf_counter_enable(void *info)
629{
630 struct perf_counter *counter = info;
631 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
632 struct perf_counter_context *ctx = counter->ctx;
633 struct perf_counter *leader = counter->group_leader;
634 unsigned long flags;
635 int err;
636
637 /*
638 * If this is a per-task counter, need to check whether this
639 * counter's task is the current task on this cpu.
640 */
641 if (ctx->task && cpuctx->task_ctx != ctx)
642 return;
643
644 spin_lock_irqsave(&ctx->lock, flags);
645 update_context_time(ctx);
646
647 counter->prev_state = counter->state;
648 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
649 goto unlock;
650 counter->state = PERF_COUNTER_STATE_INACTIVE;
651 counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
652
653 /*
654 * If the counter is in a group and isn't the group leader,
655 * then don't put it on unless the group is on.
656 */
657 if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
658 goto unlock;
659
660 if (!group_can_go_on(counter, cpuctx, 1))
661 err = -EEXIST;
662 else
663 err = counter_sched_in(counter, cpuctx, ctx,
664 smp_processor_id());
665
666 if (err) {
667 /*
668 * If this counter can't go on and it's part of a
669 * group, then the whole group has to come off.
670 */
671 if (leader != counter)
672 group_sched_out(leader, cpuctx, ctx);
673 if (leader->hw_event.pinned) {
674 update_group_times(leader);
675 leader->state = PERF_COUNTER_STATE_ERROR;
676 }
677 }
678
679 unlock:
680 spin_unlock_irqrestore(&ctx->lock, flags);
681}
682
683/*
684 * Enable a counter.
685 */
686static void perf_counter_enable(struct perf_counter *counter)
687{
688 struct perf_counter_context *ctx = counter->ctx;
689 struct task_struct *task = ctx->task;
690
691 if (!task) {
692 /*
693 * Enable the counter on the cpu that it's on
694 */
695 smp_call_function_single(counter->cpu, __perf_counter_enable,
696 counter, 1);
697 return;
698 }
699
700 spin_lock_irq(&ctx->lock);
701 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
702 goto out;
703
704 /*
705 * If the counter is in error state, clear that first.
706 * That way, if we see the counter in error state below, we
707 * know that it has gone back into error state, as distinct
708 * from the task having been scheduled away before the
709 * cross-call arrived.
710 */
711 if (counter->state == PERF_COUNTER_STATE_ERROR)
712 counter->state = PERF_COUNTER_STATE_OFF;
713
714 retry:
715 spin_unlock_irq(&ctx->lock);
716 task_oncpu_function_call(task, __perf_counter_enable, counter);
717
718 spin_lock_irq(&ctx->lock);
719
720 /*
721 * If the context is active and the counter is still off,
722 * we need to retry the cross-call.
723 */
724 if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
725 goto retry;
726
727 /*
728 * Since we have the lock this context can't be scheduled
729 * in, so we can change the state safely.
730 */
731 if (counter->state == PERF_COUNTER_STATE_OFF) {
732 counter->state = PERF_COUNTER_STATE_INACTIVE;
733 counter->tstamp_enabled =
734 ctx->time - counter->total_time_enabled;
735 }
736 out:
737 spin_unlock_irq(&ctx->lock);
738}
739
740static void perf_counter_refresh(struct perf_counter *counter, int refresh)
741{
742 atomic_add(refresh, &counter->event_limit);
743 perf_counter_enable(counter);
744}
745
746/*
747 * Enable a counter and all its children.
748 */
749static void perf_counter_enable_family(struct perf_counter *counter)
750{
751 struct perf_counter *child;
752
753 perf_counter_enable(counter);
754
755 /*
756 * Lock the mutex to protect the list of children
757 */
758 mutex_lock(&counter->mutex);
759 list_for_each_entry(child, &counter->child_list, child_list)
760 perf_counter_enable(child);
761 mutex_unlock(&counter->mutex);
762}
763
764void __perf_counter_sched_out(struct perf_counter_context *ctx,
765 struct perf_cpu_context *cpuctx)
766{
767 struct perf_counter *counter;
768 u64 flags;
769
770 spin_lock(&ctx->lock);
771 ctx->is_active = 0;
772 if (likely(!ctx->nr_counters))
773 goto out;
774 update_context_time(ctx);
775
776 flags = hw_perf_save_disable();
777 if (ctx->nr_active) {
778 list_for_each_entry(counter, &ctx->counter_list, list_entry)
779 group_sched_out(counter, cpuctx, ctx);
780 }
781 hw_perf_restore(flags);
782 out:
783 spin_unlock(&ctx->lock);
784}
785
786/*
787 * Called from scheduler to remove the counters of the current task,
788 * with interrupts disabled.
789 *
790 * We stop each counter and update the counter value in counter->count.
791 *
792 * This does not protect us against NMI, but disable()
793 * sets the disabled bit in the control field of counter _before_
794 * accessing the counter control register. If a NMI hits, then it will
795 * not restart the counter.
796 */
797void perf_counter_task_sched_out(struct task_struct *task, int cpu)
798{
799 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
800 struct perf_counter_context *ctx = &task->perf_counter_ctx;
801 struct pt_regs *regs;
802
803 if (likely(!cpuctx->task_ctx))
804 return;
805
806 update_context_time(ctx);
807
808 regs = task_pt_regs(task);
809 perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs, 0);
810 __perf_counter_sched_out(ctx, cpuctx);
811
812 cpuctx->task_ctx = NULL;
813}
814
815static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
816{
817 __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
818}
819
820static int
821group_sched_in(struct perf_counter *group_counter,
822 struct perf_cpu_context *cpuctx,
823 struct perf_counter_context *ctx,
824 int cpu)
825{
826 struct perf_counter *counter, *partial_group;
827 int ret;
828
829 if (group_counter->state == PERF_COUNTER_STATE_OFF)
830 return 0;
831
832 ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
833 if (ret)
834 return ret < 0 ? ret : 0;
835
836 group_counter->prev_state = group_counter->state;
837 if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
838 return -EAGAIN;
839
840 /*
841 * Schedule in siblings as one group (if any):
842 */
843 list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
844 counter->prev_state = counter->state;
845 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
846 partial_group = counter;
847 goto group_error;
848 }
849 }
850
851 return 0;
852
853group_error:
854 /*
855 * Groups can be scheduled in as one unit only, so undo any
856 * partial group before returning:
857 */
858 list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
859 if (counter == partial_group)
860 break;
861 counter_sched_out(counter, cpuctx, ctx);
862 }
863 counter_sched_out(group_counter, cpuctx, ctx);
864
865 return -EAGAIN;
866}
867
868static void
869__perf_counter_sched_in(struct perf_counter_context *ctx,
870 struct perf_cpu_context *cpuctx, int cpu)
871{
872 struct perf_counter *counter;
873 u64 flags;
874 int can_add_hw = 1;
875
876 spin_lock(&ctx->lock);
877 ctx->is_active = 1;
878 if (likely(!ctx->nr_counters))
879 goto out;
880
881 ctx->timestamp = perf_clock();
882
883 flags = hw_perf_save_disable();
884
885 /*
886 * First go through the list and put on any pinned groups
887 * in order to give them the best chance of going on.
888 */
889 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
890 if (counter->state <= PERF_COUNTER_STATE_OFF ||
891 !counter->hw_event.pinned)
892 continue;
893 if (counter->cpu != -1 && counter->cpu != cpu)
894 continue;
895
896 if (group_can_go_on(counter, cpuctx, 1))
897 group_sched_in(counter, cpuctx, ctx, cpu);
898
899 /*
900 * If this pinned group hasn't been scheduled,
901 * put it in error state.
902 */
903 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
904 update_group_times(counter);
905 counter->state = PERF_COUNTER_STATE_ERROR;
906 }
907 }
908
909 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
910 /*
911 * Ignore counters in OFF or ERROR state, and
912 * ignore pinned counters since we did them already.
913 */
914 if (counter->state <= PERF_COUNTER_STATE_OFF ||
915 counter->hw_event.pinned)
916 continue;
917
918 /*
919 * Listen to the 'cpu' scheduling filter constraint
920 * of counters:
921 */
922 if (counter->cpu != -1 && counter->cpu != cpu)
923 continue;
924
925 if (group_can_go_on(counter, cpuctx, can_add_hw)) {
926 if (group_sched_in(counter, cpuctx, ctx, cpu))
927 can_add_hw = 0;
928 }
929 }
930 hw_perf_restore(flags);
931 out:
932 spin_unlock(&ctx->lock);
933}
934
935/*
936 * Called from scheduler to add the counters of the current task
937 * with interrupts disabled.
938 *
939 * We restore the counter value and then enable it.
940 *
941 * This does not protect us against NMI, but enable()
942 * sets the enabled bit in the control field of counter _before_
943 * accessing the counter control register. If a NMI hits, then it will
944 * keep the counter running.
945 */
946void perf_counter_task_sched_in(struct task_struct *task, int cpu)
947{
948 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
949 struct perf_counter_context *ctx = &task->perf_counter_ctx;
950
951 __perf_counter_sched_in(ctx, cpuctx, cpu);
952 cpuctx->task_ctx = ctx;
953}
954
955static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
956{
957 struct perf_counter_context *ctx = &cpuctx->ctx;
958
959 __perf_counter_sched_in(ctx, cpuctx, cpu);
960}
961
962int perf_counter_task_disable(void)
963{
964 struct task_struct *curr = current;
965 struct perf_counter_context *ctx = &curr->perf_counter_ctx;
966 struct perf_counter *counter;
967 unsigned long flags;
968 u64 perf_flags;
969 int cpu;
970
971 if (likely(!ctx->nr_counters))
972 return 0;
973
974 local_irq_save(flags);
975 cpu = smp_processor_id();
976
977 perf_counter_task_sched_out(curr, cpu);
978
979 spin_lock(&ctx->lock);
980
981 /*
982 * Disable all the counters:
983 */
984 perf_flags = hw_perf_save_disable();
985
986 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
987 if (counter->state != PERF_COUNTER_STATE_ERROR) {
988 update_group_times(counter);
989 counter->state = PERF_COUNTER_STATE_OFF;
990 }
991 }
992
993 hw_perf_restore(perf_flags);
994
995 spin_unlock_irqrestore(&ctx->lock, flags);
996
997 return 0;
998}
999
1000int perf_counter_task_enable(void)
1001{
1002 struct task_struct *curr = current;
1003 struct perf_counter_context *ctx = &curr->perf_counter_ctx;
1004 struct perf_counter *counter;
1005 unsigned long flags;
1006 u64 perf_flags;
1007 int cpu;
1008
1009 if (likely(!ctx->nr_counters))
1010 return 0;
1011
1012 local_irq_save(flags);
1013 cpu = smp_processor_id();
1014
1015 perf_counter_task_sched_out(curr, cpu);
1016
1017 spin_lock(&ctx->lock);
1018
1019 /*
1020 * Disable all the counters:
1021 */
1022 perf_flags = hw_perf_save_disable();
1023
1024 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1025 if (counter->state > PERF_COUNTER_STATE_OFF)
1026 continue;
1027 counter->state = PERF_COUNTER_STATE_INACTIVE;
1028 counter->tstamp_enabled =
1029 ctx->time - counter->total_time_enabled;
1030 counter->hw_event.disabled = 0;
1031 }
1032 hw_perf_restore(perf_flags);
1033
1034 spin_unlock(&ctx->lock);
1035
1036 perf_counter_task_sched_in(curr, cpu);
1037
1038 local_irq_restore(flags);
1039
1040 return 0;
1041}
1042
1043/*
1044 * Round-robin a context's counters:
1045 */
1046static void rotate_ctx(struct perf_counter_context *ctx)
1047{
1048 struct perf_counter *counter;
1049 u64 perf_flags;
1050
1051 if (!ctx->nr_counters)
1052 return;
1053
1054 spin_lock(&ctx->lock);
1055 /*
1056 * Rotate the first entry last (works just fine for group counters too):
1057 */
1058 perf_flags = hw_perf_save_disable();
1059 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1060 list_move_tail(&counter->list_entry, &ctx->counter_list);
1061 break;
1062 }
1063 hw_perf_restore(perf_flags);
1064
1065 spin_unlock(&ctx->lock);
1066}
1067
1068void perf_counter_task_tick(struct task_struct *curr, int cpu)
1069{
1070 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1071 struct perf_counter_context *ctx = &curr->perf_counter_ctx;
1072 const int rotate_percpu = 0;
1073
1074 if (rotate_percpu)
1075 perf_counter_cpu_sched_out(cpuctx);
1076 perf_counter_task_sched_out(curr, cpu);
1077
1078 if (rotate_percpu)
1079 rotate_ctx(&cpuctx->ctx);
1080 rotate_ctx(ctx);
1081
1082 if (rotate_percpu)
1083 perf_counter_cpu_sched_in(cpuctx, cpu);
1084 perf_counter_task_sched_in(curr, cpu);
1085}
1086
1087/*
1088 * Cross CPU call to read the hardware counter
1089 */
1090static void __read(void *info)
1091{
1092 struct perf_counter *counter = info;
1093 struct perf_counter_context *ctx = counter->ctx;
1094 unsigned long flags;
1095
1096 local_irq_save(flags);
1097 if (ctx->is_active)
1098 update_context_time(ctx);
1099 counter->hw_ops->read(counter);
1100 update_counter_times(counter);
1101 local_irq_restore(flags);
1102}
1103
1104static u64 perf_counter_read(struct perf_counter *counter)
1105{
1106 /*
1107 * If counter is enabled and currently active on a CPU, update the
1108 * value in the counter structure:
1109 */
1110 if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1111 smp_call_function_single(counter->oncpu,
1112 __read, counter, 1);
1113 } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1114 update_counter_times(counter);
1115 }
1116
1117 return atomic64_read(&counter->count);
1118}
1119
1120static void put_context(struct perf_counter_context *ctx)
1121{
1122 if (ctx->task)
1123 put_task_struct(ctx->task);
1124}
1125
1126static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1127{
1128 struct perf_cpu_context *cpuctx;
1129 struct perf_counter_context *ctx;
1130 struct task_struct *task;
1131
1132 /*
1133 * If cpu is not a wildcard then this is a percpu counter:
1134 */
1135 if (cpu != -1) {
1136 /* Must be root to operate on a CPU counter: */
1137 if (sysctl_perf_counter_priv && !capable(CAP_SYS_ADMIN))
1138 return ERR_PTR(-EACCES);
1139
1140 if (cpu < 0 || cpu > num_possible_cpus())
1141 return ERR_PTR(-EINVAL);
1142
1143 /*
1144 * We could be clever and allow to attach a counter to an
1145 * offline CPU and activate it when the CPU comes up, but
1146 * that's for later.
1147 */
1148 if (!cpu_isset(cpu, cpu_online_map))
1149 return ERR_PTR(-ENODEV);
1150
1151 cpuctx = &per_cpu(perf_cpu_context, cpu);
1152 ctx = &cpuctx->ctx;
1153
1154 return ctx;
1155 }
1156
1157 rcu_read_lock();
1158 if (!pid)
1159 task = current;
1160 else
1161 task = find_task_by_vpid(pid);
1162 if (task)
1163 get_task_struct(task);
1164 rcu_read_unlock();
1165
1166 if (!task)
1167 return ERR_PTR(-ESRCH);
1168
1169 ctx = &task->perf_counter_ctx;
1170 ctx->task = task;
1171
1172 /* Reuse ptrace permission checks for now. */
1173 if (!ptrace_may_access(task, PTRACE_MODE_READ)) {
1174 put_context(ctx);
1175 return ERR_PTR(-EACCES);
1176 }
1177
1178 return ctx;
1179}
1180
1181static void free_counter_rcu(struct rcu_head *head)
1182{
1183 struct perf_counter *counter;
1184
1185 counter = container_of(head, struct perf_counter, rcu_head);
1186 kfree(counter);
1187}
1188
1189static void perf_pending_sync(struct perf_counter *counter);
1190
1191static void free_counter(struct perf_counter *counter)
1192{
1193 perf_pending_sync(counter);
1194
1195 if (counter->hw_event.mmap)
1196 atomic_dec(&nr_mmap_tracking);
1197 if (counter->hw_event.munmap)
1198 atomic_dec(&nr_munmap_tracking);
1199 if (counter->hw_event.comm)
1200 atomic_dec(&nr_comm_tracking);
1201
1202 if (counter->destroy)
1203 counter->destroy(counter);
1204
1205 call_rcu(&counter->rcu_head, free_counter_rcu);
1206}
1207
1208/*
1209 * Called when the last reference to the file is gone.
1210 */
1211static int perf_release(struct inode *inode, struct file *file)
1212{
1213 struct perf_counter *counter = file->private_data;
1214 struct perf_counter_context *ctx = counter->ctx;
1215
1216 file->private_data = NULL;
1217
1218 mutex_lock(&ctx->mutex);
1219 mutex_lock(&counter->mutex);
1220
1221 perf_counter_remove_from_context(counter);
1222
1223 mutex_unlock(&counter->mutex);
1224 mutex_unlock(&ctx->mutex);
1225
1226 free_counter(counter);
1227 put_context(ctx);
1228
1229 return 0;
1230}
1231
1232/*
1233 * Read the performance counter - simple non blocking version for now
1234 */
1235static ssize_t
1236perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1237{
1238 u64 values[3];
1239 int n;
1240
1241 /*
1242 * Return end-of-file for a read on a counter that is in
1243 * error state (i.e. because it was pinned but it couldn't be
1244 * scheduled on to the CPU at some point).
1245 */
1246 if (counter->state == PERF_COUNTER_STATE_ERROR)
1247 return 0;
1248
1249 mutex_lock(&counter->mutex);
1250 values[0] = perf_counter_read(counter);
1251 n = 1;
1252 if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1253 values[n++] = counter->total_time_enabled +
1254 atomic64_read(&counter->child_total_time_enabled);
1255 if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1256 values[n++] = counter->total_time_running +
1257 atomic64_read(&counter->child_total_time_running);
1258 mutex_unlock(&counter->mutex);
1259
1260 if (count < n * sizeof(u64))
1261 return -EINVAL;
1262 count = n * sizeof(u64);
1263
1264 if (copy_to_user(buf, values, count))
1265 return -EFAULT;
1266
1267 return count;
1268}
1269
1270static ssize_t
1271perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1272{
1273 struct perf_counter *counter = file->private_data;
1274
1275 return perf_read_hw(counter, buf, count);
1276}
1277
1278static unsigned int perf_poll(struct file *file, poll_table *wait)
1279{
1280 struct perf_counter *counter = file->private_data;
1281 struct perf_mmap_data *data;
1282 unsigned int events;
1283
1284 rcu_read_lock();
1285 data = rcu_dereference(counter->data);
1286 if (data)
1287 events = atomic_xchg(&data->wakeup, 0);
1288 else
1289 events = POLL_HUP;
1290 rcu_read_unlock();
1291
1292 poll_wait(file, &counter->waitq, wait);
1293
1294 return events;
1295}
1296
1297static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1298{
1299 struct perf_counter *counter = file->private_data;
1300 int err = 0;
1301
1302 switch (cmd) {
1303 case PERF_COUNTER_IOC_ENABLE:
1304 perf_counter_enable_family(counter);
1305 break;
1306 case PERF_COUNTER_IOC_DISABLE:
1307 perf_counter_disable_family(counter);
1308 break;
1309 case PERF_COUNTER_IOC_REFRESH:
1310 perf_counter_refresh(counter, arg);
1311 break;
1312 default:
1313 err = -ENOTTY;
1314 }
1315 return err;
1316}
1317
1318/*
1319 * Callers need to ensure there can be no nesting of this function, otherwise
1320 * the seqlock logic goes bad. We can not serialize this because the arch
1321 * code calls this from NMI context.
1322 */
1323void perf_counter_update_userpage(struct perf_counter *counter)
1324{
1325 struct perf_mmap_data *data;
1326 struct perf_counter_mmap_page *userpg;
1327
1328 rcu_read_lock();
1329 data = rcu_dereference(counter->data);
1330 if (!data)
1331 goto unlock;
1332
1333 userpg = data->user_page;
1334
1335 /*
1336 * Disable preemption so as to not let the corresponding user-space
1337 * spin too long if we get preempted.
1338 */
1339 preempt_disable();
1340 ++userpg->lock;
1341 barrier();
1342 userpg->index = counter->hw.idx;
1343 userpg->offset = atomic64_read(&counter->count);
1344 if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1345 userpg->offset -= atomic64_read(&counter->hw.prev_count);
1346
1347 barrier();
1348 ++userpg->lock;
1349 preempt_enable();
1350unlock:
1351 rcu_read_unlock();
1352}
1353
1354static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1355{
1356 struct perf_counter *counter = vma->vm_file->private_data;
1357 struct perf_mmap_data *data;
1358 int ret = VM_FAULT_SIGBUS;
1359
1360 rcu_read_lock();
1361 data = rcu_dereference(counter->data);
1362 if (!data)
1363 goto unlock;
1364
1365 if (vmf->pgoff == 0) {
1366 vmf->page = virt_to_page(data->user_page);
1367 } else {
1368 int nr = vmf->pgoff - 1;
1369
1370 if ((unsigned)nr > data->nr_pages)
1371 goto unlock;
1372
1373 vmf->page = virt_to_page(data->data_pages[nr]);
1374 }
1375 get_page(vmf->page);
1376 ret = 0;
1377unlock:
1378 rcu_read_unlock();
1379
1380 return ret;
1381}
1382
1383static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
1384{
1385 struct perf_mmap_data *data;
1386 unsigned long size;
1387 int i;
1388
1389 WARN_ON(atomic_read(&counter->mmap_count));
1390
1391 size = sizeof(struct perf_mmap_data);
1392 size += nr_pages * sizeof(void *);
1393
1394 data = kzalloc(size, GFP_KERNEL);
1395 if (!data)
1396 goto fail;
1397
1398 data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
1399 if (!data->user_page)
1400 goto fail_user_page;
1401
1402 for (i = 0; i < nr_pages; i++) {
1403 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
1404 if (!data->data_pages[i])
1405 goto fail_data_pages;
1406 }
1407
1408 data->nr_pages = nr_pages;
1409
1410 rcu_assign_pointer(counter->data, data);
1411
1412 return 0;
1413
1414fail_data_pages:
1415 for (i--; i >= 0; i--)
1416 free_page((unsigned long)data->data_pages[i]);
1417
1418 free_page((unsigned long)data->user_page);
1419
1420fail_user_page:
1421 kfree(data);
1422
1423fail:
1424 return -ENOMEM;
1425}
1426
1427static void __perf_mmap_data_free(struct rcu_head *rcu_head)
1428{
1429 struct perf_mmap_data *data = container_of(rcu_head,
1430 struct perf_mmap_data, rcu_head);
1431 int i;
1432
1433 free_page((unsigned long)data->user_page);
1434 for (i = 0; i < data->nr_pages; i++)
1435 free_page((unsigned long)data->data_pages[i]);
1436 kfree(data);
1437}
1438
1439static void perf_mmap_data_free(struct perf_counter *counter)
1440{
1441 struct perf_mmap_data *data = counter->data;
1442
1443 WARN_ON(atomic_read(&counter->mmap_count));
1444
1445 rcu_assign_pointer(counter->data, NULL);
1446 call_rcu(&data->rcu_head, __perf_mmap_data_free);
1447}
1448
1449static void perf_mmap_open(struct vm_area_struct *vma)
1450{
1451 struct perf_counter *counter = vma->vm_file->private_data;
1452
1453 atomic_inc(&counter->mmap_count);
1454}
1455
1456static void perf_mmap_close(struct vm_area_struct *vma)
1457{
1458 struct perf_counter *counter = vma->vm_file->private_data;
1459
1460 if (atomic_dec_and_mutex_lock(&counter->mmap_count,
1461 &counter->mmap_mutex)) {
1462 vma->vm_mm->locked_vm -= counter->data->nr_pages + 1;
1463 perf_mmap_data_free(counter);
1464 mutex_unlock(&counter->mmap_mutex);
1465 }
1466}
1467
1468static struct vm_operations_struct perf_mmap_vmops = {
1469 .open = perf_mmap_open,
1470 .close = perf_mmap_close,
1471 .fault = perf_mmap_fault,
1472};
1473
1474static int perf_mmap(struct file *file, struct vm_area_struct *vma)
1475{
1476 struct perf_counter *counter = file->private_data;
1477 unsigned long vma_size;
1478 unsigned long nr_pages;
1479 unsigned long locked, lock_limit;
1480 int ret = 0;
1481
1482 if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
1483 return -EINVAL;
1484
1485 vma_size = vma->vm_end - vma->vm_start;
1486 nr_pages = (vma_size / PAGE_SIZE) - 1;
1487
1488 /*
1489 * If we have data pages ensure they're a power-of-two number, so we
1490 * can do bitmasks instead of modulo.
1491 */
1492 if (nr_pages != 0 && !is_power_of_2(nr_pages))
1493 return -EINVAL;
1494
1495 if (vma_size != PAGE_SIZE * (1 + nr_pages))
1496 return -EINVAL;
1497
1498 if (vma->vm_pgoff != 0)
1499 return -EINVAL;
1500
1501 mutex_lock(&counter->mmap_mutex);
1502 if (atomic_inc_not_zero(&counter->mmap_count)) {
1503 if (nr_pages != counter->data->nr_pages)
1504 ret = -EINVAL;
1505 goto unlock;
1506 }
1507
1508 locked = vma->vm_mm->locked_vm;
1509 locked += nr_pages + 1;
1510
1511 lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
1512 lock_limit >>= PAGE_SHIFT;
1513
1514 if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
1515 ret = -EPERM;
1516 goto unlock;
1517 }
1518
1519 WARN_ON(counter->data);
1520 ret = perf_mmap_data_alloc(counter, nr_pages);
1521 if (ret)
1522 goto unlock;
1523
1524 atomic_set(&counter->mmap_count, 1);
1525 vma->vm_mm->locked_vm += nr_pages + 1;
1526unlock:
1527 mutex_unlock(&counter->mmap_mutex);
1528
1529 vma->vm_flags &= ~VM_MAYWRITE;
1530 vma->vm_flags |= VM_RESERVED;
1531 vma->vm_ops = &perf_mmap_vmops;
1532
1533 return ret;
1534}
1535
1536static int perf_fasync(int fd, struct file *filp, int on)
1537{
1538 struct perf_counter *counter = filp->private_data;
1539 struct inode *inode = filp->f_path.dentry->d_inode;
1540 int retval;
1541
1542 mutex_lock(&inode->i_mutex);
1543 retval = fasync_helper(fd, filp, on, &counter->fasync);
1544 mutex_unlock(&inode->i_mutex);
1545
1546 if (retval < 0)
1547 return retval;
1548
1549 return 0;
1550}
1551
1552static const struct file_operations perf_fops = {
1553 .release = perf_release,
1554 .read = perf_read,
1555 .poll = perf_poll,
1556 .unlocked_ioctl = perf_ioctl,
1557 .compat_ioctl = perf_ioctl,
1558 .mmap = perf_mmap,
1559 .fasync = perf_fasync,
1560};
1561
1562/*
1563 * Perf counter wakeup
1564 *
1565 * If there's data, ensure we set the poll() state and publish everything
1566 * to user-space before waking everybody up.
1567 */
1568
1569void perf_counter_wakeup(struct perf_counter *counter)
1570{
1571 struct perf_mmap_data *data;
1572
1573 rcu_read_lock();
1574 data = rcu_dereference(counter->data);
1575 if (data) {
1576 atomic_set(&data->wakeup, POLL_IN);
1577 /*
1578 * Ensure all data writes are issued before updating the
1579 * user-space data head information. The matching rmb()
1580 * will be in userspace after reading this value.
1581 */
1582 smp_wmb();
1583 data->user_page->data_head = atomic_read(&data->head);
1584 }
1585 rcu_read_unlock();
1586
1587 wake_up_all(&counter->waitq);
1588
1589 if (counter->pending_kill) {
1590 kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
1591 counter->pending_kill = 0;
1592 }
1593}
1594
1595/*
1596 * Pending wakeups
1597 *
1598 * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
1599 *
1600 * The NMI bit means we cannot possibly take locks. Therefore, maintain a
1601 * single linked list and use cmpxchg() to add entries lockless.
1602 */
1603
1604static void perf_pending_counter(struct perf_pending_entry *entry)
1605{
1606 struct perf_counter *counter = container_of(entry,
1607 struct perf_counter, pending);
1608
1609 if (counter->pending_disable) {
1610 counter->pending_disable = 0;
1611 perf_counter_disable(counter);
1612 }
1613
1614 if (counter->pending_wakeup) {
1615 counter->pending_wakeup = 0;
1616 perf_counter_wakeup(counter);
1617 }
1618}
1619
1620#define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
1621
1622static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
1623 PENDING_TAIL,
1624};
1625
1626static void perf_pending_queue(struct perf_pending_entry *entry,
1627 void (*func)(struct perf_pending_entry *))
1628{
1629 struct perf_pending_entry **head;
1630
1631 if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
1632 return;
1633
1634 entry->func = func;
1635
1636 head = &get_cpu_var(perf_pending_head);
1637
1638 do {
1639 entry->next = *head;
1640 } while (cmpxchg(head, entry->next, entry) != entry->next);
1641
1642 set_perf_counter_pending();
1643
1644 put_cpu_var(perf_pending_head);
1645}
1646
1647static int __perf_pending_run(void)
1648{
1649 struct perf_pending_entry *list;
1650 int nr = 0;
1651
1652 list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
1653 while (list != PENDING_TAIL) {
1654 void (*func)(struct perf_pending_entry *);
1655 struct perf_pending_entry *entry = list;
1656
1657 list = list->next;
1658
1659 func = entry->func;
1660 entry->next = NULL;
1661 /*
1662 * Ensure we observe the unqueue before we issue the wakeup,
1663 * so that we won't be waiting forever.
1664 * -- see perf_not_pending().
1665 */
1666 smp_wmb();
1667
1668 func(entry);
1669 nr++;
1670 }
1671
1672 return nr;
1673}
1674
1675static inline int perf_not_pending(struct perf_counter *counter)
1676{
1677 /*
1678 * If we flush on whatever cpu we run, there is a chance we don't
1679 * need to wait.
1680 */
1681 get_cpu();
1682 __perf_pending_run();
1683 put_cpu();
1684
1685 /*
1686 * Ensure we see the proper queue state before going to sleep
1687 * so that we do not miss the wakeup. -- see perf_pending_handle()
1688 */
1689 smp_rmb();
1690 return counter->pending.next == NULL;
1691}
1692
1693static void perf_pending_sync(struct perf_counter *counter)
1694{
1695 wait_event(counter->waitq, perf_not_pending(counter));
1696}
1697
1698void perf_counter_do_pending(void)
1699{
1700 __perf_pending_run();
1701}
1702
1703/*
1704 * Callchain support -- arch specific
1705 */
1706
1707__weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1708{
1709 return NULL;
1710}
1711
1712/*
1713 * Output
1714 */
1715
1716struct perf_output_handle {
1717 struct perf_counter *counter;
1718 struct perf_mmap_data *data;
1719 unsigned int offset;
1720 unsigned int head;
1721 int wakeup;
1722 int nmi;
1723 int overflow;
1724};
1725
1726static inline void __perf_output_wakeup(struct perf_output_handle *handle)
1727{
1728 if (handle->nmi) {
1729 handle->counter->pending_wakeup = 1;
1730 perf_pending_queue(&handle->counter->pending,
1731 perf_pending_counter);
1732 } else
1733 perf_counter_wakeup(handle->counter);
1734}
1735
1736static int perf_output_begin(struct perf_output_handle *handle,
1737 struct perf_counter *counter, unsigned int size,
1738 int nmi, int overflow)
1739{
1740 struct perf_mmap_data *data;
1741 unsigned int offset, head;
1742
1743 rcu_read_lock();
1744 data = rcu_dereference(counter->data);
1745 if (!data)
1746 goto out;
1747
1748 handle->counter = counter;
1749 handle->nmi = nmi;
1750 handle->overflow = overflow;
1751
1752 if (!data->nr_pages)
1753 goto fail;
1754
1755 do {
1756 offset = head = atomic_read(&data->head);
1757 head += size;
1758 } while (atomic_cmpxchg(&data->head, offset, head) != offset);
1759
1760 handle->data = data;
1761 handle->offset = offset;
1762 handle->head = head;
1763 handle->wakeup = (offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT);
1764
1765 return 0;
1766
1767fail:
1768 __perf_output_wakeup(handle);
1769out:
1770 rcu_read_unlock();
1771
1772 return -ENOSPC;
1773}
1774
1775static void perf_output_copy(struct perf_output_handle *handle,
1776 void *buf, unsigned int len)
1777{
1778 unsigned int pages_mask;
1779 unsigned int offset;
1780 unsigned int size;
1781 void **pages;
1782
1783 offset = handle->offset;
1784 pages_mask = handle->data->nr_pages - 1;
1785 pages = handle->data->data_pages;
1786
1787 do {
1788 unsigned int page_offset;
1789 int nr;
1790
1791 nr = (offset >> PAGE_SHIFT) & pages_mask;
1792 page_offset = offset & (PAGE_SIZE - 1);
1793 size = min_t(unsigned int, PAGE_SIZE - page_offset, len);
1794
1795 memcpy(pages[nr] + page_offset, buf, size);
1796
1797 len -= size;
1798 buf += size;
1799 offset += size;
1800 } while (len);
1801
1802 handle->offset = offset;
1803
1804 WARN_ON_ONCE(handle->offset > handle->head);
1805}
1806
1807#define perf_output_put(handle, x) \
1808 perf_output_copy((handle), &(x), sizeof(x))
1809
1810static void perf_output_end(struct perf_output_handle *handle)
1811{
1812 int wakeup_events = handle->counter->hw_event.wakeup_events;
1813
1814 if (handle->overflow && wakeup_events) {
1815 int events = atomic_inc_return(&handle->data->events);
1816 if (events >= wakeup_events) {
1817 atomic_sub(wakeup_events, &handle->data->events);
1818 __perf_output_wakeup(handle);
1819 }
1820 } else if (handle->wakeup)
1821 __perf_output_wakeup(handle);
1822 rcu_read_unlock();
1823}
1824
1825static void perf_counter_output(struct perf_counter *counter,
1826 int nmi, struct pt_regs *regs, u64 addr)
1827{
1828 int ret;
1829 u64 record_type = counter->hw_event.record_type;
1830 struct perf_output_handle handle;
1831 struct perf_event_header header;
1832 u64 ip;
1833 struct {
1834 u32 pid, tid;
1835 } tid_entry;
1836 struct {
1837 u64 event;
1838 u64 counter;
1839 } group_entry;
1840 struct perf_callchain_entry *callchain = NULL;
1841 int callchain_size = 0;
1842 u64 time;
1843
1844 header.type = 0;
1845 header.size = sizeof(header);
1846
1847 header.misc = PERF_EVENT_MISC_OVERFLOW;
1848 header.misc |= user_mode(regs) ?
1849 PERF_EVENT_MISC_USER : PERF_EVENT_MISC_KERNEL;
1850
1851 if (record_type & PERF_RECORD_IP) {
1852 ip = instruction_pointer(regs);
1853 header.type |= PERF_RECORD_IP;
1854 header.size += sizeof(ip);
1855 }
1856
1857 if (record_type & PERF_RECORD_TID) {
1858 /* namespace issues */
1859 tid_entry.pid = current->group_leader->pid;
1860 tid_entry.tid = current->pid;
1861
1862 header.type |= PERF_RECORD_TID;
1863 header.size += sizeof(tid_entry);
1864 }
1865
1866 if (record_type & PERF_RECORD_TIME) {
1867 /*
1868 * Maybe do better on x86 and provide cpu_clock_nmi()
1869 */
1870 time = sched_clock();
1871
1872 header.type |= PERF_RECORD_TIME;
1873 header.size += sizeof(u64);
1874 }
1875
1876 if (record_type & PERF_RECORD_ADDR) {
1877 header.type |= PERF_RECORD_ADDR;
1878 header.size += sizeof(u64);
1879 }
1880
1881 if (record_type & PERF_RECORD_GROUP) {
1882 header.type |= PERF_RECORD_GROUP;
1883 header.size += sizeof(u64) +
1884 counter->nr_siblings * sizeof(group_entry);
1885 }
1886
1887 if (record_type & PERF_RECORD_CALLCHAIN) {
1888 callchain = perf_callchain(regs);
1889
1890 if (callchain) {
1891 callchain_size = (1 + callchain->nr) * sizeof(u64);
1892
1893 header.type |= PERF_RECORD_CALLCHAIN;
1894 header.size += callchain_size;
1895 }
1896 }
1897
1898 ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
1899 if (ret)
1900 return;
1901
1902 perf_output_put(&handle, header);
1903
1904 if (record_type & PERF_RECORD_IP)
1905 perf_output_put(&handle, ip);
1906
1907 if (record_type & PERF_RECORD_TID)
1908 perf_output_put(&handle, tid_entry);
1909
1910 if (record_type & PERF_RECORD_TIME)
1911 perf_output_put(&handle, time);
1912
1913 if (record_type & PERF_RECORD_ADDR)
1914 perf_output_put(&handle, addr);
1915
1916 if (record_type & PERF_RECORD_GROUP) {
1917 struct perf_counter *leader, *sub;
1918 u64 nr = counter->nr_siblings;
1919
1920 perf_output_put(&handle, nr);
1921
1922 leader = counter->group_leader;
1923 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
1924 if (sub != counter)
1925 sub->hw_ops->read(sub);
1926
1927 group_entry.event = sub->hw_event.config;
1928 group_entry.counter = atomic64_read(&sub->count);
1929
1930 perf_output_put(&handle, group_entry);
1931 }
1932 }
1933
1934 if (callchain)
1935 perf_output_copy(&handle, callchain, callchain_size);
1936
1937 perf_output_end(&handle);
1938}
1939
1940/*
1941 * comm tracking
1942 */
1943
1944struct perf_comm_event {
1945 struct task_struct *task;
1946 char *comm;
1947 int comm_size;
1948
1949 struct {
1950 struct perf_event_header header;
1951
1952 u32 pid;
1953 u32 tid;
1954 } event;
1955};
1956
1957static void perf_counter_comm_output(struct perf_counter *counter,
1958 struct perf_comm_event *comm_event)
1959{
1960 struct perf_output_handle handle;
1961 int size = comm_event->event.header.size;
1962 int ret = perf_output_begin(&handle, counter, size, 0, 0);
1963
1964 if (ret)
1965 return;
1966
1967 perf_output_put(&handle, comm_event->event);
1968 perf_output_copy(&handle, comm_event->comm,
1969 comm_event->comm_size);
1970 perf_output_end(&handle);
1971}
1972
1973static int perf_counter_comm_match(struct perf_counter *counter,
1974 struct perf_comm_event *comm_event)
1975{
1976 if (counter->hw_event.comm &&
1977 comm_event->event.header.type == PERF_EVENT_COMM)
1978 return 1;
1979
1980 return 0;
1981}
1982
1983static void perf_counter_comm_ctx(struct perf_counter_context *ctx,
1984 struct perf_comm_event *comm_event)
1985{
1986 struct perf_counter *counter;
1987
1988 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
1989 return;
1990
1991 rcu_read_lock();
1992 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
1993 if (perf_counter_comm_match(counter, comm_event))
1994 perf_counter_comm_output(counter, comm_event);
1995 }
1996 rcu_read_unlock();
1997}
1998
1999static void perf_counter_comm_event(struct perf_comm_event *comm_event)
2000{
2001 struct perf_cpu_context *cpuctx;
2002 unsigned int size;
2003 char *comm = comm_event->task->comm;
2004
2005 size = ALIGN(strlen(comm)+1, sizeof(u64));
2006
2007 comm_event->comm = comm;
2008 comm_event->comm_size = size;
2009
2010 comm_event->event.header.size = sizeof(comm_event->event) + size;
2011
2012 cpuctx = &get_cpu_var(perf_cpu_context);
2013 perf_counter_comm_ctx(&cpuctx->ctx, comm_event);
2014 put_cpu_var(perf_cpu_context);
2015
2016 perf_counter_comm_ctx(&current->perf_counter_ctx, comm_event);
2017}
2018
2019void perf_counter_comm(struct task_struct *task)
2020{
2021 struct perf_comm_event comm_event;
2022
2023 if (!atomic_read(&nr_comm_tracking))
2024 return;
2025
2026 comm_event = (struct perf_comm_event){
2027 .task = task,
2028 .event = {
2029 .header = { .type = PERF_EVENT_COMM, },
2030 .pid = task->group_leader->pid,
2031 .tid = task->pid,
2032 },
2033 };
2034
2035 perf_counter_comm_event(&comm_event);
2036}
2037
2038/*
2039 * mmap tracking
2040 */
2041
2042struct perf_mmap_event {
2043 struct file *file;
2044 char *file_name;
2045 int file_size;
2046
2047 struct {
2048 struct perf_event_header header;
2049
2050 u32 pid;
2051 u32 tid;
2052 u64 start;
2053 u64 len;
2054 u64 pgoff;
2055 } event;
2056};
2057
2058static void perf_counter_mmap_output(struct perf_counter *counter,
2059 struct perf_mmap_event *mmap_event)
2060{
2061 struct perf_output_handle handle;
2062 int size = mmap_event->event.header.size;
2063 int ret = perf_output_begin(&handle, counter, size, 0, 0);
2064
2065 if (ret)
2066 return;
2067
2068 perf_output_put(&handle, mmap_event->event);
2069 perf_output_copy(&handle, mmap_event->file_name,
2070 mmap_event->file_size);
2071 perf_output_end(&handle);
2072}
2073
2074static int perf_counter_mmap_match(struct perf_counter *counter,
2075 struct perf_mmap_event *mmap_event)
2076{
2077 if (counter->hw_event.mmap &&
2078 mmap_event->event.header.type == PERF_EVENT_MMAP)
2079 return 1;
2080
2081 if (counter->hw_event.munmap &&
2082 mmap_event->event.header.type == PERF_EVENT_MUNMAP)
2083 return 1;
2084
2085 return 0;
2086}
2087
2088static void perf_counter_mmap_ctx(struct perf_counter_context *ctx,
2089 struct perf_mmap_event *mmap_event)
2090{
2091 struct perf_counter *counter;
2092
2093 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2094 return;
2095
2096 rcu_read_lock();
2097 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2098 if (perf_counter_mmap_match(counter, mmap_event))
2099 perf_counter_mmap_output(counter, mmap_event);
2100 }
2101 rcu_read_unlock();
2102}
2103
2104static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
2105{
2106 struct perf_cpu_context *cpuctx;
2107 struct file *file = mmap_event->file;
2108 unsigned int size;
2109 char tmp[16];
2110 char *buf = NULL;
2111 char *name;
2112
2113 if (file) {
2114 buf = kzalloc(PATH_MAX, GFP_KERNEL);
2115 if (!buf) {
2116 name = strncpy(tmp, "//enomem", sizeof(tmp));
2117 goto got_name;
2118 }
2119 name = d_path(&file->f_path, buf, PATH_MAX);
2120 if (IS_ERR(name)) {
2121 name = strncpy(tmp, "//toolong", sizeof(tmp));
2122 goto got_name;
2123 }
2124 } else {
2125 name = strncpy(tmp, "//anon", sizeof(tmp));
2126 goto got_name;
2127 }
2128
2129got_name:
2130 size = ALIGN(strlen(name)+1, sizeof(u64));
2131
2132 mmap_event->file_name = name;
2133 mmap_event->file_size = size;
2134
2135 mmap_event->event.header.size = sizeof(mmap_event->event) + size;
2136
2137 cpuctx = &get_cpu_var(perf_cpu_context);
2138 perf_counter_mmap_ctx(&cpuctx->ctx, mmap_event);
2139 put_cpu_var(perf_cpu_context);
2140
2141 perf_counter_mmap_ctx(&current->perf_counter_ctx, mmap_event);
2142
2143 kfree(buf);
2144}
2145
2146void perf_counter_mmap(unsigned long addr, unsigned long len,
2147 unsigned long pgoff, struct file *file)
2148{
2149 struct perf_mmap_event mmap_event;
2150
2151 if (!atomic_read(&nr_mmap_tracking))
2152 return;
2153
2154 mmap_event = (struct perf_mmap_event){
2155 .file = file,
2156 .event = {
2157 .header = { .type = PERF_EVENT_MMAP, },
2158 .pid = current->group_leader->pid,
2159 .tid = current->pid,
2160 .start = addr,
2161 .len = len,
2162 .pgoff = pgoff,
2163 },
2164 };
2165
2166 perf_counter_mmap_event(&mmap_event);
2167}
2168
2169void perf_counter_munmap(unsigned long addr, unsigned long len,
2170 unsigned long pgoff, struct file *file)
2171{
2172 struct perf_mmap_event mmap_event;
2173
2174 if (!atomic_read(&nr_munmap_tracking))
2175 return;
2176
2177 mmap_event = (struct perf_mmap_event){
2178 .file = file,
2179 .event = {
2180 .header = { .type = PERF_EVENT_MUNMAP, },
2181 .pid = current->group_leader->pid,
2182 .tid = current->pid,
2183 .start = addr,
2184 .len = len,
2185 .pgoff = pgoff,
2186 },
2187 };
2188
2189 perf_counter_mmap_event(&mmap_event);
2190}
2191
2192/*
2193 * Generic counter overflow handling.
2194 */
2195
2196int perf_counter_overflow(struct perf_counter *counter,
2197 int nmi, struct pt_regs *regs, u64 addr)
2198{
2199 int events = atomic_read(&counter->event_limit);
2200 int ret = 0;
2201
2202 counter->pending_kill = POLL_IN;
2203 if (events && atomic_dec_and_test(&counter->event_limit)) {
2204 ret = 1;
2205 counter->pending_kill = POLL_HUP;
2206 if (nmi) {
2207 counter->pending_disable = 1;
2208 perf_pending_queue(&counter->pending,
2209 perf_pending_counter);
2210 } else
2211 perf_counter_disable(counter);
2212 }
2213
2214 perf_counter_output(counter, nmi, regs, addr);
2215 return ret;
2216}
2217
2218/*
2219 * Generic software counter infrastructure
2220 */
2221
2222static void perf_swcounter_update(struct perf_counter *counter)
2223{
2224 struct hw_perf_counter *hwc = &counter->hw;
2225 u64 prev, now;
2226 s64 delta;
2227
2228again:
2229 prev = atomic64_read(&hwc->prev_count);
2230 now = atomic64_read(&hwc->count);
2231 if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
2232 goto again;
2233
2234 delta = now - prev;
2235
2236 atomic64_add(delta, &counter->count);
2237 atomic64_sub(delta, &hwc->period_left);
2238}
2239
2240static void perf_swcounter_set_period(struct perf_counter *counter)
2241{
2242 struct hw_perf_counter *hwc = &counter->hw;
2243 s64 left = atomic64_read(&hwc->period_left);
2244 s64 period = hwc->irq_period;
2245
2246 if (unlikely(left <= -period)) {
2247 left = period;
2248 atomic64_set(&hwc->period_left, left);
2249 }
2250
2251 if (unlikely(left <= 0)) {
2252 left += period;
2253 atomic64_add(period, &hwc->period_left);
2254 }
2255
2256 atomic64_set(&hwc->prev_count, -left);
2257 atomic64_set(&hwc->count, -left);
2258}
2259
2260static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
2261{
2262 enum hrtimer_restart ret = HRTIMER_RESTART;
2263 struct perf_counter *counter;
2264 struct pt_regs *regs;
2265
2266 counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
2267 counter->hw_ops->read(counter);
2268
2269 regs = get_irq_regs();
2270 /*
2271 * In case we exclude kernel IPs or are somehow not in interrupt
2272 * context, provide the next best thing, the user IP.
2273 */
2274 if ((counter->hw_event.exclude_kernel || !regs) &&
2275 !counter->hw_event.exclude_user)
2276 regs = task_pt_regs(current);
2277
2278 if (regs) {
2279 if (perf_counter_overflow(counter, 0, regs, 0))
2280 ret = HRTIMER_NORESTART;
2281 }
2282
2283 hrtimer_forward_now(hrtimer, ns_to_ktime(counter->hw.irq_period));
2284
2285 return ret;
2286}
2287
2288static void perf_swcounter_overflow(struct perf_counter *counter,
2289 int nmi, struct pt_regs *regs, u64 addr)
2290{
2291 perf_swcounter_update(counter);
2292 perf_swcounter_set_period(counter);
2293 if (perf_counter_overflow(counter, nmi, regs, addr))
2294 /* soft-disable the counter */
2295 ;
2296
2297}
2298
2299static int perf_swcounter_match(struct perf_counter *counter,
2300 enum perf_event_types type,
2301 u32 event, struct pt_regs *regs)
2302{
2303 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
2304 return 0;
2305
2306 if (perf_event_raw(&counter->hw_event))
2307 return 0;
2308
2309 if (perf_event_type(&counter->hw_event) != type)
2310 return 0;
2311
2312 if (perf_event_id(&counter->hw_event) != event)
2313 return 0;
2314
2315 if (counter->hw_event.exclude_user && user_mode(regs))
2316 return 0;
2317
2318 if (counter->hw_event.exclude_kernel && !user_mode(regs))
2319 return 0;
2320
2321 return 1;
2322}
2323
2324static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
2325 int nmi, struct pt_regs *regs, u64 addr)
2326{
2327 int neg = atomic64_add_negative(nr, &counter->hw.count);
2328 if (counter->hw.irq_period && !neg)
2329 perf_swcounter_overflow(counter, nmi, regs, addr);
2330}
2331
2332static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
2333 enum perf_event_types type, u32 event,
2334 u64 nr, int nmi, struct pt_regs *regs,
2335 u64 addr)
2336{
2337 struct perf_counter *counter;
2338
2339 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2340 return;
2341
2342 rcu_read_lock();
2343 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2344 if (perf_swcounter_match(counter, type, event, regs))
2345 perf_swcounter_add(counter, nr, nmi, regs, addr);
2346 }
2347 rcu_read_unlock();
2348}
2349
2350static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
2351{
2352 if (in_nmi())
2353 return &cpuctx->recursion[3];
2354
2355 if (in_irq())
2356 return &cpuctx->recursion[2];
2357
2358 if (in_softirq())
2359 return &cpuctx->recursion[1];
2360
2361 return &cpuctx->recursion[0];
2362}
2363
2364static void __perf_swcounter_event(enum perf_event_types type, u32 event,
2365 u64 nr, int nmi, struct pt_regs *regs,
2366 u64 addr)
2367{
2368 struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
2369 int *recursion = perf_swcounter_recursion_context(cpuctx);
2370
2371 if (*recursion)
2372 goto out;
2373
2374 (*recursion)++;
2375 barrier();
2376
2377 perf_swcounter_ctx_event(&cpuctx->ctx, type, event,
2378 nr, nmi, regs, addr);
2379 if (cpuctx->task_ctx) {
2380 perf_swcounter_ctx_event(cpuctx->task_ctx, type, event,
2381 nr, nmi, regs, addr);
2382 }
2383
2384 barrier();
2385 (*recursion)--;
2386
2387out:
2388 put_cpu_var(perf_cpu_context);
2389}
2390
2391void
2392perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
2393{
2394 __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs, addr);
2395}
2396
2397static void perf_swcounter_read(struct perf_counter *counter)
2398{
2399 perf_swcounter_update(counter);
2400}
2401
2402static int perf_swcounter_enable(struct perf_counter *counter)
2403{
2404 perf_swcounter_set_period(counter);
2405 return 0;
2406}
2407
2408static void perf_swcounter_disable(struct perf_counter *counter)
2409{
2410 perf_swcounter_update(counter);
2411}
2412
2413static const struct hw_perf_counter_ops perf_ops_generic = {
2414 .enable = perf_swcounter_enable,
2415 .disable = perf_swcounter_disable,
2416 .read = perf_swcounter_read,
2417};
2418
2419/*
2420 * Software counter: cpu wall time clock
2421 */
2422
2423static void cpu_clock_perf_counter_update(struct perf_counter *counter)
2424{
2425 int cpu = raw_smp_processor_id();
2426 s64 prev;
2427 u64 now;
2428
2429 now = cpu_clock(cpu);
2430 prev = atomic64_read(&counter->hw.prev_count);
2431 atomic64_set(&counter->hw.prev_count, now);
2432 atomic64_add(now - prev, &counter->count);
2433}
2434
2435static int cpu_clock_perf_counter_enable(struct perf_counter *counter)
2436{
2437 struct hw_perf_counter *hwc = &counter->hw;
2438 int cpu = raw_smp_processor_id();
2439
2440 atomic64_set(&hwc->prev_count, cpu_clock(cpu));
2441 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2442 hwc->hrtimer.function = perf_swcounter_hrtimer;
2443 if (hwc->irq_period) {
2444 __hrtimer_start_range_ns(&hwc->hrtimer,
2445 ns_to_ktime(hwc->irq_period), 0,
2446 HRTIMER_MODE_REL, 0);
2447 }
2448
2449 return 0;
2450}
2451
2452static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
2453{
2454 hrtimer_cancel(&counter->hw.hrtimer);
2455 cpu_clock_perf_counter_update(counter);
2456}
2457
2458static void cpu_clock_perf_counter_read(struct perf_counter *counter)
2459{
2460 cpu_clock_perf_counter_update(counter);
2461}
2462
2463static const struct hw_perf_counter_ops perf_ops_cpu_clock = {
2464 .enable = cpu_clock_perf_counter_enable,
2465 .disable = cpu_clock_perf_counter_disable,
2466 .read = cpu_clock_perf_counter_read,
2467};
2468
2469/*
2470 * Software counter: task time clock
2471 */
2472
2473static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
2474{
2475 u64 prev;
2476 s64 delta;
2477
2478 prev = atomic64_xchg(&counter->hw.prev_count, now);
2479 delta = now - prev;
2480 atomic64_add(delta, &counter->count);
2481}
2482
2483static int task_clock_perf_counter_enable(struct perf_counter *counter)
2484{
2485 struct hw_perf_counter *hwc = &counter->hw;
2486 u64 now;
2487
2488 now = counter->ctx->time;
2489
2490 atomic64_set(&hwc->prev_count, now);
2491 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2492 hwc->hrtimer.function = perf_swcounter_hrtimer;
2493 if (hwc->irq_period) {
2494 __hrtimer_start_range_ns(&hwc->hrtimer,
2495 ns_to_ktime(hwc->irq_period), 0,
2496 HRTIMER_MODE_REL, 0);
2497 }
2498
2499 return 0;
2500}
2501
2502static void task_clock_perf_counter_disable(struct perf_counter *counter)
2503{
2504 hrtimer_cancel(&counter->hw.hrtimer);
2505 task_clock_perf_counter_update(counter, counter->ctx->time);
2506
2507}
2508
2509static void task_clock_perf_counter_read(struct perf_counter *counter)
2510{
2511 u64 time;
2512
2513 if (!in_nmi()) {
2514 update_context_time(counter->ctx);
2515 time = counter->ctx->time;
2516 } else {
2517 u64 now = perf_clock();
2518 u64 delta = now - counter->ctx->timestamp;
2519 time = counter->ctx->time + delta;
2520 }
2521
2522 task_clock_perf_counter_update(counter, time);
2523}
2524
2525static const struct hw_perf_counter_ops perf_ops_task_clock = {
2526 .enable = task_clock_perf_counter_enable,
2527 .disable = task_clock_perf_counter_disable,
2528 .read = task_clock_perf_counter_read,
2529};
2530
2531/*
2532 * Software counter: cpu migrations
2533 */
2534
2535static inline u64 get_cpu_migrations(struct perf_counter *counter)
2536{
2537 struct task_struct *curr = counter->ctx->task;
2538
2539 if (curr)
2540 return curr->se.nr_migrations;
2541 return cpu_nr_migrations(smp_processor_id());
2542}
2543
2544static void cpu_migrations_perf_counter_update(struct perf_counter *counter)
2545{
2546 u64 prev, now;
2547 s64 delta;
2548
2549 prev = atomic64_read(&counter->hw.prev_count);
2550 now = get_cpu_migrations(counter);
2551
2552 atomic64_set(&counter->hw.prev_count, now);
2553
2554 delta = now - prev;
2555
2556 atomic64_add(delta, &counter->count);
2557}
2558
2559static void cpu_migrations_perf_counter_read(struct perf_counter *counter)
2560{
2561 cpu_migrations_perf_counter_update(counter);
2562}
2563
2564static int cpu_migrations_perf_counter_enable(struct perf_counter *counter)
2565{
2566 if (counter->prev_state <= PERF_COUNTER_STATE_OFF)
2567 atomic64_set(&counter->hw.prev_count,
2568 get_cpu_migrations(counter));
2569 return 0;
2570}
2571
2572static void cpu_migrations_perf_counter_disable(struct perf_counter *counter)
2573{
2574 cpu_migrations_perf_counter_update(counter);
2575}
2576
2577static const struct hw_perf_counter_ops perf_ops_cpu_migrations = {
2578 .enable = cpu_migrations_perf_counter_enable,
2579 .disable = cpu_migrations_perf_counter_disable,
2580 .read = cpu_migrations_perf_counter_read,
2581};
2582
2583#ifdef CONFIG_EVENT_PROFILE
2584void perf_tpcounter_event(int event_id)
2585{
2586 struct pt_regs *regs = get_irq_regs();
2587
2588 if (!regs)
2589 regs = task_pt_regs(current);
2590
2591 __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs, 0);
2592}
2593EXPORT_SYMBOL_GPL(perf_tpcounter_event);
2594
2595extern int ftrace_profile_enable(int);
2596extern void ftrace_profile_disable(int);
2597
2598static void tp_perf_counter_destroy(struct perf_counter *counter)
2599{
2600 ftrace_profile_disable(perf_event_id(&counter->hw_event));
2601}
2602
2603static const struct hw_perf_counter_ops *
2604tp_perf_counter_init(struct perf_counter *counter)
2605{
2606 int event_id = perf_event_id(&counter->hw_event);
2607 int ret;
2608
2609 ret = ftrace_profile_enable(event_id);
2610 if (ret)
2611 return NULL;
2612
2613 counter->destroy = tp_perf_counter_destroy;
2614 counter->hw.irq_period = counter->hw_event.irq_period;
2615
2616 return &perf_ops_generic;
2617}
2618#else
2619static const struct hw_perf_counter_ops *
2620tp_perf_counter_init(struct perf_counter *counter)
2621{
2622 return NULL;
2623}
2624#endif
2625
2626static const struct hw_perf_counter_ops *
2627sw_perf_counter_init(struct perf_counter *counter)
2628{
2629 struct perf_counter_hw_event *hw_event = &counter->hw_event;
2630 const struct hw_perf_counter_ops *hw_ops = NULL;
2631 struct hw_perf_counter *hwc = &counter->hw;
2632
2633 /*
2634 * Software counters (currently) can't in general distinguish
2635 * between user, kernel and hypervisor events.
2636 * However, context switches and cpu migrations are considered
2637 * to be kernel events, and page faults are never hypervisor
2638 * events.
2639 */
2640 switch (perf_event_id(&counter->hw_event)) {
2641 case PERF_COUNT_CPU_CLOCK:
2642 hw_ops = &perf_ops_cpu_clock;
2643
2644 if (hw_event->irq_period && hw_event->irq_period < 10000)
2645 hw_event->irq_period = 10000;
2646 break;
2647 case PERF_COUNT_TASK_CLOCK:
2648 /*
2649 * If the user instantiates this as a per-cpu counter,
2650 * use the cpu_clock counter instead.
2651 */
2652 if (counter->ctx->task)
2653 hw_ops = &perf_ops_task_clock;
2654 else
2655 hw_ops = &perf_ops_cpu_clock;
2656
2657 if (hw_event->irq_period && hw_event->irq_period < 10000)
2658 hw_event->irq_period = 10000;
2659 break;
2660 case PERF_COUNT_PAGE_FAULTS:
2661 case PERF_COUNT_PAGE_FAULTS_MIN:
2662 case PERF_COUNT_PAGE_FAULTS_MAJ:
2663 case PERF_COUNT_CONTEXT_SWITCHES:
2664 hw_ops = &perf_ops_generic;
2665 break;
2666 case PERF_COUNT_CPU_MIGRATIONS:
2667 if (!counter->hw_event.exclude_kernel)
2668 hw_ops = &perf_ops_cpu_migrations;
2669 break;
2670 }
2671
2672 if (hw_ops)
2673 hwc->irq_period = hw_event->irq_period;
2674
2675 return hw_ops;
2676}
2677
2678/*
2679 * Allocate and initialize a counter structure
2680 */
2681static struct perf_counter *
2682perf_counter_alloc(struct perf_counter_hw_event *hw_event,
2683 int cpu,
2684 struct perf_counter_context *ctx,
2685 struct perf_counter *group_leader,
2686 gfp_t gfpflags)
2687{
2688 const struct hw_perf_counter_ops *hw_ops;
2689 struct perf_counter *counter;
2690 long err;
2691
2692 counter = kzalloc(sizeof(*counter), gfpflags);
2693 if (!counter)
2694 return ERR_PTR(-ENOMEM);
2695
2696 /*
2697 * Single counters are their own group leaders, with an
2698 * empty sibling list:
2699 */
2700 if (!group_leader)
2701 group_leader = counter;
2702
2703 mutex_init(&counter->mutex);
2704 INIT_LIST_HEAD(&counter->list_entry);
2705 INIT_LIST_HEAD(&counter->event_entry);
2706 INIT_LIST_HEAD(&counter->sibling_list);
2707 init_waitqueue_head(&counter->waitq);
2708
2709 mutex_init(&counter->mmap_mutex);
2710
2711 INIT_LIST_HEAD(&counter->child_list);
2712
2713 counter->cpu = cpu;
2714 counter->hw_event = *hw_event;
2715 counter->group_leader = group_leader;
2716 counter->hw_ops = NULL;
2717 counter->ctx = ctx;
2718
2719 counter->state = PERF_COUNTER_STATE_INACTIVE;
2720 if (hw_event->disabled)
2721 counter->state = PERF_COUNTER_STATE_OFF;
2722
2723 hw_ops = NULL;
2724
2725 if (perf_event_raw(hw_event)) {
2726 hw_ops = hw_perf_counter_init(counter);
2727 goto done;
2728 }
2729
2730 switch (perf_event_type(hw_event)) {
2731 case PERF_TYPE_HARDWARE:
2732 hw_ops = hw_perf_counter_init(counter);
2733 break;
2734
2735 case PERF_TYPE_SOFTWARE:
2736 hw_ops = sw_perf_counter_init(counter);
2737 break;
2738
2739 case PERF_TYPE_TRACEPOINT:
2740 hw_ops = tp_perf_counter_init(counter);
2741 break;
2742 }
2743done:
2744 err = 0;
2745 if (!hw_ops)
2746 err = -EINVAL;
2747 else if (IS_ERR(hw_ops))
2748 err = PTR_ERR(hw_ops);
2749
2750 if (err) {
2751 kfree(counter);
2752 return ERR_PTR(err);
2753 }
2754
2755 counter->hw_ops = hw_ops;
2756
2757 if (counter->hw_event.mmap)
2758 atomic_inc(&nr_mmap_tracking);
2759 if (counter->hw_event.munmap)
2760 atomic_inc(&nr_munmap_tracking);
2761 if (counter->hw_event.comm)
2762 atomic_inc(&nr_comm_tracking);
2763
2764 return counter;
2765}
2766
2767/**
2768 * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
2769 *
2770 * @hw_event_uptr: event type attributes for monitoring/sampling
2771 * @pid: target pid
2772 * @cpu: target cpu
2773 * @group_fd: group leader counter fd
2774 */
2775SYSCALL_DEFINE5(perf_counter_open,
2776 const struct perf_counter_hw_event __user *, hw_event_uptr,
2777 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
2778{
2779 struct perf_counter *counter, *group_leader;
2780 struct perf_counter_hw_event hw_event;
2781 struct perf_counter_context *ctx;
2782 struct file *counter_file = NULL;
2783 struct file *group_file = NULL;
2784 int fput_needed = 0;
2785 int fput_needed2 = 0;
2786 int ret;
2787
2788 /* for future expandability... */
2789 if (flags)
2790 return -EINVAL;
2791
2792 if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)
2793 return -EFAULT;
2794
2795 /*
2796 * Get the target context (task or percpu):
2797 */
2798 ctx = find_get_context(pid, cpu);
2799 if (IS_ERR(ctx))
2800 return PTR_ERR(ctx);
2801
2802 /*
2803 * Look up the group leader (we will attach this counter to it):
2804 */
2805 group_leader = NULL;
2806 if (group_fd != -1) {
2807 ret = -EINVAL;
2808 group_file = fget_light(group_fd, &fput_needed);
2809 if (!group_file)
2810 goto err_put_context;
2811 if (group_file->f_op != &perf_fops)
2812 goto err_put_context;
2813
2814 group_leader = group_file->private_data;
2815 /*
2816 * Do not allow a recursive hierarchy (this new sibling
2817 * becoming part of another group-sibling):
2818 */
2819 if (group_leader->group_leader != group_leader)
2820 goto err_put_context;
2821 /*
2822 * Do not allow to attach to a group in a different
2823 * task or CPU context:
2824 */
2825 if (group_leader->ctx != ctx)
2826 goto err_put_context;
2827 /*
2828 * Only a group leader can be exclusive or pinned
2829 */
2830 if (hw_event.exclusive || hw_event.pinned)
2831 goto err_put_context;
2832 }
2833
2834 counter = perf_counter_alloc(&hw_event, cpu, ctx, group_leader,
2835 GFP_KERNEL);
2836 ret = PTR_ERR(counter);
2837 if (IS_ERR(counter))
2838 goto err_put_context;
2839
2840 ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
2841 if (ret < 0)
2842 goto err_free_put_context;
2843
2844 counter_file = fget_light(ret, &fput_needed2);
2845 if (!counter_file)
2846 goto err_free_put_context;
2847
2848 counter->filp = counter_file;
2849 mutex_lock(&ctx->mutex);
2850 perf_install_in_context(ctx, counter, cpu);
2851 mutex_unlock(&ctx->mutex);
2852
2853 fput_light(counter_file, fput_needed2);
2854
2855out_fput:
2856 fput_light(group_file, fput_needed);
2857
2858 return ret;
2859
2860err_free_put_context:
2861 kfree(counter);
2862
2863err_put_context:
2864 put_context(ctx);
2865
2866 goto out_fput;
2867}
2868
2869/*
2870 * Initialize the perf_counter context in a task_struct:
2871 */
2872static void
2873__perf_counter_init_context(struct perf_counter_context *ctx,
2874 struct task_struct *task)
2875{
2876 memset(ctx, 0, sizeof(*ctx));
2877 spin_lock_init(&ctx->lock);
2878 mutex_init(&ctx->mutex);
2879 INIT_LIST_HEAD(&ctx->counter_list);
2880 INIT_LIST_HEAD(&ctx->event_list);
2881 ctx->task = task;
2882}
2883
2884/*
2885 * inherit a counter from parent task to child task:
2886 */
2887static struct perf_counter *
2888inherit_counter(struct perf_counter *parent_counter,
2889 struct task_struct *parent,
2890 struct perf_counter_context *parent_ctx,
2891 struct task_struct *child,
2892 struct perf_counter *group_leader,
2893 struct perf_counter_context *child_ctx)
2894{
2895 struct perf_counter *child_counter;
2896
2897 /*
2898 * Instead of creating recursive hierarchies of counters,
2899 * we link inherited counters back to the original parent,
2900 * which has a filp for sure, which we use as the reference
2901 * count:
2902 */
2903 if (parent_counter->parent)
2904 parent_counter = parent_counter->parent;
2905
2906 child_counter = perf_counter_alloc(&parent_counter->hw_event,
2907 parent_counter->cpu, child_ctx,
2908 group_leader, GFP_KERNEL);
2909 if (IS_ERR(child_counter))
2910 return child_counter;
2911
2912 /*
2913 * Link it up in the child's context:
2914 */
2915 child_counter->task = child;
2916 add_counter_to_ctx(child_counter, child_ctx);
2917
2918 child_counter->parent = parent_counter;
2919 /*
2920 * inherit into child's child as well:
2921 */
2922 child_counter->hw_event.inherit = 1;
2923
2924 /*
2925 * Get a reference to the parent filp - we will fput it
2926 * when the child counter exits. This is safe to do because
2927 * we are in the parent and we know that the filp still
2928 * exists and has a nonzero count:
2929 */
2930 atomic_long_inc(&parent_counter->filp->f_count);
2931
2932 /*
2933 * Link this into the parent counter's child list
2934 */
2935 mutex_lock(&parent_counter->mutex);
2936 list_add_tail(&child_counter->child_list, &parent_counter->child_list);
2937
2938 /*
2939 * Make the child state follow the state of the parent counter,
2940 * not its hw_event.disabled bit. We hold the parent's mutex,
2941 * so we won't race with perf_counter_{en,dis}able_family.
2942 */
2943 if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
2944 child_counter->state = PERF_COUNTER_STATE_INACTIVE;
2945 else
2946 child_counter->state = PERF_COUNTER_STATE_OFF;
2947
2948 mutex_unlock(&parent_counter->mutex);
2949
2950 return child_counter;
2951}
2952
2953static int inherit_group(struct perf_counter *parent_counter,
2954 struct task_struct *parent,
2955 struct perf_counter_context *parent_ctx,
2956 struct task_struct *child,
2957 struct perf_counter_context *child_ctx)
2958{
2959 struct perf_counter *leader;
2960 struct perf_counter *sub;
2961 struct perf_counter *child_ctr;
2962
2963 leader = inherit_counter(parent_counter, parent, parent_ctx,
2964 child, NULL, child_ctx);
2965 if (IS_ERR(leader))
2966 return PTR_ERR(leader);
2967 list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
2968 child_ctr = inherit_counter(sub, parent, parent_ctx,
2969 child, leader, child_ctx);
2970 if (IS_ERR(child_ctr))
2971 return PTR_ERR(child_ctr);
2972 }
2973 return 0;
2974}
2975
2976static void sync_child_counter(struct perf_counter *child_counter,
2977 struct perf_counter *parent_counter)
2978{
2979 u64 parent_val, child_val;
2980
2981 parent_val = atomic64_read(&parent_counter->count);
2982 child_val = atomic64_read(&child_counter->count);
2983
2984 /*
2985 * Add back the child's count to the parent's count:
2986 */
2987 atomic64_add(child_val, &parent_counter->count);
2988 atomic64_add(child_counter->total_time_enabled,
2989 &parent_counter->child_total_time_enabled);
2990 atomic64_add(child_counter->total_time_running,
2991 &parent_counter->child_total_time_running);
2992
2993 /*
2994 * Remove this counter from the parent's list
2995 */
2996 mutex_lock(&parent_counter->mutex);
2997 list_del_init(&child_counter->child_list);
2998 mutex_unlock(&parent_counter->mutex);
2999
3000 /*
3001 * Release the parent counter, if this was the last
3002 * reference to it.
3003 */
3004 fput(parent_counter->filp);
3005}
3006
3007static void
3008__perf_counter_exit_task(struct task_struct *child,
3009 struct perf_counter *child_counter,
3010 struct perf_counter_context *child_ctx)
3011{
3012 struct perf_counter *parent_counter;
3013 struct perf_counter *sub, *tmp;
3014
3015 /*
3016 * If we do not self-reap then we have to wait for the
3017 * child task to unschedule (it will happen for sure),
3018 * so that its counter is at its final count. (This
3019 * condition triggers rarely - child tasks usually get
3020 * off their CPU before the parent has a chance to
3021 * get this far into the reaping action)
3022 */
3023 if (child != current) {
3024 wait_task_inactive(child, 0);
3025 list_del_init(&child_counter->list_entry);
3026 update_counter_times(child_counter);
3027 } else {
3028 struct perf_cpu_context *cpuctx;
3029 unsigned long flags;
3030 u64 perf_flags;
3031
3032 /*
3033 * Disable and unlink this counter.
3034 *
3035 * Be careful about zapping the list - IRQ/NMI context
3036 * could still be processing it:
3037 */
3038 local_irq_save(flags);
3039 perf_flags = hw_perf_save_disable();
3040
3041 cpuctx = &__get_cpu_var(perf_cpu_context);
3042
3043 group_sched_out(child_counter, cpuctx, child_ctx);
3044 update_counter_times(child_counter);
3045
3046 list_del_init(&child_counter->list_entry);
3047
3048 child_ctx->nr_counters--;
3049
3050 hw_perf_restore(perf_flags);
3051 local_irq_restore(flags);
3052 }
3053
3054 parent_counter = child_counter->parent;
3055 /*
3056 * It can happen that parent exits first, and has counters
3057 * that are still around due to the child reference. These
3058 * counters need to be zapped - but otherwise linger.
3059 */
3060 if (parent_counter) {
3061 sync_child_counter(child_counter, parent_counter);
3062 list_for_each_entry_safe(sub, tmp, &child_counter->sibling_list,
3063 list_entry) {
3064 if (sub->parent) {
3065 sync_child_counter(sub, sub->parent);
3066 free_counter(sub);
3067 }
3068 }
3069 free_counter(child_counter);
3070 }
3071}
3072
3073/*
3074 * When a child task exits, feed back counter values to parent counters.
3075 *
3076 * Note: we may be running in child context, but the PID is not hashed
3077 * anymore so new counters will not be added.
3078 */
3079void perf_counter_exit_task(struct task_struct *child)
3080{
3081 struct perf_counter *child_counter, *tmp;
3082 struct perf_counter_context *child_ctx;
3083
3084 child_ctx = &child->perf_counter_ctx;
3085
3086 if (likely(!child_ctx->nr_counters))
3087 return;
3088
3089 list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
3090 list_entry)
3091 __perf_counter_exit_task(child, child_counter, child_ctx);
3092}
3093
3094/*
3095 * Initialize the perf_counter context in task_struct
3096 */
3097void perf_counter_init_task(struct task_struct *child)
3098{
3099 struct perf_counter_context *child_ctx, *parent_ctx;
3100 struct perf_counter *counter;
3101 struct task_struct *parent = current;
3102
3103 child_ctx = &child->perf_counter_ctx;
3104 parent_ctx = &parent->perf_counter_ctx;
3105
3106 __perf_counter_init_context(child_ctx, child);
3107
3108 /*
3109 * This is executed from the parent task context, so inherit
3110 * counters that have been marked for cloning:
3111 */
3112
3113 if (likely(!parent_ctx->nr_counters))
3114 return;
3115
3116 /*
3117 * Lock the parent list. No need to lock the child - not PID
3118 * hashed yet and not running, so nobody can access it.
3119 */
3120 mutex_lock(&parent_ctx->mutex);
3121
3122 /*
3123 * We dont have to disable NMIs - we are only looking at
3124 * the list, not manipulating it:
3125 */
3126 list_for_each_entry(counter, &parent_ctx->counter_list, list_entry) {
3127 if (!counter->hw_event.inherit)
3128 continue;
3129
3130 if (inherit_group(counter, parent,
3131 parent_ctx, child, child_ctx))
3132 break;
3133 }
3134
3135 mutex_unlock(&parent_ctx->mutex);
3136}
3137
3138static void __cpuinit perf_counter_init_cpu(int cpu)
3139{
3140 struct perf_cpu_context *cpuctx;
3141
3142 cpuctx = &per_cpu(perf_cpu_context, cpu);
3143 __perf_counter_init_context(&cpuctx->ctx, NULL);
3144
3145 mutex_lock(&perf_resource_mutex);
3146 cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
3147 mutex_unlock(&perf_resource_mutex);
3148
3149 hw_perf_counter_setup(cpu);
3150}
3151
3152#ifdef CONFIG_HOTPLUG_CPU
3153static void __perf_counter_exit_cpu(void *info)
3154{
3155 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
3156 struct perf_counter_context *ctx = &cpuctx->ctx;
3157 struct perf_counter *counter, *tmp;
3158
3159 list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
3160 __perf_counter_remove_from_context(counter);
3161}
3162static void perf_counter_exit_cpu(int cpu)
3163{
3164 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
3165 struct perf_counter_context *ctx = &cpuctx->ctx;
3166
3167 mutex_lock(&ctx->mutex);
3168 smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
3169 mutex_unlock(&ctx->mutex);
3170}
3171#else
3172static inline void perf_counter_exit_cpu(int cpu) { }
3173#endif
3174
3175static int __cpuinit
3176perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
3177{
3178 unsigned int cpu = (long)hcpu;
3179
3180 switch (action) {
3181
3182 case CPU_UP_PREPARE:
3183 case CPU_UP_PREPARE_FROZEN:
3184 perf_counter_init_cpu(cpu);
3185 break;
3186
3187 case CPU_DOWN_PREPARE:
3188 case CPU_DOWN_PREPARE_FROZEN:
3189 perf_counter_exit_cpu(cpu);
3190 break;
3191
3192 default:
3193 break;
3194 }
3195
3196 return NOTIFY_OK;
3197}
3198
3199static struct notifier_block __cpuinitdata perf_cpu_nb = {
3200 .notifier_call = perf_cpu_notify,
3201};
3202
3203static int __init perf_counter_init(void)
3204{
3205 perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
3206 (void *)(long)smp_processor_id());
3207 register_cpu_notifier(&perf_cpu_nb);
3208
3209 return 0;
3210}
3211early_initcall(perf_counter_init);
3212
3213static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
3214{
3215 return sprintf(buf, "%d\n", perf_reserved_percpu);
3216}
3217
3218static ssize_t
3219perf_set_reserve_percpu(struct sysdev_class *class,
3220 const char *buf,
3221 size_t count)
3222{
3223 struct perf_cpu_context *cpuctx;
3224 unsigned long val;
3225 int err, cpu, mpt;
3226
3227 err = strict_strtoul(buf, 10, &val);
3228 if (err)
3229 return err;
3230 if (val > perf_max_counters)
3231 return -EINVAL;
3232
3233 mutex_lock(&perf_resource_mutex);
3234 perf_reserved_percpu = val;
3235 for_each_online_cpu(cpu) {
3236 cpuctx = &per_cpu(perf_cpu_context, cpu);
3237 spin_lock_irq(&cpuctx->ctx.lock);
3238 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
3239 perf_max_counters - perf_reserved_percpu);
3240 cpuctx->max_pertask = mpt;
3241 spin_unlock_irq(&cpuctx->ctx.lock);
3242 }
3243 mutex_unlock(&perf_resource_mutex);
3244
3245 return count;
3246}
3247
3248static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
3249{
3250 return sprintf(buf, "%d\n", perf_overcommit);
3251}
3252
3253static ssize_t
3254perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
3255{
3256 unsigned long val;
3257 int err;
3258
3259 err = strict_strtoul(buf, 10, &val);
3260 if (err)
3261 return err;
3262 if (val > 1)
3263 return -EINVAL;
3264
3265 mutex_lock(&perf_resource_mutex);
3266 perf_overcommit = val;
3267 mutex_unlock(&perf_resource_mutex);
3268
3269 return count;
3270}
3271
3272static SYSDEV_CLASS_ATTR(
3273 reserve_percpu,
3274 0644,
3275 perf_show_reserve_percpu,
3276 perf_set_reserve_percpu
3277 );
3278
3279static SYSDEV_CLASS_ATTR(
3280 overcommit,
3281 0644,
3282 perf_show_overcommit,
3283 perf_set_overcommit
3284 );
3285
3286static struct attribute *perfclass_attrs[] = {
3287 &attr_reserve_percpu.attr,
3288 &attr_overcommit.attr,
3289 NULL
3290};
3291
3292static struct attribute_group perfclass_attr_group = {
3293 .attrs = perfclass_attrs,
3294 .name = "perf_counters",
3295};
3296
3297static int __init perf_counter_sysfs_init(void)
3298{
3299 return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
3300 &perfclass_attr_group);
3301}
3302device_initcall(perf_counter_sysfs_init);
diff --git a/kernel/sched.c b/kernel/sched.c
index b902e587a3a0..2f600e30dcf0 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -584,6 +584,7 @@ struct rq {
584 struct load_weight load; 584 struct load_weight load;
585 unsigned long nr_load_updates; 585 unsigned long nr_load_updates;
586 u64 nr_switches; 586 u64 nr_switches;
587 u64 nr_migrations_in;
587 588
588 struct cfs_rq cfs; 589 struct cfs_rq cfs;
589 struct rt_rq rt; 590 struct rt_rq rt;
@@ -692,7 +693,7 @@ static inline int cpu_of(struct rq *rq)
692#define task_rq(p) cpu_rq(task_cpu(p)) 693#define task_rq(p) cpu_rq(task_cpu(p))
693#define cpu_curr(cpu) (cpu_rq(cpu)->curr) 694#define cpu_curr(cpu) (cpu_rq(cpu)->curr)
694 695
695static inline void update_rq_clock(struct rq *rq) 696inline void update_rq_clock(struct rq *rq)
696{ 697{
697 rq->clock = sched_clock_cpu(cpu_of(rq)); 698 rq->clock = sched_clock_cpu(cpu_of(rq));
698} 699}
@@ -1967,12 +1968,15 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
1967 p->se.sleep_start -= clock_offset; 1968 p->se.sleep_start -= clock_offset;
1968 if (p->se.block_start) 1969 if (p->se.block_start)
1969 p->se.block_start -= clock_offset; 1970 p->se.block_start -= clock_offset;
1971#endif
1970 if (old_cpu != new_cpu) { 1972 if (old_cpu != new_cpu) {
1971 schedstat_inc(p, se.nr_migrations); 1973 p->se.nr_migrations++;
1974 new_rq->nr_migrations_in++;
1975#ifdef CONFIG_SCHEDSTATS
1972 if (task_hot(p, old_rq->clock, NULL)) 1976 if (task_hot(p, old_rq->clock, NULL))
1973 schedstat_inc(p, se.nr_forced2_migrations); 1977 schedstat_inc(p, se.nr_forced2_migrations);
1974 }
1975#endif 1978#endif
1979 }
1976 p->se.vruntime -= old_cfsrq->min_vruntime - 1980 p->se.vruntime -= old_cfsrq->min_vruntime -
1977 new_cfsrq->min_vruntime; 1981 new_cfsrq->min_vruntime;
1978 1982
@@ -2324,6 +2328,27 @@ static int sched_balance_self(int cpu, int flag)
2324 2328
2325#endif /* CONFIG_SMP */ 2329#endif /* CONFIG_SMP */
2326 2330
2331/**
2332 * task_oncpu_function_call - call a function on the cpu on which a task runs
2333 * @p: the task to evaluate
2334 * @func: the function to be called
2335 * @info: the function call argument
2336 *
2337 * Calls the function @func when the task is currently running. This might
2338 * be on the current CPU, which just calls the function directly
2339 */
2340void task_oncpu_function_call(struct task_struct *p,
2341 void (*func) (void *info), void *info)
2342{
2343 int cpu;
2344
2345 preempt_disable();
2346 cpu = task_cpu(p);
2347 if (task_curr(p))
2348 smp_call_function_single(cpu, func, info, 1);
2349 preempt_enable();
2350}
2351
2327/*** 2352/***
2328 * try_to_wake_up - wake up a thread 2353 * try_to_wake_up - wake up a thread
2329 * @p: the to-be-woken-up thread 2354 * @p: the to-be-woken-up thread
@@ -2480,6 +2505,7 @@ static void __sched_fork(struct task_struct *p)
2480 p->se.exec_start = 0; 2505 p->se.exec_start = 0;
2481 p->se.sum_exec_runtime = 0; 2506 p->se.sum_exec_runtime = 0;
2482 p->se.prev_sum_exec_runtime = 0; 2507 p->se.prev_sum_exec_runtime = 0;
2508 p->se.nr_migrations = 0;
2483 p->se.last_wakeup = 0; 2509 p->se.last_wakeup = 0;
2484 p->se.avg_overlap = 0; 2510 p->se.avg_overlap = 0;
2485 p->se.start_runtime = 0; 2511 p->se.start_runtime = 0;
@@ -2710,6 +2736,7 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
2710 */ 2736 */
2711 prev_state = prev->state; 2737 prev_state = prev->state;
2712 finish_arch_switch(prev); 2738 finish_arch_switch(prev);
2739 perf_counter_task_sched_in(current, cpu_of(rq));
2713 finish_lock_switch(rq, prev); 2740 finish_lock_switch(rq, prev);
2714#ifdef CONFIG_SMP 2741#ifdef CONFIG_SMP
2715 if (post_schedule) 2742 if (post_schedule)
@@ -2872,6 +2899,15 @@ unsigned long nr_active(void)
2872} 2899}
2873 2900
2874/* 2901/*
2902 * Externally visible per-cpu scheduler statistics:
2903 * cpu_nr_migrations(cpu) - number of migrations into that cpu
2904 */
2905u64 cpu_nr_migrations(int cpu)
2906{
2907 return cpu_rq(cpu)->nr_migrations_in;
2908}
2909
2910/*
2875 * Update rq->cpu_load[] statistics. This function is usually called every 2911 * Update rq->cpu_load[] statistics. This function is usually called every
2876 * scheduler tick (TICK_NSEC). 2912 * scheduler tick (TICK_NSEC).
2877 */ 2913 */
@@ -4838,6 +4874,7 @@ void scheduler_tick(void)
4838 update_rq_clock(rq); 4874 update_rq_clock(rq);
4839 update_cpu_load(rq); 4875 update_cpu_load(rq);
4840 curr->sched_class->task_tick(rq, curr, 0); 4876 curr->sched_class->task_tick(rq, curr, 0);
4877 perf_counter_task_tick(curr, cpu);
4841 spin_unlock(&rq->lock); 4878 spin_unlock(&rq->lock);
4842 4879
4843#ifdef CONFIG_SMP 4880#ifdef CONFIG_SMP
@@ -5053,6 +5090,7 @@ need_resched_nonpreemptible:
5053 5090
5054 if (likely(prev != next)) { 5091 if (likely(prev != next)) {
5055 sched_info_switch(prev, next); 5092 sched_info_switch(prev, next);
5093 perf_counter_task_sched_out(prev, cpu);
5056 5094
5057 rq->nr_switches++; 5095 rq->nr_switches++;
5058 rq->curr = next; 5096 rq->curr = next;
diff --git a/kernel/sys.c b/kernel/sys.c
index e7998cf31498..438d99a38c87 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -14,6 +14,7 @@
14#include <linux/prctl.h> 14#include <linux/prctl.h>
15#include <linux/highuid.h> 15#include <linux/highuid.h>
16#include <linux/fs.h> 16#include <linux/fs.h>
17#include <linux/perf_counter.h>
17#include <linux/resource.h> 18#include <linux/resource.h>
18#include <linux/kernel.h> 19#include <linux/kernel.h>
19#include <linux/kexec.h> 20#include <linux/kexec.h>
@@ -1793,6 +1794,12 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
1793 case PR_SET_TSC: 1794 case PR_SET_TSC:
1794 error = SET_TSC_CTL(arg2); 1795 error = SET_TSC_CTL(arg2);
1795 break; 1796 break;
1797 case PR_TASK_PERF_COUNTERS_DISABLE:
1798 error = perf_counter_task_disable();
1799 break;
1800 case PR_TASK_PERF_COUNTERS_ENABLE:
1801 error = perf_counter_task_enable();
1802 break;
1796 case PR_GET_TIMERSLACK: 1803 case PR_GET_TIMERSLACK:
1797 error = current->timer_slack_ns; 1804 error = current->timer_slack_ns;
1798 break; 1805 break;
diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c
index 27dad2967387..68320f6b07b5 100644
--- a/kernel/sys_ni.c
+++ b/kernel/sys_ni.c
@@ -175,3 +175,6 @@ cond_syscall(compat_sys_timerfd_settime);
175cond_syscall(compat_sys_timerfd_gettime); 175cond_syscall(compat_sys_timerfd_gettime);
176cond_syscall(sys_eventfd); 176cond_syscall(sys_eventfd);
177cond_syscall(sys_eventfd2); 177cond_syscall(sys_eventfd2);
178
179/* performance counters: */
180cond_syscall(sys_perf_counter_open);
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index e3d2c7dd59b9..8203d70928d5 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -49,6 +49,7 @@
49#include <linux/reboot.h> 49#include <linux/reboot.h>
50#include <linux/ftrace.h> 50#include <linux/ftrace.h>
51#include <linux/slow-work.h> 51#include <linux/slow-work.h>
52#include <linux/perf_counter.h>
52 53
53#include <asm/uaccess.h> 54#include <asm/uaccess.h>
54#include <asm/processor.h> 55#include <asm/processor.h>
@@ -910,6 +911,16 @@ static struct ctl_table kern_table[] = {
910 .child = slow_work_sysctls, 911 .child = slow_work_sysctls,
911 }, 912 },
912#endif 913#endif
914#ifdef CONFIG_PERF_COUNTERS
915 {
916 .ctl_name = CTL_UNNUMBERED,
917 .procname = "perf_counter_privileged",
918 .data = &sysctl_perf_counter_priv,
919 .maxlen = sizeof(sysctl_perf_counter_priv),
920 .mode = 0644,
921 .proc_handler = &proc_dointvec,
922 },
923#endif
913/* 924/*
914 * NOTE: do not add new entries to this table unless you have read 925 * NOTE: do not add new entries to this table unless you have read
915 * Documentation/sysctl/ctl_unnumbered.txt 926 * Documentation/sysctl/ctl_unnumbered.txt
diff --git a/kernel/timer.c b/kernel/timer.c
index cffffad01c31..fed53be44fd9 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -37,6 +37,7 @@
37#include <linux/delay.h> 37#include <linux/delay.h>
38#include <linux/tick.h> 38#include <linux/tick.h>
39#include <linux/kallsyms.h> 39#include <linux/kallsyms.h>
40#include <linux/perf_counter.h>
40 41
41#include <asm/uaccess.h> 42#include <asm/uaccess.h>
42#include <asm/unistd.h> 43#include <asm/unistd.h>
@@ -1170,6 +1171,8 @@ static void run_timer_softirq(struct softirq_action *h)
1170{ 1171{
1171 struct tvec_base *base = __get_cpu_var(tvec_bases); 1172 struct tvec_base *base = __get_cpu_var(tvec_bases);
1172 1173
1174 perf_counter_do_pending();
1175
1173 hrtimer_run_pending(); 1176 hrtimer_run_pending();
1174 1177
1175 if (time_after_eq(jiffies, base->timer_jiffies)) 1178 if (time_after_eq(jiffies, base->timer_jiffies))
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-08 03:12:54 -0500