aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/Makefile1
-rw-r--r--kernel/exit.c16
-rw-r--r--kernel/fork.c12
-rw-r--r--kernel/mutex.c2
-rw-r--r--kernel/perf_counter.c4260
-rw-r--r--kernel/sched.c57
-rw-r--r--kernel/sys.c7
-rw-r--r--kernel/sys_ni.c3
-rw-r--r--kernel/sysctl.c27
-rw-r--r--kernel/timer.c3
10 files changed, 4377 insertions, 11 deletions
diff --git a/kernel/Makefile b/kernel/Makefile
index a35eee3436de..90b53f6dc226 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -96,6 +96,7 @@ obj-$(CONFIG_TRACING) += trace/
96obj-$(CONFIG_X86_DS) += trace/ 96obj-$(CONFIG_X86_DS) += trace/
97obj-$(CONFIG_SMP) += sched_cpupri.o 97obj-$(CONFIG_SMP) += sched_cpupri.o
98obj-$(CONFIG_SLOW_WORK) += slow-work.o 98obj-$(CONFIG_SLOW_WORK) += slow-work.o
99obj-$(CONFIG_PERF_COUNTERS) += perf_counter.o
99 100
100ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) 101ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y)
101# According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is 102# According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is
diff --git a/kernel/exit.c b/kernel/exit.c
index cab535c427b8..49cdf6946f34 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -48,6 +48,7 @@
48#include <linux/tracehook.h> 48#include <linux/tracehook.h>
49#include <linux/fs_struct.h> 49#include <linux/fs_struct.h>
50#include <linux/init_task.h> 50#include <linux/init_task.h>
51#include <linux/perf_counter.h>
51#include <trace/events/sched.h> 52#include <trace/events/sched.h>
52 53
53#include <asm/uaccess.h> 54#include <asm/uaccess.h>
@@ -154,6 +155,9 @@ static void delayed_put_task_struct(struct rcu_head *rhp)
154{ 155{
155 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu); 156 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
156 157
158#ifdef CONFIG_PERF_COUNTERS
159 WARN_ON_ONCE(tsk->perf_counter_ctxp);
160#endif
157 trace_sched_process_free(tsk); 161 trace_sched_process_free(tsk);
158 put_task_struct(tsk); 162 put_task_struct(tsk);
159} 163}
@@ -170,6 +174,7 @@ repeat:
170 atomic_dec(&__task_cred(p)->user->processes); 174 atomic_dec(&__task_cred(p)->user->processes);
171 175
172 proc_flush_task(p); 176 proc_flush_task(p);
177
173 write_lock_irq(&tasklist_lock); 178 write_lock_irq(&tasklist_lock);
174 tracehook_finish_release_task(p); 179 tracehook_finish_release_task(p);
175 __exit_signal(p); 180 __exit_signal(p);
@@ -971,16 +976,19 @@ NORET_TYPE void do_exit(long code)
971 module_put(tsk->binfmt->module); 976 module_put(tsk->binfmt->module);
972 977
973 proc_exit_connector(tsk); 978 proc_exit_connector(tsk);
979
980 /*
981 * Flush inherited counters to the parent - before the parent
982 * gets woken up by child-exit notifications.
983 */
984 perf_counter_exit_task(tsk);
985
974 exit_notify(tsk, group_dead); 986 exit_notify(tsk, group_dead);
975#ifdef CONFIG_NUMA 987#ifdef CONFIG_NUMA
976 mpol_put(tsk->mempolicy); 988 mpol_put(tsk->mempolicy);
977 tsk->mempolicy = NULL; 989 tsk->mempolicy = NULL;
978#endif 990#endif
979#ifdef CONFIG_FUTEX 991#ifdef CONFIG_FUTEX
980 /*
981 * This must happen late, after the PID is not
982 * hashed anymore:
983 */
984 if (unlikely(!list_empty(&tsk->pi_state_list))) 992 if (unlikely(!list_empty(&tsk->pi_state_list)))
985 exit_pi_state_list(tsk); 993 exit_pi_state_list(tsk);
986 if (unlikely(current->pi_state_cache)) 994 if (unlikely(current->pi_state_cache))
diff --git a/kernel/fork.c b/kernel/fork.c
index bb762b4dd217..4430eb1376f2 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -62,6 +62,7 @@
62#include <linux/blkdev.h> 62#include <linux/blkdev.h>
63#include <linux/fs_struct.h> 63#include <linux/fs_struct.h>
64#include <linux/magic.h> 64#include <linux/magic.h>
65#include <linux/perf_counter.h>
65 66
66#include <asm/pgtable.h> 67#include <asm/pgtable.h>
67#include <asm/pgalloc.h> 68#include <asm/pgalloc.h>
@@ -1096,6 +1097,10 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1096 /* Perform scheduler related setup. Assign this task to a CPU. */ 1097 /* Perform scheduler related setup. Assign this task to a CPU. */
1097 sched_fork(p, clone_flags); 1098 sched_fork(p, clone_flags);
1098 1099
1100 retval = perf_counter_init_task(p);
1101 if (retval)
1102 goto bad_fork_cleanup_policy;
1103
1099 if ((retval = audit_alloc(p))) 1104 if ((retval = audit_alloc(p)))
1100 goto bad_fork_cleanup_policy; 1105 goto bad_fork_cleanup_policy;
1101 /* copy all the process information */ 1106 /* copy all the process information */
@@ -1290,6 +1295,7 @@ bad_fork_cleanup_semundo:
1290bad_fork_cleanup_audit: 1295bad_fork_cleanup_audit:
1291 audit_free(p); 1296 audit_free(p);
1292bad_fork_cleanup_policy: 1297bad_fork_cleanup_policy:
1298 perf_counter_free_task(p);
1293#ifdef CONFIG_NUMA 1299#ifdef CONFIG_NUMA
1294 mpol_put(p->mempolicy); 1300 mpol_put(p->mempolicy);
1295bad_fork_cleanup_cgroup: 1301bad_fork_cleanup_cgroup:
@@ -1403,6 +1409,12 @@ long do_fork(unsigned long clone_flags,
1403 if (clone_flags & CLONE_VFORK) { 1409 if (clone_flags & CLONE_VFORK) {
1404 p->vfork_done = &vfork; 1410 p->vfork_done = &vfork;
1405 init_completion(&vfork); 1411 init_completion(&vfork);
1412 } else if (!(clone_flags & CLONE_VM)) {
1413 /*
1414 * vfork will do an exec which will call
1415 * set_task_comm()
1416 */
1417 perf_counter_fork(p);
1406 } 1418 }
1407 1419
1408 audit_finish_fork(p); 1420 audit_finish_fork(p);
diff --git a/kernel/mutex.c b/kernel/mutex.c
index e5cc0cd28d54..947b3ad551f8 100644
--- a/kernel/mutex.c
+++ b/kernel/mutex.c
@@ -89,7 +89,7 @@ __mutex_lock_slowpath(atomic_t *lock_count);
89 * 89 *
90 * This function is similar to (but not equivalent to) down(). 90 * This function is similar to (but not equivalent to) down().
91 */ 91 */
92void inline __sched mutex_lock(struct mutex *lock) 92void __sched mutex_lock(struct mutex *lock)
93{ 93{
94 might_sleep(); 94 might_sleep();
95 /* 95 /*
diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c
new file mode 100644
index 000000000000..ef5d8a5b2453
--- /dev/null
+++ b/kernel/perf_counter.c
@@ -0,0 +1,4260 @@
1/*
2 * Performance counter core code
3 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8 *
9 * For licensing details see kernel-base/COPYING
10 */
11
12#include <linux/fs.h>
13#include <linux/mm.h>
14#include <linux/cpu.h>
15#include <linux/smp.h>
16#include <linux/file.h>
17#include <linux/poll.h>
18#include <linux/sysfs.h>
19#include <linux/dcache.h>
20#include <linux/percpu.h>
21#include <linux/ptrace.h>
22#include <linux/vmstat.h>
23#include <linux/hardirq.h>
24#include <linux/rculist.h>
25#include <linux/uaccess.h>
26#include <linux/syscalls.h>
27#include <linux/anon_inodes.h>
28#include <linux/kernel_stat.h>
29#include <linux/perf_counter.h>
30
31#include <asm/irq_regs.h>
32
33/*
34 * Each CPU has a list of per CPU counters:
35 */
36DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
37
38int perf_max_counters __read_mostly = 1;
39static int perf_reserved_percpu __read_mostly;
40static int perf_overcommit __read_mostly = 1;
41
42static atomic_t nr_counters __read_mostly;
43static atomic_t nr_mmap_counters __read_mostly;
44static atomic_t nr_comm_counters __read_mostly;
45
46/*
47 * perf counter paranoia level:
48 * 0 - not paranoid
49 * 1 - disallow cpu counters to unpriv
50 * 2 - disallow kernel profiling to unpriv
51 */
52int sysctl_perf_counter_paranoid __read_mostly;
53
54static inline bool perf_paranoid_cpu(void)
55{
56 return sysctl_perf_counter_paranoid > 0;
57}
58
59static inline bool perf_paranoid_kernel(void)
60{
61 return sysctl_perf_counter_paranoid > 1;
62}
63
64int sysctl_perf_counter_mlock __read_mostly = 512; /* 'free' kb per user */
65
66/*
67 * max perf counter sample rate
68 */
69int sysctl_perf_counter_sample_rate __read_mostly = 100000;
70
71static atomic64_t perf_counter_id;
72
73/*
74 * Lock for (sysadmin-configurable) counter reservations:
75 */
76static DEFINE_SPINLOCK(perf_resource_lock);
77
78/*
79 * Architecture provided APIs - weak aliases:
80 */
81extern __weak const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
82{
83 return NULL;
84}
85
86void __weak hw_perf_disable(void) { barrier(); }
87void __weak hw_perf_enable(void) { barrier(); }
88
89void __weak hw_perf_counter_setup(int cpu) { barrier(); }
90
91int __weak
92hw_perf_group_sched_in(struct perf_counter *group_leader,
93 struct perf_cpu_context *cpuctx,
94 struct perf_counter_context *ctx, int cpu)
95{
96 return 0;
97}
98
99void __weak perf_counter_print_debug(void) { }
100
101static DEFINE_PER_CPU(int, disable_count);
102
103void __perf_disable(void)
104{
105 __get_cpu_var(disable_count)++;
106}
107
108bool __perf_enable(void)
109{
110 return !--__get_cpu_var(disable_count);
111}
112
113void perf_disable(void)
114{
115 __perf_disable();
116 hw_perf_disable();
117}
118
119void perf_enable(void)
120{
121 if (__perf_enable())
122 hw_perf_enable();
123}
124
125static void get_ctx(struct perf_counter_context *ctx)
126{
127 atomic_inc(&ctx->refcount);
128}
129
130static void free_ctx(struct rcu_head *head)
131{
132 struct perf_counter_context *ctx;
133
134 ctx = container_of(head, struct perf_counter_context, rcu_head);
135 kfree(ctx);
136}
137
138static void put_ctx(struct perf_counter_context *ctx)
139{
140 if (atomic_dec_and_test(&ctx->refcount)) {
141 if (ctx->parent_ctx)
142 put_ctx(ctx->parent_ctx);
143 if (ctx->task)
144 put_task_struct(ctx->task);
145 call_rcu(&ctx->rcu_head, free_ctx);
146 }
147}
148
149/*
150 * Get the perf_counter_context for a task and lock it.
151 * This has to cope with with the fact that until it is locked,
152 * the context could get moved to another task.
153 */
154static struct perf_counter_context *
155perf_lock_task_context(struct task_struct *task, unsigned long *flags)
156{
157 struct perf_counter_context *ctx;
158
159 rcu_read_lock();
160 retry:
161 ctx = rcu_dereference(task->perf_counter_ctxp);
162 if (ctx) {
163 /*
164 * If this context is a clone of another, it might
165 * get swapped for another underneath us by
166 * perf_counter_task_sched_out, though the
167 * rcu_read_lock() protects us from any context
168 * getting freed. Lock the context and check if it
169 * got swapped before we could get the lock, and retry
170 * if so. If we locked the right context, then it
171 * can't get swapped on us any more.
172 */
173 spin_lock_irqsave(&ctx->lock, *flags);
174 if (ctx != rcu_dereference(task->perf_counter_ctxp)) {
175 spin_unlock_irqrestore(&ctx->lock, *flags);
176 goto retry;
177 }
178 }
179 rcu_read_unlock();
180 return ctx;
181}
182
183/*
184 * Get the context for a task and increment its pin_count so it
185 * can't get swapped to another task. This also increments its
186 * reference count so that the context can't get freed.
187 */
188static struct perf_counter_context *perf_pin_task_context(struct task_struct *task)
189{
190 struct perf_counter_context *ctx;
191 unsigned long flags;
192
193 ctx = perf_lock_task_context(task, &flags);
194 if (ctx) {
195 ++ctx->pin_count;
196 get_ctx(ctx);
197 spin_unlock_irqrestore(&ctx->lock, flags);
198 }
199 return ctx;
200}
201
202static void perf_unpin_context(struct perf_counter_context *ctx)
203{
204 unsigned long flags;
205
206 spin_lock_irqsave(&ctx->lock, flags);
207 --ctx->pin_count;
208 spin_unlock_irqrestore(&ctx->lock, flags);
209 put_ctx(ctx);
210}
211
212/*
213 * Add a counter from the lists for its context.
214 * Must be called with ctx->mutex and ctx->lock held.
215 */
216static void
217list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
218{
219 struct perf_counter *group_leader = counter->group_leader;
220
221 /*
222 * Depending on whether it is a standalone or sibling counter,
223 * add it straight to the context's counter list, or to the group
224 * leader's sibling list:
225 */
226 if (group_leader == counter)
227 list_add_tail(&counter->list_entry, &ctx->counter_list);
228 else {
229 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
230 group_leader->nr_siblings++;
231 }
232
233 list_add_rcu(&counter->event_entry, &ctx->event_list);
234 ctx->nr_counters++;
235}
236
237/*
238 * Remove a counter from the lists for its context.
239 * Must be called with ctx->mutex and ctx->lock held.
240 */
241static void
242list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
243{
244 struct perf_counter *sibling, *tmp;
245
246 if (list_empty(&counter->list_entry))
247 return;
248 ctx->nr_counters--;
249
250 list_del_init(&counter->list_entry);
251 list_del_rcu(&counter->event_entry);
252
253 if (counter->group_leader != counter)
254 counter->group_leader->nr_siblings--;
255
256 /*
257 * If this was a group counter with sibling counters then
258 * upgrade the siblings to singleton counters by adding them
259 * to the context list directly:
260 */
261 list_for_each_entry_safe(sibling, tmp,
262 &counter->sibling_list, list_entry) {
263
264 list_move_tail(&sibling->list_entry, &ctx->counter_list);
265 sibling->group_leader = sibling;
266 }
267}
268
269static void
270counter_sched_out(struct perf_counter *counter,
271 struct perf_cpu_context *cpuctx,
272 struct perf_counter_context *ctx)
273{
274 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
275 return;
276
277 counter->state = PERF_COUNTER_STATE_INACTIVE;
278 counter->tstamp_stopped = ctx->time;
279 counter->pmu->disable(counter);
280 counter->oncpu = -1;
281
282 if (!is_software_counter(counter))
283 cpuctx->active_oncpu--;
284 ctx->nr_active--;
285 if (counter->attr.exclusive || !cpuctx->active_oncpu)
286 cpuctx->exclusive = 0;
287}
288
289static void
290group_sched_out(struct perf_counter *group_counter,
291 struct perf_cpu_context *cpuctx,
292 struct perf_counter_context *ctx)
293{
294 struct perf_counter *counter;
295
296 if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
297 return;
298
299 counter_sched_out(group_counter, cpuctx, ctx);
300
301 /*
302 * Schedule out siblings (if any):
303 */
304 list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
305 counter_sched_out(counter, cpuctx, ctx);
306
307 if (group_counter->attr.exclusive)
308 cpuctx->exclusive = 0;
309}
310
311/*
312 * Cross CPU call to remove a performance counter
313 *
314 * We disable the counter on the hardware level first. After that we
315 * remove it from the context list.
316 */
317static void __perf_counter_remove_from_context(void *info)
318{
319 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
320 struct perf_counter *counter = info;
321 struct perf_counter_context *ctx = counter->ctx;
322
323 /*
324 * If this is a task context, we need to check whether it is
325 * the current task context of this cpu. If not it has been
326 * scheduled out before the smp call arrived.
327 */
328 if (ctx->task && cpuctx->task_ctx != ctx)
329 return;
330
331 spin_lock(&ctx->lock);
332 /*
333 * Protect the list operation against NMI by disabling the
334 * counters on a global level.
335 */
336 perf_disable();
337
338 counter_sched_out(counter, cpuctx, ctx);
339
340 list_del_counter(counter, ctx);
341
342 if (!ctx->task) {
343 /*
344 * Allow more per task counters with respect to the
345 * reservation:
346 */
347 cpuctx->max_pertask =
348 min(perf_max_counters - ctx->nr_counters,
349 perf_max_counters - perf_reserved_percpu);
350 }
351
352 perf_enable();
353 spin_unlock(&ctx->lock);
354}
355
356
357/*
358 * Remove the counter from a task's (or a CPU's) list of counters.
359 *
360 * Must be called with ctx->mutex held.
361 *
362 * CPU counters are removed with a smp call. For task counters we only
363 * call when the task is on a CPU.
364 *
365 * If counter->ctx is a cloned context, callers must make sure that
366 * every task struct that counter->ctx->task could possibly point to
367 * remains valid. This is OK when called from perf_release since
368 * that only calls us on the top-level context, which can't be a clone.
369 * When called from perf_counter_exit_task, it's OK because the
370 * context has been detached from its task.
371 */
372static void perf_counter_remove_from_context(struct perf_counter *counter)
373{
374 struct perf_counter_context *ctx = counter->ctx;
375 struct task_struct *task = ctx->task;
376
377 if (!task) {
378 /*
379 * Per cpu counters are removed via an smp call and
380 * the removal is always sucessful.
381 */
382 smp_call_function_single(counter->cpu,
383 __perf_counter_remove_from_context,
384 counter, 1);
385 return;
386 }
387
388retry:
389 task_oncpu_function_call(task, __perf_counter_remove_from_context,
390 counter);
391
392 spin_lock_irq(&ctx->lock);
393 /*
394 * If the context is active we need to retry the smp call.
395 */
396 if (ctx->nr_active && !list_empty(&counter->list_entry)) {
397 spin_unlock_irq(&ctx->lock);
398 goto retry;
399 }
400
401 /*
402 * The lock prevents that this context is scheduled in so we
403 * can remove the counter safely, if the call above did not
404 * succeed.
405 */
406 if (!list_empty(&counter->list_entry)) {
407 list_del_counter(counter, ctx);
408 }
409 spin_unlock_irq(&ctx->lock);
410}
411
412static inline u64 perf_clock(void)
413{
414 return cpu_clock(smp_processor_id());
415}
416
417/*
418 * Update the record of the current time in a context.
419 */
420static void update_context_time(struct perf_counter_context *ctx)
421{
422 u64 now = perf_clock();
423
424 ctx->time += now - ctx->timestamp;
425 ctx->timestamp = now;
426}
427
428/*
429 * Update the total_time_enabled and total_time_running fields for a counter.
430 */
431static void update_counter_times(struct perf_counter *counter)
432{
433 struct perf_counter_context *ctx = counter->ctx;
434 u64 run_end;
435
436 if (counter->state < PERF_COUNTER_STATE_INACTIVE)
437 return;
438
439 counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
440
441 if (counter->state == PERF_COUNTER_STATE_INACTIVE)
442 run_end = counter->tstamp_stopped;
443 else
444 run_end = ctx->time;
445
446 counter->total_time_running = run_end - counter->tstamp_running;
447}
448
449/*
450 * Update total_time_enabled and total_time_running for all counters in a group.
451 */
452static void update_group_times(struct perf_counter *leader)
453{
454 struct perf_counter *counter;
455
456 update_counter_times(leader);
457 list_for_each_entry(counter, &leader->sibling_list, list_entry)
458 update_counter_times(counter);
459}
460
461/*
462 * Cross CPU call to disable a performance counter
463 */
464static void __perf_counter_disable(void *info)
465{
466 struct perf_counter *counter = info;
467 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
468 struct perf_counter_context *ctx = counter->ctx;
469
470 /*
471 * If this is a per-task counter, need to check whether this
472 * counter's task is the current task on this cpu.
473 */
474 if (ctx->task && cpuctx->task_ctx != ctx)
475 return;
476
477 spin_lock(&ctx->lock);
478
479 /*
480 * If the counter is on, turn it off.
481 * If it is in error state, leave it in error state.
482 */
483 if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
484 update_context_time(ctx);
485 update_counter_times(counter);
486 if (counter == counter->group_leader)
487 group_sched_out(counter, cpuctx, ctx);
488 else
489 counter_sched_out(counter, cpuctx, ctx);
490 counter->state = PERF_COUNTER_STATE_OFF;
491 }
492
493 spin_unlock(&ctx->lock);
494}
495
496/*
497 * Disable a counter.
498 *
499 * If counter->ctx is a cloned context, callers must make sure that
500 * every task struct that counter->ctx->task could possibly point to
501 * remains valid. This condition is satisifed when called through
502 * perf_counter_for_each_child or perf_counter_for_each because they
503 * hold the top-level counter's child_mutex, so any descendant that
504 * goes to exit will block in sync_child_counter.
505 * When called from perf_pending_counter it's OK because counter->ctx
506 * is the current context on this CPU and preemption is disabled,
507 * hence we can't get into perf_counter_task_sched_out for this context.
508 */
509static void perf_counter_disable(struct perf_counter *counter)
510{
511 struct perf_counter_context *ctx = counter->ctx;
512 struct task_struct *task = ctx->task;
513
514 if (!task) {
515 /*
516 * Disable the counter on the cpu that it's on
517 */
518 smp_call_function_single(counter->cpu, __perf_counter_disable,
519 counter, 1);
520 return;
521 }
522
523 retry:
524 task_oncpu_function_call(task, __perf_counter_disable, counter);
525
526 spin_lock_irq(&ctx->lock);
527 /*
528 * If the counter is still active, we need to retry the cross-call.
529 */
530 if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
531 spin_unlock_irq(&ctx->lock);
532 goto retry;
533 }
534
535 /*
536 * Since we have the lock this context can't be scheduled
537 * in, so we can change the state safely.
538 */
539 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
540 update_counter_times(counter);
541 counter->state = PERF_COUNTER_STATE_OFF;
542 }
543
544 spin_unlock_irq(&ctx->lock);
545}
546
547static int
548counter_sched_in(struct perf_counter *counter,
549 struct perf_cpu_context *cpuctx,
550 struct perf_counter_context *ctx,
551 int cpu)
552{
553 if (counter->state <= PERF_COUNTER_STATE_OFF)
554 return 0;
555
556 counter->state = PERF_COUNTER_STATE_ACTIVE;
557 counter->oncpu = cpu; /* TODO: put 'cpu' into cpuctx->cpu */
558 /*
559 * The new state must be visible before we turn it on in the hardware:
560 */
561 smp_wmb();
562
563 if (counter->pmu->enable(counter)) {
564 counter->state = PERF_COUNTER_STATE_INACTIVE;
565 counter->oncpu = -1;
566 return -EAGAIN;
567 }
568
569 counter->tstamp_running += ctx->time - counter->tstamp_stopped;
570
571 if (!is_software_counter(counter))
572 cpuctx->active_oncpu++;
573 ctx->nr_active++;
574
575 if (counter->attr.exclusive)
576 cpuctx->exclusive = 1;
577
578 return 0;
579}
580
581static int
582group_sched_in(struct perf_counter *group_counter,
583 struct perf_cpu_context *cpuctx,
584 struct perf_counter_context *ctx,
585 int cpu)
586{
587 struct perf_counter *counter, *partial_group;
588 int ret;
589
590 if (group_counter->state == PERF_COUNTER_STATE_OFF)
591 return 0;
592
593 ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
594 if (ret)
595 return ret < 0 ? ret : 0;
596
597 if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
598 return -EAGAIN;
599
600 /*
601 * Schedule in siblings as one group (if any):
602 */
603 list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
604 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
605 partial_group = counter;
606 goto group_error;
607 }
608 }
609
610 return 0;
611
612group_error:
613 /*
614 * Groups can be scheduled in as one unit only, so undo any
615 * partial group before returning:
616 */
617 list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
618 if (counter == partial_group)
619 break;
620 counter_sched_out(counter, cpuctx, ctx);
621 }
622 counter_sched_out(group_counter, cpuctx, ctx);
623
624 return -EAGAIN;
625}
626
627/*
628 * Return 1 for a group consisting entirely of software counters,
629 * 0 if the group contains any hardware counters.
630 */
631static int is_software_only_group(struct perf_counter *leader)
632{
633 struct perf_counter *counter;
634
635 if (!is_software_counter(leader))
636 return 0;
637
638 list_for_each_entry(counter, &leader->sibling_list, list_entry)
639 if (!is_software_counter(counter))
640 return 0;
641
642 return 1;
643}
644
645/*
646 * Work out whether we can put this counter group on the CPU now.
647 */
648static int group_can_go_on(struct perf_counter *counter,
649 struct perf_cpu_context *cpuctx,
650 int can_add_hw)
651{
652 /*
653 * Groups consisting entirely of software counters can always go on.
654 */
655 if (is_software_only_group(counter))
656 return 1;
657 /*
658 * If an exclusive group is already on, no other hardware
659 * counters can go on.
660 */
661 if (cpuctx->exclusive)
662 return 0;
663 /*
664 * If this group is exclusive and there are already
665 * counters on the CPU, it can't go on.
666 */
667 if (counter->attr.exclusive && cpuctx->active_oncpu)
668 return 0;
669 /*
670 * Otherwise, try to add it if all previous groups were able
671 * to go on.
672 */
673 return can_add_hw;
674}
675
676static void add_counter_to_ctx(struct perf_counter *counter,
677 struct perf_counter_context *ctx)
678{
679 list_add_counter(counter, ctx);
680 counter->tstamp_enabled = ctx->time;
681 counter->tstamp_running = ctx->time;
682 counter->tstamp_stopped = ctx->time;
683}
684
685/*
686 * Cross CPU call to install and enable a performance counter
687 *
688 * Must be called with ctx->mutex held
689 */
690static void __perf_install_in_context(void *info)
691{
692 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
693 struct perf_counter *counter = info;
694 struct perf_counter_context *ctx = counter->ctx;
695 struct perf_counter *leader = counter->group_leader;
696 int cpu = smp_processor_id();
697 int err;
698
699 /*
700 * If this is a task context, we need to check whether it is
701 * the current task context of this cpu. If not it has been
702 * scheduled out before the smp call arrived.
703 * Or possibly this is the right context but it isn't
704 * on this cpu because it had no counters.
705 */
706 if (ctx->task && cpuctx->task_ctx != ctx) {
707 if (cpuctx->task_ctx || ctx->task != current)
708 return;
709 cpuctx->task_ctx = ctx;
710 }
711
712 spin_lock(&ctx->lock);
713 ctx->is_active = 1;
714 update_context_time(ctx);
715
716 /*
717 * Protect the list operation against NMI by disabling the
718 * counters on a global level. NOP for non NMI based counters.
719 */
720 perf_disable();
721
722 add_counter_to_ctx(counter, ctx);
723
724 /*
725 * Don't put the counter on if it is disabled or if
726 * it is in a group and the group isn't on.
727 */
728 if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
729 (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
730 goto unlock;
731
732 /*
733 * An exclusive counter can't go on if there are already active
734 * hardware counters, and no hardware counter can go on if there
735 * is already an exclusive counter on.
736 */
737 if (!group_can_go_on(counter, cpuctx, 1))
738 err = -EEXIST;
739 else
740 err = counter_sched_in(counter, cpuctx, ctx, cpu);
741
742 if (err) {
743 /*
744 * This counter couldn't go on. If it is in a group
745 * then we have to pull the whole group off.
746 * If the counter group is pinned then put it in error state.
747 */
748 if (leader != counter)
749 group_sched_out(leader, cpuctx, ctx);
750 if (leader->attr.pinned) {
751 update_group_times(leader);
752 leader->state = PERF_COUNTER_STATE_ERROR;
753 }
754 }
755
756 if (!err && !ctx->task && cpuctx->max_pertask)
757 cpuctx->max_pertask--;
758
759 unlock:
760 perf_enable();
761
762 spin_unlock(&ctx->lock);
763}
764
765/*
766 * Attach a performance counter to a context
767 *
768 * First we add the counter to the list with the hardware enable bit
769 * in counter->hw_config cleared.
770 *
771 * If the counter is attached to a task which is on a CPU we use a smp
772 * call to enable it in the task context. The task might have been
773 * scheduled away, but we check this in the smp call again.
774 *
775 * Must be called with ctx->mutex held.
776 */
777static void
778perf_install_in_context(struct perf_counter_context *ctx,
779 struct perf_counter *counter,
780 int cpu)
781{
782 struct task_struct *task = ctx->task;
783
784 if (!task) {
785 /*
786 * Per cpu counters are installed via an smp call and
787 * the install is always sucessful.
788 */
789 smp_call_function_single(cpu, __perf_install_in_context,
790 counter, 1);
791 return;
792 }
793
794retry:
795 task_oncpu_function_call(task, __perf_install_in_context,
796 counter);
797
798 spin_lock_irq(&ctx->lock);
799 /*
800 * we need to retry the smp call.
801 */
802 if (ctx->is_active && list_empty(&counter->list_entry)) {
803 spin_unlock_irq(&ctx->lock);
804 goto retry;
805 }
806
807 /*
808 * The lock prevents that this context is scheduled in so we
809 * can add the counter safely, if it the call above did not
810 * succeed.
811 */
812 if (list_empty(&counter->list_entry))
813 add_counter_to_ctx(counter, ctx);
814 spin_unlock_irq(&ctx->lock);
815}
816
817/*
818 * Cross CPU call to enable a performance counter
819 */
820static void __perf_counter_enable(void *info)
821{
822 struct perf_counter *counter = info;
823 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
824 struct perf_counter_context *ctx = counter->ctx;
825 struct perf_counter *leader = counter->group_leader;
826 int err;
827
828 /*
829 * If this is a per-task counter, need to check whether this
830 * counter's task is the current task on this cpu.
831 */
832 if (ctx->task && cpuctx->task_ctx != ctx) {
833 if (cpuctx->task_ctx || ctx->task != current)
834 return;
835 cpuctx->task_ctx = ctx;
836 }
837
838 spin_lock(&ctx->lock);
839 ctx->is_active = 1;
840 update_context_time(ctx);
841
842 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
843 goto unlock;
844 counter->state = PERF_COUNTER_STATE_INACTIVE;
845 counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
846
847 /*
848 * If the counter is in a group and isn't the group leader,
849 * then don't put it on unless the group is on.
850 */
851 if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
852 goto unlock;
853
854 if (!group_can_go_on(counter, cpuctx, 1)) {
855 err = -EEXIST;
856 } else {
857 perf_disable();
858 if (counter == leader)
859 err = group_sched_in(counter, cpuctx, ctx,
860 smp_processor_id());
861 else
862 err = counter_sched_in(counter, cpuctx, ctx,
863 smp_processor_id());
864 perf_enable();
865 }
866
867 if (err) {
868 /*
869 * If this counter can't go on and it's part of a
870 * group, then the whole group has to come off.
871 */
872 if (leader != counter)
873 group_sched_out(leader, cpuctx, ctx);
874 if (leader->attr.pinned) {
875 update_group_times(leader);
876 leader->state = PERF_COUNTER_STATE_ERROR;
877 }
878 }
879
880 unlock:
881 spin_unlock(&ctx->lock);
882}
883
884/*
885 * Enable a counter.
886 *
887 * If counter->ctx is a cloned context, callers must make sure that
888 * every task struct that counter->ctx->task could possibly point to
889 * remains valid. This condition is satisfied when called through
890 * perf_counter_for_each_child or perf_counter_for_each as described
891 * for perf_counter_disable.
892 */
893static void perf_counter_enable(struct perf_counter *counter)
894{
895 struct perf_counter_context *ctx = counter->ctx;
896 struct task_struct *task = ctx->task;
897
898 if (!task) {
899 /*
900 * Enable the counter on the cpu that it's on
901 */
902 smp_call_function_single(counter->cpu, __perf_counter_enable,
903 counter, 1);
904 return;
905 }
906
907 spin_lock_irq(&ctx->lock);
908 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
909 goto out;
910
911 /*
912 * If the counter is in error state, clear that first.
913 * That way, if we see the counter in error state below, we
914 * know that it has gone back into error state, as distinct
915 * from the task having been scheduled away before the
916 * cross-call arrived.
917 */
918 if (counter->state == PERF_COUNTER_STATE_ERROR)
919 counter->state = PERF_COUNTER_STATE_OFF;
920
921 retry:
922 spin_unlock_irq(&ctx->lock);
923 task_oncpu_function_call(task, __perf_counter_enable, counter);
924
925 spin_lock_irq(&ctx->lock);
926
927 /*
928 * If the context is active and the counter is still off,
929 * we need to retry the cross-call.
930 */
931 if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
932 goto retry;
933
934 /*
935 * Since we have the lock this context can't be scheduled
936 * in, so we can change the state safely.
937 */
938 if (counter->state == PERF_COUNTER_STATE_OFF) {
939 counter->state = PERF_COUNTER_STATE_INACTIVE;
940 counter->tstamp_enabled =
941 ctx->time - counter->total_time_enabled;
942 }
943 out:
944 spin_unlock_irq(&ctx->lock);
945}
946
947static int perf_counter_refresh(struct perf_counter *counter, int refresh)
948{
949 /*
950 * not supported on inherited counters
951 */
952 if (counter->attr.inherit)
953 return -EINVAL;
954
955 atomic_add(refresh, &counter->event_limit);
956 perf_counter_enable(counter);
957
958 return 0;
959}
960
961void __perf_counter_sched_out(struct perf_counter_context *ctx,
962 struct perf_cpu_context *cpuctx)
963{
964 struct perf_counter *counter;
965
966 spin_lock(&ctx->lock);
967 ctx->is_active = 0;
968 if (likely(!ctx->nr_counters))
969 goto out;
970 update_context_time(ctx);
971
972 perf_disable();
973 if (ctx->nr_active) {
974 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
975 if (counter != counter->group_leader)
976 counter_sched_out(counter, cpuctx, ctx);
977 else
978 group_sched_out(counter, cpuctx, ctx);
979 }
980 }
981 perf_enable();
982 out:
983 spin_unlock(&ctx->lock);
984}
985
986/*
987 * Test whether two contexts are equivalent, i.e. whether they
988 * have both been cloned from the same version of the same context
989 * and they both have the same number of enabled counters.
990 * If the number of enabled counters is the same, then the set
991 * of enabled counters should be the same, because these are both
992 * inherited contexts, therefore we can't access individual counters
993 * in them directly with an fd; we can only enable/disable all
994 * counters via prctl, or enable/disable all counters in a family
995 * via ioctl, which will have the same effect on both contexts.
996 */
997static int context_equiv(struct perf_counter_context *ctx1,
998 struct perf_counter_context *ctx2)
999{
1000 return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1001 && ctx1->parent_gen == ctx2->parent_gen
1002 && !ctx1->pin_count && !ctx2->pin_count;
1003}
1004
1005/*
1006 * Called from scheduler to remove the counters of the current task,
1007 * with interrupts disabled.
1008 *
1009 * We stop each counter and update the counter value in counter->count.
1010 *
1011 * This does not protect us against NMI, but disable()
1012 * sets the disabled bit in the control field of counter _before_
1013 * accessing the counter control register. If a NMI hits, then it will
1014 * not restart the counter.
1015 */
1016void perf_counter_task_sched_out(struct task_struct *task,
1017 struct task_struct *next, int cpu)
1018{
1019 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1020 struct perf_counter_context *ctx = task->perf_counter_ctxp;
1021 struct perf_counter_context *next_ctx;
1022 struct perf_counter_context *parent;
1023 struct pt_regs *regs;
1024 int do_switch = 1;
1025
1026 regs = task_pt_regs(task);
1027 perf_swcounter_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, regs, 0);
1028
1029 if (likely(!ctx || !cpuctx->task_ctx))
1030 return;
1031
1032 update_context_time(ctx);
1033
1034 rcu_read_lock();
1035 parent = rcu_dereference(ctx->parent_ctx);
1036 next_ctx = next->perf_counter_ctxp;
1037 if (parent && next_ctx &&
1038 rcu_dereference(next_ctx->parent_ctx) == parent) {
1039 /*
1040 * Looks like the two contexts are clones, so we might be
1041 * able to optimize the context switch. We lock both
1042 * contexts and check that they are clones under the
1043 * lock (including re-checking that neither has been
1044 * uncloned in the meantime). It doesn't matter which
1045 * order we take the locks because no other cpu could
1046 * be trying to lock both of these tasks.
1047 */
1048 spin_lock(&ctx->lock);
1049 spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1050 if (context_equiv(ctx, next_ctx)) {
1051 /*
1052 * XXX do we need a memory barrier of sorts
1053 * wrt to rcu_dereference() of perf_counter_ctxp
1054 */
1055 task->perf_counter_ctxp = next_ctx;
1056 next->perf_counter_ctxp = ctx;
1057 ctx->task = next;
1058 next_ctx->task = task;
1059 do_switch = 0;
1060 }
1061 spin_unlock(&next_ctx->lock);
1062 spin_unlock(&ctx->lock);
1063 }
1064 rcu_read_unlock();
1065
1066 if (do_switch) {
1067 __perf_counter_sched_out(ctx, cpuctx);
1068 cpuctx->task_ctx = NULL;
1069 }
1070}
1071
1072/*
1073 * Called with IRQs disabled
1074 */
1075static void __perf_counter_task_sched_out(struct perf_counter_context *ctx)
1076{
1077 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1078
1079 if (!cpuctx->task_ctx)
1080 return;
1081
1082 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
1083 return;
1084
1085 __perf_counter_sched_out(ctx, cpuctx);
1086 cpuctx->task_ctx = NULL;
1087}
1088
1089/*
1090 * Called with IRQs disabled
1091 */
1092static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
1093{
1094 __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
1095}
1096
1097static void
1098__perf_counter_sched_in(struct perf_counter_context *ctx,
1099 struct perf_cpu_context *cpuctx, int cpu)
1100{
1101 struct perf_counter *counter;
1102 int can_add_hw = 1;
1103
1104 spin_lock(&ctx->lock);
1105 ctx->is_active = 1;
1106 if (likely(!ctx->nr_counters))
1107 goto out;
1108
1109 ctx->timestamp = perf_clock();
1110
1111 perf_disable();
1112
1113 /*
1114 * First go through the list and put on any pinned groups
1115 * in order to give them the best chance of going on.
1116 */
1117 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1118 if (counter->state <= PERF_COUNTER_STATE_OFF ||
1119 !counter->attr.pinned)
1120 continue;
1121 if (counter->cpu != -1 && counter->cpu != cpu)
1122 continue;
1123
1124 if (counter != counter->group_leader)
1125 counter_sched_in(counter, cpuctx, ctx, cpu);
1126 else {
1127 if (group_can_go_on(counter, cpuctx, 1))
1128 group_sched_in(counter, cpuctx, ctx, cpu);
1129 }
1130
1131 /*
1132 * If this pinned group hasn't been scheduled,
1133 * put it in error state.
1134 */
1135 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1136 update_group_times(counter);
1137 counter->state = PERF_COUNTER_STATE_ERROR;
1138 }
1139 }
1140
1141 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1142 /*
1143 * Ignore counters in OFF or ERROR state, and
1144 * ignore pinned counters since we did them already.
1145 */
1146 if (counter->state <= PERF_COUNTER_STATE_OFF ||
1147 counter->attr.pinned)
1148 continue;
1149
1150 /*
1151 * Listen to the 'cpu' scheduling filter constraint
1152 * of counters:
1153 */
1154 if (counter->cpu != -1 && counter->cpu != cpu)
1155 continue;
1156
1157 if (counter != counter->group_leader) {
1158 if (counter_sched_in(counter, cpuctx, ctx, cpu))
1159 can_add_hw = 0;
1160 } else {
1161 if (group_can_go_on(counter, cpuctx, can_add_hw)) {
1162 if (group_sched_in(counter, cpuctx, ctx, cpu))
1163 can_add_hw = 0;
1164 }
1165 }
1166 }
1167 perf_enable();
1168 out:
1169 spin_unlock(&ctx->lock);
1170}
1171
1172/*
1173 * Called from scheduler to add the counters of the current task
1174 * with interrupts disabled.
1175 *
1176 * We restore the counter value and then enable it.
1177 *
1178 * This does not protect us against NMI, but enable()
1179 * sets the enabled bit in the control field of counter _before_
1180 * accessing the counter control register. If a NMI hits, then it will
1181 * keep the counter running.
1182 */
1183void perf_counter_task_sched_in(struct task_struct *task, int cpu)
1184{
1185 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1186 struct perf_counter_context *ctx = task->perf_counter_ctxp;
1187
1188 if (likely(!ctx))
1189 return;
1190 if (cpuctx->task_ctx == ctx)
1191 return;
1192 __perf_counter_sched_in(ctx, cpuctx, cpu);
1193 cpuctx->task_ctx = ctx;
1194}
1195
1196static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
1197{
1198 struct perf_counter_context *ctx = &cpuctx->ctx;
1199
1200 __perf_counter_sched_in(ctx, cpuctx, cpu);
1201}
1202
1203#define MAX_INTERRUPTS (~0ULL)
1204
1205static void perf_log_throttle(struct perf_counter *counter, int enable);
1206static void perf_log_period(struct perf_counter *counter, u64 period);
1207
1208static void perf_adjust_period(struct perf_counter *counter, u64 events)
1209{
1210 struct hw_perf_counter *hwc = &counter->hw;
1211 u64 period, sample_period;
1212 s64 delta;
1213
1214 events *= hwc->sample_period;
1215 period = div64_u64(events, counter->attr.sample_freq);
1216
1217 delta = (s64)(period - hwc->sample_period);
1218 delta = (delta + 7) / 8; /* low pass filter */
1219
1220 sample_period = hwc->sample_period + delta;
1221
1222 if (!sample_period)
1223 sample_period = 1;
1224
1225 perf_log_period(counter, sample_period);
1226
1227 hwc->sample_period = sample_period;
1228}
1229
1230static void perf_ctx_adjust_freq(struct perf_counter_context *ctx)
1231{
1232 struct perf_counter *counter;
1233 struct hw_perf_counter *hwc;
1234 u64 interrupts, freq;
1235
1236 spin_lock(&ctx->lock);
1237 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1238 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
1239 continue;
1240
1241 hwc = &counter->hw;
1242
1243 interrupts = hwc->interrupts;
1244 hwc->interrupts = 0;
1245
1246 /*
1247 * unthrottle counters on the tick
1248 */
1249 if (interrupts == MAX_INTERRUPTS) {
1250 perf_log_throttle(counter, 1);
1251 counter->pmu->unthrottle(counter);
1252 interrupts = 2*sysctl_perf_counter_sample_rate/HZ;
1253 }
1254
1255 if (!counter->attr.freq || !counter->attr.sample_freq)
1256 continue;
1257
1258 /*
1259 * if the specified freq < HZ then we need to skip ticks
1260 */
1261 if (counter->attr.sample_freq < HZ) {
1262 freq = counter->attr.sample_freq;
1263
1264 hwc->freq_count += freq;
1265 hwc->freq_interrupts += interrupts;
1266
1267 if (hwc->freq_count < HZ)
1268 continue;
1269
1270 interrupts = hwc->freq_interrupts;
1271 hwc->freq_interrupts = 0;
1272 hwc->freq_count -= HZ;
1273 } else
1274 freq = HZ;
1275
1276 perf_adjust_period(counter, freq * interrupts);
1277
1278 /*
1279 * In order to avoid being stalled by an (accidental) huge
1280 * sample period, force reset the sample period if we didn't
1281 * get any events in this freq period.
1282 */
1283 if (!interrupts) {
1284 perf_disable();
1285 counter->pmu->disable(counter);
1286 atomic_set(&hwc->period_left, 0);
1287 counter->pmu->enable(counter);
1288 perf_enable();
1289 }
1290 }
1291 spin_unlock(&ctx->lock);
1292}
1293
1294/*
1295 * Round-robin a context's counters:
1296 */
1297static void rotate_ctx(struct perf_counter_context *ctx)
1298{
1299 struct perf_counter *counter;
1300
1301 if (!ctx->nr_counters)
1302 return;
1303
1304 spin_lock(&ctx->lock);
1305 /*
1306 * Rotate the first entry last (works just fine for group counters too):
1307 */
1308 perf_disable();
1309 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1310 list_move_tail(&counter->list_entry, &ctx->counter_list);
1311 break;
1312 }
1313 perf_enable();
1314
1315 spin_unlock(&ctx->lock);
1316}
1317
1318void perf_counter_task_tick(struct task_struct *curr, int cpu)
1319{
1320 struct perf_cpu_context *cpuctx;
1321 struct perf_counter_context *ctx;
1322
1323 if (!atomic_read(&nr_counters))
1324 return;
1325
1326 cpuctx = &per_cpu(perf_cpu_context, cpu);
1327 ctx = curr->perf_counter_ctxp;
1328
1329 perf_ctx_adjust_freq(&cpuctx->ctx);
1330 if (ctx)
1331 perf_ctx_adjust_freq(ctx);
1332
1333 perf_counter_cpu_sched_out(cpuctx);
1334 if (ctx)
1335 __perf_counter_task_sched_out(ctx);
1336
1337 rotate_ctx(&cpuctx->ctx);
1338 if (ctx)
1339 rotate_ctx(ctx);
1340
1341 perf_counter_cpu_sched_in(cpuctx, cpu);
1342 if (ctx)
1343 perf_counter_task_sched_in(curr, cpu);
1344}
1345
1346/*
1347 * Cross CPU call to read the hardware counter
1348 */
1349static void __read(void *info)
1350{
1351 struct perf_counter *counter = info;
1352 struct perf_counter_context *ctx = counter->ctx;
1353 unsigned long flags;
1354
1355 local_irq_save(flags);
1356 if (ctx->is_active)
1357 update_context_time(ctx);
1358 counter->pmu->read(counter);
1359 update_counter_times(counter);
1360 local_irq_restore(flags);
1361}
1362
1363static u64 perf_counter_read(struct perf_counter *counter)
1364{
1365 /*
1366 * If counter is enabled and currently active on a CPU, update the
1367 * value in the counter structure:
1368 */
1369 if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1370 smp_call_function_single(counter->oncpu,
1371 __read, counter, 1);
1372 } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1373 update_counter_times(counter);
1374 }
1375
1376 return atomic64_read(&counter->count);
1377}
1378
1379/*
1380 * Initialize the perf_counter context in a task_struct:
1381 */
1382static void
1383__perf_counter_init_context(struct perf_counter_context *ctx,
1384 struct task_struct *task)
1385{
1386 memset(ctx, 0, sizeof(*ctx));
1387 spin_lock_init(&ctx->lock);
1388 mutex_init(&ctx->mutex);
1389 INIT_LIST_HEAD(&ctx->counter_list);
1390 INIT_LIST_HEAD(&ctx->event_list);
1391 atomic_set(&ctx->refcount, 1);
1392 ctx->task = task;
1393}
1394
1395static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1396{
1397 struct perf_counter_context *parent_ctx;
1398 struct perf_counter_context *ctx;
1399 struct perf_cpu_context *cpuctx;
1400 struct task_struct *task;
1401 unsigned long flags;
1402 int err;
1403
1404 /*
1405 * If cpu is not a wildcard then this is a percpu counter:
1406 */
1407 if (cpu != -1) {
1408 /* Must be root to operate on a CPU counter: */
1409 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
1410 return ERR_PTR(-EACCES);
1411
1412 if (cpu < 0 || cpu > num_possible_cpus())
1413 return ERR_PTR(-EINVAL);
1414
1415 /*
1416 * We could be clever and allow to attach a counter to an
1417 * offline CPU and activate it when the CPU comes up, but
1418 * that's for later.
1419 */
1420 if (!cpu_isset(cpu, cpu_online_map))
1421 return ERR_PTR(-ENODEV);
1422
1423 cpuctx = &per_cpu(perf_cpu_context, cpu);
1424 ctx = &cpuctx->ctx;
1425 get_ctx(ctx);
1426
1427 return ctx;
1428 }
1429
1430 rcu_read_lock();
1431 if (!pid)
1432 task = current;
1433 else
1434 task = find_task_by_vpid(pid);
1435 if (task)
1436 get_task_struct(task);
1437 rcu_read_unlock();
1438
1439 if (!task)
1440 return ERR_PTR(-ESRCH);
1441
1442 /*
1443 * Can't attach counters to a dying task.
1444 */
1445 err = -ESRCH;
1446 if (task->flags & PF_EXITING)
1447 goto errout;
1448
1449 /* Reuse ptrace permission checks for now. */
1450 err = -EACCES;
1451 if (!ptrace_may_access(task, PTRACE_MODE_READ))
1452 goto errout;
1453
1454 retry:
1455 ctx = perf_lock_task_context(task, &flags);
1456 if (ctx) {
1457 parent_ctx = ctx->parent_ctx;
1458 if (parent_ctx) {
1459 put_ctx(parent_ctx);
1460 ctx->parent_ctx = NULL; /* no longer a clone */
1461 }
1462 /*
1463 * Get an extra reference before dropping the lock so that
1464 * this context won't get freed if the task exits.
1465 */
1466 get_ctx(ctx);
1467 spin_unlock_irqrestore(&ctx->lock, flags);
1468 }
1469
1470 if (!ctx) {
1471 ctx = kmalloc(sizeof(struct perf_counter_context), GFP_KERNEL);
1472 err = -ENOMEM;
1473 if (!ctx)
1474 goto errout;
1475 __perf_counter_init_context(ctx, task);
1476 get_ctx(ctx);
1477 if (cmpxchg(&task->perf_counter_ctxp, NULL, ctx)) {
1478 /*
1479 * We raced with some other task; use
1480 * the context they set.
1481 */
1482 kfree(ctx);
1483 goto retry;
1484 }
1485 get_task_struct(task);
1486 }
1487
1488 put_task_struct(task);
1489 return ctx;
1490
1491 errout:
1492 put_task_struct(task);
1493 return ERR_PTR(err);
1494}
1495
1496static void free_counter_rcu(struct rcu_head *head)
1497{
1498 struct perf_counter *counter;
1499
1500 counter = container_of(head, struct perf_counter, rcu_head);
1501 if (counter->ns)
1502 put_pid_ns(counter->ns);
1503 kfree(counter);
1504}
1505
1506static void perf_pending_sync(struct perf_counter *counter);
1507
1508static void free_counter(struct perf_counter *counter)
1509{
1510 perf_pending_sync(counter);
1511
1512 atomic_dec(&nr_counters);
1513 if (counter->attr.mmap)
1514 atomic_dec(&nr_mmap_counters);
1515 if (counter->attr.comm)
1516 atomic_dec(&nr_comm_counters);
1517
1518 if (counter->destroy)
1519 counter->destroy(counter);
1520
1521 put_ctx(counter->ctx);
1522 call_rcu(&counter->rcu_head, free_counter_rcu);
1523}
1524
1525/*
1526 * Called when the last reference to the file is gone.
1527 */
1528static int perf_release(struct inode *inode, struct file *file)
1529{
1530 struct perf_counter *counter = file->private_data;
1531 struct perf_counter_context *ctx = counter->ctx;
1532
1533 file->private_data = NULL;
1534
1535 WARN_ON_ONCE(ctx->parent_ctx);
1536 mutex_lock(&ctx->mutex);
1537 perf_counter_remove_from_context(counter);
1538 mutex_unlock(&ctx->mutex);
1539
1540 mutex_lock(&counter->owner->perf_counter_mutex);
1541 list_del_init(&counter->owner_entry);
1542 mutex_unlock(&counter->owner->perf_counter_mutex);
1543 put_task_struct(counter->owner);
1544
1545 free_counter(counter);
1546
1547 return 0;
1548}
1549
1550/*
1551 * Read the performance counter - simple non blocking version for now
1552 */
1553static ssize_t
1554perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1555{
1556 u64 values[3];
1557 int n;
1558
1559 /*
1560 * Return end-of-file for a read on a counter that is in
1561 * error state (i.e. because it was pinned but it couldn't be
1562 * scheduled on to the CPU at some point).
1563 */
1564 if (counter->state == PERF_COUNTER_STATE_ERROR)
1565 return 0;
1566
1567 WARN_ON_ONCE(counter->ctx->parent_ctx);
1568 mutex_lock(&counter->child_mutex);
1569 values[0] = perf_counter_read(counter);
1570 n = 1;
1571 if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1572 values[n++] = counter->total_time_enabled +
1573 atomic64_read(&counter->child_total_time_enabled);
1574 if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1575 values[n++] = counter->total_time_running +
1576 atomic64_read(&counter->child_total_time_running);
1577 if (counter->attr.read_format & PERF_FORMAT_ID)
1578 values[n++] = counter->id;
1579 mutex_unlock(&counter->child_mutex);
1580
1581 if (count < n * sizeof(u64))
1582 return -EINVAL;
1583 count = n * sizeof(u64);
1584
1585 if (copy_to_user(buf, values, count))
1586 return -EFAULT;
1587
1588 return count;
1589}
1590
1591static ssize_t
1592perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1593{
1594 struct perf_counter *counter = file->private_data;
1595
1596 return perf_read_hw(counter, buf, count);
1597}
1598
1599static unsigned int perf_poll(struct file *file, poll_table *wait)
1600{
1601 struct perf_counter *counter = file->private_data;
1602 struct perf_mmap_data *data;
1603 unsigned int events = POLL_HUP;
1604
1605 rcu_read_lock();
1606 data = rcu_dereference(counter->data);
1607 if (data)
1608 events = atomic_xchg(&data->poll, 0);
1609 rcu_read_unlock();
1610
1611 poll_wait(file, &counter->waitq, wait);
1612
1613 return events;
1614}
1615
1616static void perf_counter_reset(struct perf_counter *counter)
1617{
1618 (void)perf_counter_read(counter);
1619 atomic64_set(&counter->count, 0);
1620 perf_counter_update_userpage(counter);
1621}
1622
1623static void perf_counter_for_each_sibling(struct perf_counter *counter,
1624 void (*func)(struct perf_counter *))
1625{
1626 struct perf_counter_context *ctx = counter->ctx;
1627 struct perf_counter *sibling;
1628
1629 WARN_ON_ONCE(ctx->parent_ctx);
1630 mutex_lock(&ctx->mutex);
1631 counter = counter->group_leader;
1632
1633 func(counter);
1634 list_for_each_entry(sibling, &counter->sibling_list, list_entry)
1635 func(sibling);
1636 mutex_unlock(&ctx->mutex);
1637}
1638
1639/*
1640 * Holding the top-level counter's child_mutex means that any
1641 * descendant process that has inherited this counter will block
1642 * in sync_child_counter if it goes to exit, thus satisfying the
1643 * task existence requirements of perf_counter_enable/disable.
1644 */
1645static void perf_counter_for_each_child(struct perf_counter *counter,
1646 void (*func)(struct perf_counter *))
1647{
1648 struct perf_counter *child;
1649
1650 WARN_ON_ONCE(counter->ctx->parent_ctx);
1651 mutex_lock(&counter->child_mutex);
1652 func(counter);
1653 list_for_each_entry(child, &counter->child_list, child_list)
1654 func(child);
1655 mutex_unlock(&counter->child_mutex);
1656}
1657
1658static void perf_counter_for_each(struct perf_counter *counter,
1659 void (*func)(struct perf_counter *))
1660{
1661 struct perf_counter *child;
1662
1663 WARN_ON_ONCE(counter->ctx->parent_ctx);
1664 mutex_lock(&counter->child_mutex);
1665 perf_counter_for_each_sibling(counter, func);
1666 list_for_each_entry(child, &counter->child_list, child_list)
1667 perf_counter_for_each_sibling(child, func);
1668 mutex_unlock(&counter->child_mutex);
1669}
1670
1671static int perf_counter_period(struct perf_counter *counter, u64 __user *arg)
1672{
1673 struct perf_counter_context *ctx = counter->ctx;
1674 unsigned long size;
1675 int ret = 0;
1676 u64 value;
1677
1678 if (!counter->attr.sample_period)
1679 return -EINVAL;
1680
1681 size = copy_from_user(&value, arg, sizeof(value));
1682 if (size != sizeof(value))
1683 return -EFAULT;
1684
1685 if (!value)
1686 return -EINVAL;
1687
1688 spin_lock_irq(&ctx->lock);
1689 if (counter->attr.freq) {
1690 if (value > sysctl_perf_counter_sample_rate) {
1691 ret = -EINVAL;
1692 goto unlock;
1693 }
1694
1695 counter->attr.sample_freq = value;
1696 } else {
1697 perf_log_period(counter, value);
1698
1699 counter->attr.sample_period = value;
1700 counter->hw.sample_period = value;
1701 }
1702unlock:
1703 spin_unlock_irq(&ctx->lock);
1704
1705 return ret;
1706}
1707
1708static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1709{
1710 struct perf_counter *counter = file->private_data;
1711 void (*func)(struct perf_counter *);
1712 u32 flags = arg;
1713
1714 switch (cmd) {
1715 case PERF_COUNTER_IOC_ENABLE:
1716 func = perf_counter_enable;
1717 break;
1718 case PERF_COUNTER_IOC_DISABLE:
1719 func = perf_counter_disable;
1720 break;
1721 case PERF_COUNTER_IOC_RESET:
1722 func = perf_counter_reset;
1723 break;
1724
1725 case PERF_COUNTER_IOC_REFRESH:
1726 return perf_counter_refresh(counter, arg);
1727
1728 case PERF_COUNTER_IOC_PERIOD:
1729 return perf_counter_period(counter, (u64 __user *)arg);
1730
1731 default:
1732 return -ENOTTY;
1733 }
1734
1735 if (flags & PERF_IOC_FLAG_GROUP)
1736 perf_counter_for_each(counter, func);
1737 else
1738 perf_counter_for_each_child(counter, func);
1739
1740 return 0;
1741}
1742
1743int perf_counter_task_enable(void)
1744{
1745 struct perf_counter *counter;
1746
1747 mutex_lock(&current->perf_counter_mutex);
1748 list_for_each_entry(counter, &current->perf_counter_list, owner_entry)
1749 perf_counter_for_each_child(counter, perf_counter_enable);
1750 mutex_unlock(&current->perf_counter_mutex);
1751
1752 return 0;
1753}
1754
1755int perf_counter_task_disable(void)
1756{
1757 struct perf_counter *counter;
1758
1759 mutex_lock(&current->perf_counter_mutex);
1760 list_for_each_entry(counter, &current->perf_counter_list, owner_entry)
1761 perf_counter_for_each_child(counter, perf_counter_disable);
1762 mutex_unlock(&current->perf_counter_mutex);
1763
1764 return 0;
1765}
1766
1767/*
1768 * Callers need to ensure there can be no nesting of this function, otherwise
1769 * the seqlock logic goes bad. We can not serialize this because the arch
1770 * code calls this from NMI context.
1771 */
1772void perf_counter_update_userpage(struct perf_counter *counter)
1773{
1774 struct perf_counter_mmap_page *userpg;
1775 struct perf_mmap_data *data;
1776
1777 rcu_read_lock();
1778 data = rcu_dereference(counter->data);
1779 if (!data)
1780 goto unlock;
1781
1782 userpg = data->user_page;
1783
1784 /*
1785 * Disable preemption so as to not let the corresponding user-space
1786 * spin too long if we get preempted.
1787 */
1788 preempt_disable();
1789 ++userpg->lock;
1790 barrier();
1791 userpg->index = counter->hw.idx;
1792 userpg->offset = atomic64_read(&counter->count);
1793 if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1794 userpg->offset -= atomic64_read(&counter->hw.prev_count);
1795
1796 barrier();
1797 ++userpg->lock;
1798 preempt_enable();
1799unlock:
1800 rcu_read_unlock();
1801}
1802
1803static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1804{
1805 struct perf_counter *counter = vma->vm_file->private_data;
1806 struct perf_mmap_data *data;
1807 int ret = VM_FAULT_SIGBUS;
1808
1809 rcu_read_lock();
1810 data = rcu_dereference(counter->data);
1811 if (!data)
1812 goto unlock;
1813
1814 if (vmf->pgoff == 0) {
1815 vmf->page = virt_to_page(data->user_page);
1816 } else {
1817 int nr = vmf->pgoff - 1;
1818
1819 if ((unsigned)nr > data->nr_pages)
1820 goto unlock;
1821
1822 vmf->page = virt_to_page(data->data_pages[nr]);
1823 }
1824 get_page(vmf->page);
1825 ret = 0;
1826unlock:
1827 rcu_read_unlock();
1828
1829 return ret;
1830}
1831
1832static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
1833{
1834 struct perf_mmap_data *data;
1835 unsigned long size;
1836 int i;
1837
1838 WARN_ON(atomic_read(&counter->mmap_count));
1839
1840 size = sizeof(struct perf_mmap_data);
1841 size += nr_pages * sizeof(void *);
1842
1843 data = kzalloc(size, GFP_KERNEL);
1844 if (!data)
1845 goto fail;
1846
1847 data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
1848 if (!data->user_page)
1849 goto fail_user_page;
1850
1851 for (i = 0; i < nr_pages; i++) {
1852 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
1853 if (!data->data_pages[i])
1854 goto fail_data_pages;
1855 }
1856
1857 data->nr_pages = nr_pages;
1858 atomic_set(&data->lock, -1);
1859
1860 rcu_assign_pointer(counter->data, data);
1861
1862 return 0;
1863
1864fail_data_pages:
1865 for (i--; i >= 0; i--)
1866 free_page((unsigned long)data->data_pages[i]);
1867
1868 free_page((unsigned long)data->user_page);
1869
1870fail_user_page:
1871 kfree(data);
1872
1873fail:
1874 return -ENOMEM;
1875}
1876
1877static void __perf_mmap_data_free(struct rcu_head *rcu_head)
1878{
1879 struct perf_mmap_data *data;
1880 int i;
1881
1882 data = container_of(rcu_head, struct perf_mmap_data, rcu_head);
1883
1884 free_page((unsigned long)data->user_page);
1885 for (i = 0; i < data->nr_pages; i++)
1886 free_page((unsigned long)data->data_pages[i]);
1887 kfree(data);
1888}
1889
1890static void perf_mmap_data_free(struct perf_counter *counter)
1891{
1892 struct perf_mmap_data *data = counter->data;
1893
1894 WARN_ON(atomic_read(&counter->mmap_count));
1895
1896 rcu_assign_pointer(counter->data, NULL);
1897 call_rcu(&data->rcu_head, __perf_mmap_data_free);
1898}
1899
1900static void perf_mmap_open(struct vm_area_struct *vma)
1901{
1902 struct perf_counter *counter = vma->vm_file->private_data;
1903
1904 atomic_inc(&counter->mmap_count);
1905}
1906
1907static void perf_mmap_close(struct vm_area_struct *vma)
1908{
1909 struct perf_counter *counter = vma->vm_file->private_data;
1910
1911 WARN_ON_ONCE(counter->ctx->parent_ctx);
1912 if (atomic_dec_and_mutex_lock(&counter->mmap_count, &counter->mmap_mutex)) {
1913 struct user_struct *user = current_user();
1914
1915 atomic_long_sub(counter->data->nr_pages + 1, &user->locked_vm);
1916 vma->vm_mm->locked_vm -= counter->data->nr_locked;
1917 perf_mmap_data_free(counter);
1918 mutex_unlock(&counter->mmap_mutex);
1919 }
1920}
1921
1922static struct vm_operations_struct perf_mmap_vmops = {
1923 .open = perf_mmap_open,
1924 .close = perf_mmap_close,
1925 .fault = perf_mmap_fault,
1926};
1927
1928static int perf_mmap(struct file *file, struct vm_area_struct *vma)
1929{
1930 struct perf_counter *counter = file->private_data;
1931 unsigned long user_locked, user_lock_limit;
1932 struct user_struct *user = current_user();
1933 unsigned long locked, lock_limit;
1934 unsigned long vma_size;
1935 unsigned long nr_pages;
1936 long user_extra, extra;
1937 int ret = 0;
1938
1939 if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
1940 return -EINVAL;
1941
1942 vma_size = vma->vm_end - vma->vm_start;
1943 nr_pages = (vma_size / PAGE_SIZE) - 1;
1944
1945 /*
1946 * If we have data pages ensure they're a power-of-two number, so we
1947 * can do bitmasks instead of modulo.
1948 */
1949 if (nr_pages != 0 && !is_power_of_2(nr_pages))
1950 return -EINVAL;
1951
1952 if (vma_size != PAGE_SIZE * (1 + nr_pages))
1953 return -EINVAL;
1954
1955 if (vma->vm_pgoff != 0)
1956 return -EINVAL;
1957
1958 WARN_ON_ONCE(counter->ctx->parent_ctx);
1959 mutex_lock(&counter->mmap_mutex);
1960 if (atomic_inc_not_zero(&counter->mmap_count)) {
1961 if (nr_pages != counter->data->nr_pages)
1962 ret = -EINVAL;
1963 goto unlock;
1964 }
1965
1966 user_extra = nr_pages + 1;
1967 user_lock_limit = sysctl_perf_counter_mlock >> (PAGE_SHIFT - 10);
1968
1969 /*
1970 * Increase the limit linearly with more CPUs:
1971 */
1972 user_lock_limit *= num_online_cpus();
1973
1974 user_locked = atomic_long_read(&user->locked_vm) + user_extra;
1975
1976 extra = 0;
1977 if (user_locked > user_lock_limit)
1978 extra = user_locked - user_lock_limit;
1979
1980 lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
1981 lock_limit >>= PAGE_SHIFT;
1982 locked = vma->vm_mm->locked_vm + extra;
1983
1984 if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
1985 ret = -EPERM;
1986 goto unlock;
1987 }
1988
1989 WARN_ON(counter->data);
1990 ret = perf_mmap_data_alloc(counter, nr_pages);
1991 if (ret)
1992 goto unlock;
1993
1994 atomic_set(&counter->mmap_count, 1);
1995 atomic_long_add(user_extra, &user->locked_vm);
1996 vma->vm_mm->locked_vm += extra;
1997 counter->data->nr_locked = extra;
1998unlock:
1999 mutex_unlock(&counter->mmap_mutex);
2000
2001 vma->vm_flags &= ~VM_MAYWRITE;
2002 vma->vm_flags |= VM_RESERVED;
2003 vma->vm_ops = &perf_mmap_vmops;
2004
2005 return ret;
2006}
2007
2008static int perf_fasync(int fd, struct file *filp, int on)
2009{
2010 struct inode *inode = filp->f_path.dentry->d_inode;
2011 struct perf_counter *counter = filp->private_data;
2012 int retval;
2013
2014 mutex_lock(&inode->i_mutex);
2015 retval = fasync_helper(fd, filp, on, &counter->fasync);
2016 mutex_unlock(&inode->i_mutex);
2017
2018 if (retval < 0)
2019 return retval;
2020
2021 return 0;
2022}
2023
2024static const struct file_operations perf_fops = {
2025 .release = perf_release,
2026 .read = perf_read,
2027 .poll = perf_poll,
2028 .unlocked_ioctl = perf_ioctl,
2029 .compat_ioctl = perf_ioctl,
2030 .mmap = perf_mmap,
2031 .fasync = perf_fasync,
2032};
2033
2034/*
2035 * Perf counter wakeup
2036 *
2037 * If there's data, ensure we set the poll() state and publish everything
2038 * to user-space before waking everybody up.
2039 */
2040
2041void perf_counter_wakeup(struct perf_counter *counter)
2042{
2043 wake_up_all(&counter->waitq);
2044
2045 if (counter->pending_kill) {
2046 kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
2047 counter->pending_kill = 0;
2048 }
2049}
2050
2051/*
2052 * Pending wakeups
2053 *
2054 * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
2055 *
2056 * The NMI bit means we cannot possibly take locks. Therefore, maintain a
2057 * single linked list and use cmpxchg() to add entries lockless.
2058 */
2059
2060static void perf_pending_counter(struct perf_pending_entry *entry)
2061{
2062 struct perf_counter *counter = container_of(entry,
2063 struct perf_counter, pending);
2064
2065 if (counter->pending_disable) {
2066 counter->pending_disable = 0;
2067 perf_counter_disable(counter);
2068 }
2069
2070 if (counter->pending_wakeup) {
2071 counter->pending_wakeup = 0;
2072 perf_counter_wakeup(counter);
2073 }
2074}
2075
2076#define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
2077
2078static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
2079 PENDING_TAIL,
2080};
2081
2082static void perf_pending_queue(struct perf_pending_entry *entry,
2083 void (*func)(struct perf_pending_entry *))
2084{
2085 struct perf_pending_entry **head;
2086
2087 if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
2088 return;
2089
2090 entry->func = func;
2091
2092 head = &get_cpu_var(perf_pending_head);
2093
2094 do {
2095 entry->next = *head;
2096 } while (cmpxchg(head, entry->next, entry) != entry->next);
2097
2098 set_perf_counter_pending();
2099
2100 put_cpu_var(perf_pending_head);
2101}
2102
2103static int __perf_pending_run(void)
2104{
2105 struct perf_pending_entry *list;
2106 int nr = 0;
2107
2108 list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
2109 while (list != PENDING_TAIL) {
2110 void (*func)(struct perf_pending_entry *);
2111 struct perf_pending_entry *entry = list;
2112
2113 list = list->next;
2114
2115 func = entry->func;
2116 entry->next = NULL;
2117 /*
2118 * Ensure we observe the unqueue before we issue the wakeup,
2119 * so that we won't be waiting forever.
2120 * -- see perf_not_pending().
2121 */
2122 smp_wmb();
2123
2124 func(entry);
2125 nr++;
2126 }
2127
2128 return nr;
2129}
2130
2131static inline int perf_not_pending(struct perf_counter *counter)
2132{
2133 /*
2134 * If we flush on whatever cpu we run, there is a chance we don't
2135 * need to wait.
2136 */
2137 get_cpu();
2138 __perf_pending_run();
2139 put_cpu();
2140
2141 /*
2142 * Ensure we see the proper queue state before going to sleep
2143 * so that we do not miss the wakeup. -- see perf_pending_handle()
2144 */
2145 smp_rmb();
2146 return counter->pending.next == NULL;
2147}
2148
2149static void perf_pending_sync(struct perf_counter *counter)
2150{
2151 wait_event(counter->waitq, perf_not_pending(counter));
2152}
2153
2154void perf_counter_do_pending(void)
2155{
2156 __perf_pending_run();
2157}
2158
2159/*
2160 * Callchain support -- arch specific
2161 */
2162
2163__weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2164{
2165 return NULL;
2166}
2167
2168/*
2169 * Output
2170 */
2171
2172struct perf_output_handle {
2173 struct perf_counter *counter;
2174 struct perf_mmap_data *data;
2175 unsigned long head;
2176 unsigned long offset;
2177 int nmi;
2178 int overflow;
2179 int locked;
2180 unsigned long flags;
2181};
2182
2183static void perf_output_wakeup(struct perf_output_handle *handle)
2184{
2185 atomic_set(&handle->data->poll, POLL_IN);
2186
2187 if (handle->nmi) {
2188 handle->counter->pending_wakeup = 1;
2189 perf_pending_queue(&handle->counter->pending,
2190 perf_pending_counter);
2191 } else
2192 perf_counter_wakeup(handle->counter);
2193}
2194
2195/*
2196 * Curious locking construct.
2197 *
2198 * We need to ensure a later event doesn't publish a head when a former
2199 * event isn't done writing. However since we need to deal with NMIs we
2200 * cannot fully serialize things.
2201 *
2202 * What we do is serialize between CPUs so we only have to deal with NMI
2203 * nesting on a single CPU.
2204 *
2205 * We only publish the head (and generate a wakeup) when the outer-most
2206 * event completes.
2207 */
2208static void perf_output_lock(struct perf_output_handle *handle)
2209{
2210 struct perf_mmap_data *data = handle->data;
2211 int cpu;
2212
2213 handle->locked = 0;
2214
2215 local_irq_save(handle->flags);
2216 cpu = smp_processor_id();
2217
2218 if (in_nmi() && atomic_read(&data->lock) == cpu)
2219 return;
2220
2221 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2222 cpu_relax();
2223
2224 handle->locked = 1;
2225}
2226
2227static void perf_output_unlock(struct perf_output_handle *handle)
2228{
2229 struct perf_mmap_data *data = handle->data;
2230 unsigned long head;
2231 int cpu;
2232
2233 data->done_head = data->head;
2234
2235 if (!handle->locked)
2236 goto out;
2237
2238again:
2239 /*
2240 * The xchg implies a full barrier that ensures all writes are done
2241 * before we publish the new head, matched by a rmb() in userspace when
2242 * reading this position.
2243 */
2244 while ((head = atomic_long_xchg(&data->done_head, 0)))
2245 data->user_page->data_head = head;
2246
2247 /*
2248 * NMI can happen here, which means we can miss a done_head update.
2249 */
2250
2251 cpu = atomic_xchg(&data->lock, -1);
2252 WARN_ON_ONCE(cpu != smp_processor_id());
2253
2254 /*
2255 * Therefore we have to validate we did not indeed do so.
2256 */
2257 if (unlikely(atomic_long_read(&data->done_head))) {
2258 /*
2259 * Since we had it locked, we can lock it again.
2260 */
2261 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2262 cpu_relax();
2263
2264 goto again;
2265 }
2266
2267 if (atomic_xchg(&data->wakeup, 0))
2268 perf_output_wakeup(handle);
2269out:
2270 local_irq_restore(handle->flags);
2271}
2272
2273static int perf_output_begin(struct perf_output_handle *handle,
2274 struct perf_counter *counter, unsigned int size,
2275 int nmi, int overflow)
2276{
2277 struct perf_mmap_data *data;
2278 unsigned int offset, head;
2279
2280 /*
2281 * For inherited counters we send all the output towards the parent.
2282 */
2283 if (counter->parent)
2284 counter = counter->parent;
2285
2286 rcu_read_lock();
2287 data = rcu_dereference(counter->data);
2288 if (!data)
2289 goto out;
2290
2291 handle->data = data;
2292 handle->counter = counter;
2293 handle->nmi = nmi;
2294 handle->overflow = overflow;
2295
2296 if (!data->nr_pages)
2297 goto fail;
2298
2299 perf_output_lock(handle);
2300
2301 do {
2302 offset = head = atomic_long_read(&data->head);
2303 head += size;
2304 } while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
2305
2306 handle->offset = offset;
2307 handle->head = head;
2308
2309 if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
2310 atomic_set(&data->wakeup, 1);
2311
2312 return 0;
2313
2314fail:
2315 perf_output_wakeup(handle);
2316out:
2317 rcu_read_unlock();
2318
2319 return -ENOSPC;
2320}
2321
2322static void perf_output_copy(struct perf_output_handle *handle,
2323 const void *buf, unsigned int len)
2324{
2325 unsigned int pages_mask;
2326 unsigned int offset;
2327 unsigned int size;
2328 void **pages;
2329
2330 offset = handle->offset;
2331 pages_mask = handle->data->nr_pages - 1;
2332 pages = handle->data->data_pages;
2333
2334 do {
2335 unsigned int page_offset;
2336 int nr;
2337
2338 nr = (offset >> PAGE_SHIFT) & pages_mask;
2339 page_offset = offset & (PAGE_SIZE - 1);
2340 size = min_t(unsigned int, PAGE_SIZE - page_offset, len);
2341
2342 memcpy(pages[nr] + page_offset, buf, size);
2343
2344 len -= size;
2345 buf += size;
2346 offset += size;
2347 } while (len);
2348
2349 handle->offset = offset;
2350
2351 /*
2352 * Check we didn't copy past our reservation window, taking the
2353 * possible unsigned int wrap into account.
2354 */
2355 WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
2356}
2357
2358#define perf_output_put(handle, x) \
2359 perf_output_copy((handle), &(x), sizeof(x))
2360
2361static void perf_output_end(struct perf_output_handle *handle)
2362{
2363 struct perf_counter *counter = handle->counter;
2364 struct perf_mmap_data *data = handle->data;
2365
2366 int wakeup_events = counter->attr.wakeup_events;
2367
2368 if (handle->overflow && wakeup_events) {
2369 int events = atomic_inc_return(&data->events);
2370 if (events >= wakeup_events) {
2371 atomic_sub(wakeup_events, &data->events);
2372 atomic_set(&data->wakeup, 1);
2373 }
2374 }
2375
2376 perf_output_unlock(handle);
2377 rcu_read_unlock();
2378}
2379
2380static u32 perf_counter_pid(struct perf_counter *counter, struct task_struct *p)
2381{
2382 /*
2383 * only top level counters have the pid namespace they were created in
2384 */
2385 if (counter->parent)
2386 counter = counter->parent;
2387
2388 return task_tgid_nr_ns(p, counter->ns);
2389}
2390
2391static u32 perf_counter_tid(struct perf_counter *counter, struct task_struct *p)
2392{
2393 /*
2394 * only top level counters have the pid namespace they were created in
2395 */
2396 if (counter->parent)
2397 counter = counter->parent;
2398
2399 return task_pid_nr_ns(p, counter->ns);
2400}
2401
2402static void perf_counter_output(struct perf_counter *counter, int nmi,
2403 struct perf_sample_data *data)
2404{
2405 int ret;
2406 u64 sample_type = counter->attr.sample_type;
2407 struct perf_output_handle handle;
2408 struct perf_event_header header;
2409 u64 ip;
2410 struct {
2411 u32 pid, tid;
2412 } tid_entry;
2413 struct {
2414 u64 id;
2415 u64 counter;
2416 } group_entry;
2417 struct perf_callchain_entry *callchain = NULL;
2418 int callchain_size = 0;
2419 u64 time;
2420 struct {
2421 u32 cpu, reserved;
2422 } cpu_entry;
2423
2424 header.type = 0;
2425 header.size = sizeof(header);
2426
2427 header.misc = PERF_EVENT_MISC_OVERFLOW;
2428 header.misc |= perf_misc_flags(data->regs);
2429
2430 if (sample_type & PERF_SAMPLE_IP) {
2431 ip = perf_instruction_pointer(data->regs);
2432 header.type |= PERF_SAMPLE_IP;
2433 header.size += sizeof(ip);
2434 }
2435
2436 if (sample_type & PERF_SAMPLE_TID) {
2437 /* namespace issues */
2438 tid_entry.pid = perf_counter_pid(counter, current);
2439 tid_entry.tid = perf_counter_tid(counter, current);
2440
2441 header.type |= PERF_SAMPLE_TID;
2442 header.size += sizeof(tid_entry);
2443 }
2444
2445 if (sample_type & PERF_SAMPLE_TIME) {
2446 /*
2447 * Maybe do better on x86 and provide cpu_clock_nmi()
2448 */
2449 time = sched_clock();
2450
2451 header.type |= PERF_SAMPLE_TIME;
2452 header.size += sizeof(u64);
2453 }
2454
2455 if (sample_type & PERF_SAMPLE_ADDR) {
2456 header.type |= PERF_SAMPLE_ADDR;
2457 header.size += sizeof(u64);
2458 }
2459
2460 if (sample_type & PERF_SAMPLE_ID) {
2461 header.type |= PERF_SAMPLE_ID;
2462 header.size += sizeof(u64);
2463 }
2464
2465 if (sample_type & PERF_SAMPLE_CPU) {
2466 header.type |= PERF_SAMPLE_CPU;
2467 header.size += sizeof(cpu_entry);
2468
2469 cpu_entry.cpu = raw_smp_processor_id();
2470 }
2471
2472 if (sample_type & PERF_SAMPLE_PERIOD) {
2473 header.type |= PERF_SAMPLE_PERIOD;
2474 header.size += sizeof(u64);
2475 }
2476
2477 if (sample_type & PERF_SAMPLE_GROUP) {
2478 header.type |= PERF_SAMPLE_GROUP;
2479 header.size += sizeof(u64) +
2480 counter->nr_siblings * sizeof(group_entry);
2481 }
2482
2483 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
2484 callchain = perf_callchain(data->regs);
2485
2486 if (callchain) {
2487 callchain_size = (1 + callchain->nr) * sizeof(u64);
2488
2489 header.type |= PERF_SAMPLE_CALLCHAIN;
2490 header.size += callchain_size;
2491 }
2492 }
2493
2494 ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
2495 if (ret)
2496 return;
2497
2498 perf_output_put(&handle, header);
2499
2500 if (sample_type & PERF_SAMPLE_IP)
2501 perf_output_put(&handle, ip);
2502
2503 if (sample_type & PERF_SAMPLE_TID)
2504 perf_output_put(&handle, tid_entry);
2505
2506 if (sample_type & PERF_SAMPLE_TIME)
2507 perf_output_put(&handle, time);
2508
2509 if (sample_type & PERF_SAMPLE_ADDR)
2510 perf_output_put(&handle, data->addr);
2511
2512 if (sample_type & PERF_SAMPLE_ID)
2513 perf_output_put(&handle, counter->id);
2514
2515 if (sample_type & PERF_SAMPLE_CPU)
2516 perf_output_put(&handle, cpu_entry);
2517
2518 if (sample_type & PERF_SAMPLE_PERIOD)
2519 perf_output_put(&handle, data->period);
2520
2521 /*
2522 * XXX PERF_SAMPLE_GROUP vs inherited counters seems difficult.
2523 */
2524 if (sample_type & PERF_SAMPLE_GROUP) {
2525 struct perf_counter *leader, *sub;
2526 u64 nr = counter->nr_siblings;
2527
2528 perf_output_put(&handle, nr);
2529
2530 leader = counter->group_leader;
2531 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2532 if (sub != counter)
2533 sub->pmu->read(sub);
2534
2535 group_entry.id = sub->id;
2536 group_entry.counter = atomic64_read(&sub->count);
2537
2538 perf_output_put(&handle, group_entry);
2539 }
2540 }
2541
2542 if (callchain)
2543 perf_output_copy(&handle, callchain, callchain_size);
2544
2545 perf_output_end(&handle);
2546}
2547
2548/*
2549 * fork tracking
2550 */
2551
2552struct perf_fork_event {
2553 struct task_struct *task;
2554
2555 struct {
2556 struct perf_event_header header;
2557
2558 u32 pid;
2559 u32 ppid;
2560 } event;
2561};
2562
2563static void perf_counter_fork_output(struct perf_counter *counter,
2564 struct perf_fork_event *fork_event)
2565{
2566 struct perf_output_handle handle;
2567 int size = fork_event->event.header.size;
2568 struct task_struct *task = fork_event->task;
2569 int ret = perf_output_begin(&handle, counter, size, 0, 0);
2570
2571 if (ret)
2572 return;
2573
2574 fork_event->event.pid = perf_counter_pid(counter, task);
2575 fork_event->event.ppid = perf_counter_pid(counter, task->real_parent);
2576
2577 perf_output_put(&handle, fork_event->event);
2578 perf_output_end(&handle);
2579}
2580
2581static int perf_counter_fork_match(struct perf_counter *counter)
2582{
2583 if (counter->attr.comm || counter->attr.mmap)
2584 return 1;
2585
2586 return 0;
2587}
2588
2589static void perf_counter_fork_ctx(struct perf_counter_context *ctx,
2590 struct perf_fork_event *fork_event)
2591{
2592 struct perf_counter *counter;
2593
2594 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2595 return;
2596
2597 rcu_read_lock();
2598 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2599 if (perf_counter_fork_match(counter))
2600 perf_counter_fork_output(counter, fork_event);
2601 }
2602 rcu_read_unlock();
2603}
2604
2605static void perf_counter_fork_event(struct perf_fork_event *fork_event)
2606{
2607 struct perf_cpu_context *cpuctx;
2608 struct perf_counter_context *ctx;
2609
2610 cpuctx = &get_cpu_var(perf_cpu_context);
2611 perf_counter_fork_ctx(&cpuctx->ctx, fork_event);
2612 put_cpu_var(perf_cpu_context);
2613
2614 rcu_read_lock();
2615 /*
2616 * doesn't really matter which of the child contexts the
2617 * events ends up in.
2618 */
2619 ctx = rcu_dereference(current->perf_counter_ctxp);
2620 if (ctx)
2621 perf_counter_fork_ctx(ctx, fork_event);
2622 rcu_read_unlock();
2623}
2624
2625void perf_counter_fork(struct task_struct *task)
2626{
2627 struct perf_fork_event fork_event;
2628
2629 if (!atomic_read(&nr_comm_counters) &&
2630 !atomic_read(&nr_mmap_counters))
2631 return;
2632
2633 fork_event = (struct perf_fork_event){
2634 .task = task,
2635 .event = {
2636 .header = {
2637 .type = PERF_EVENT_FORK,
2638 .size = sizeof(fork_event.event),
2639 },
2640 },
2641 };
2642
2643 perf_counter_fork_event(&fork_event);
2644}
2645
2646/*
2647 * comm tracking
2648 */
2649
2650struct perf_comm_event {
2651 struct task_struct *task;
2652 char *comm;
2653 int comm_size;
2654
2655 struct {
2656 struct perf_event_header header;
2657
2658 u32 pid;
2659 u32 tid;
2660 } event;
2661};
2662
2663static void perf_counter_comm_output(struct perf_counter *counter,
2664 struct perf_comm_event *comm_event)
2665{
2666 struct perf_output_handle handle;
2667 int size = comm_event->event.header.size;
2668 int ret = perf_output_begin(&handle, counter, size, 0, 0);
2669
2670 if (ret)
2671 return;
2672
2673 comm_event->event.pid = perf_counter_pid(counter, comm_event->task);
2674 comm_event->event.tid = perf_counter_tid(counter, comm_event->task);
2675
2676 perf_output_put(&handle, comm_event->event);
2677 perf_output_copy(&handle, comm_event->comm,
2678 comm_event->comm_size);
2679 perf_output_end(&handle);
2680}
2681
2682static int perf_counter_comm_match(struct perf_counter *counter)
2683{
2684 if (counter->attr.comm)
2685 return 1;
2686
2687 return 0;
2688}
2689
2690static void perf_counter_comm_ctx(struct perf_counter_context *ctx,
2691 struct perf_comm_event *comm_event)
2692{
2693 struct perf_counter *counter;
2694
2695 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2696 return;
2697
2698 rcu_read_lock();
2699 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2700 if (perf_counter_comm_match(counter))
2701 perf_counter_comm_output(counter, comm_event);
2702 }
2703 rcu_read_unlock();
2704}
2705
2706static void perf_counter_comm_event(struct perf_comm_event *comm_event)
2707{
2708 struct perf_cpu_context *cpuctx;
2709 struct perf_counter_context *ctx;
2710 unsigned int size;
2711 char *comm = comm_event->task->comm;
2712
2713 size = ALIGN(strlen(comm)+1, sizeof(u64));
2714
2715 comm_event->comm = comm;
2716 comm_event->comm_size = size;
2717
2718 comm_event->event.header.size = sizeof(comm_event->event) + size;
2719
2720 cpuctx = &get_cpu_var(perf_cpu_context);
2721 perf_counter_comm_ctx(&cpuctx->ctx, comm_event);
2722 put_cpu_var(perf_cpu_context);
2723
2724 rcu_read_lock();
2725 /*
2726 * doesn't really matter which of the child contexts the
2727 * events ends up in.
2728 */
2729 ctx = rcu_dereference(current->perf_counter_ctxp);
2730 if (ctx)
2731 perf_counter_comm_ctx(ctx, comm_event);
2732 rcu_read_unlock();
2733}
2734
2735void perf_counter_comm(struct task_struct *task)
2736{
2737 struct perf_comm_event comm_event;
2738
2739 if (!atomic_read(&nr_comm_counters))
2740 return;
2741
2742 comm_event = (struct perf_comm_event){
2743 .task = task,
2744 .event = {
2745 .header = { .type = PERF_EVENT_COMM, },
2746 },
2747 };
2748
2749 perf_counter_comm_event(&comm_event);
2750}
2751
2752/*
2753 * mmap tracking
2754 */
2755
2756struct perf_mmap_event {
2757 struct vm_area_struct *vma;
2758
2759 const char *file_name;
2760 int file_size;
2761
2762 struct {
2763 struct perf_event_header header;
2764
2765 u32 pid;
2766 u32 tid;
2767 u64 start;
2768 u64 len;
2769 u64 pgoff;
2770 } event;
2771};
2772
2773static void perf_counter_mmap_output(struct perf_counter *counter,
2774 struct perf_mmap_event *mmap_event)
2775{
2776 struct perf_output_handle handle;
2777 int size = mmap_event->event.header.size;
2778 int ret = perf_output_begin(&handle, counter, size, 0, 0);
2779
2780 if (ret)
2781 return;
2782
2783 mmap_event->event.pid = perf_counter_pid(counter, current);
2784 mmap_event->event.tid = perf_counter_tid(counter, current);
2785
2786 perf_output_put(&handle, mmap_event->event);
2787 perf_output_copy(&handle, mmap_event->file_name,
2788 mmap_event->file_size);
2789 perf_output_end(&handle);
2790}
2791
2792static int perf_counter_mmap_match(struct perf_counter *counter,
2793 struct perf_mmap_event *mmap_event)
2794{
2795 if (counter->attr.mmap)
2796 return 1;
2797
2798 return 0;
2799}
2800
2801static void perf_counter_mmap_ctx(struct perf_counter_context *ctx,
2802 struct perf_mmap_event *mmap_event)
2803{
2804 struct perf_counter *counter;
2805
2806 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2807 return;
2808
2809 rcu_read_lock();
2810 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2811 if (perf_counter_mmap_match(counter, mmap_event))
2812 perf_counter_mmap_output(counter, mmap_event);
2813 }
2814 rcu_read_unlock();
2815}
2816
2817static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
2818{
2819 struct perf_cpu_context *cpuctx;
2820 struct perf_counter_context *ctx;
2821 struct vm_area_struct *vma = mmap_event->vma;
2822 struct file *file = vma->vm_file;
2823 unsigned int size;
2824 char tmp[16];
2825 char *buf = NULL;
2826 const char *name;
2827
2828 if (file) {
2829 buf = kzalloc(PATH_MAX, GFP_KERNEL);
2830 if (!buf) {
2831 name = strncpy(tmp, "//enomem", sizeof(tmp));
2832 goto got_name;
2833 }
2834 name = d_path(&file->f_path, buf, PATH_MAX);
2835 if (IS_ERR(name)) {
2836 name = strncpy(tmp, "//toolong", sizeof(tmp));
2837 goto got_name;
2838 }
2839 } else {
2840 name = arch_vma_name(mmap_event->vma);
2841 if (name)
2842 goto got_name;
2843
2844 if (!vma->vm_mm) {
2845 name = strncpy(tmp, "[vdso]", sizeof(tmp));
2846 goto got_name;
2847 }
2848
2849 name = strncpy(tmp, "//anon", sizeof(tmp));
2850 goto got_name;
2851 }
2852
2853got_name:
2854 size = ALIGN(strlen(name)+1, sizeof(u64));
2855
2856 mmap_event->file_name = name;
2857 mmap_event->file_size = size;
2858
2859 mmap_event->event.header.size = sizeof(mmap_event->event) + size;
2860
2861 cpuctx = &get_cpu_var(perf_cpu_context);
2862 perf_counter_mmap_ctx(&cpuctx->ctx, mmap_event);
2863 put_cpu_var(perf_cpu_context);
2864
2865 rcu_read_lock();
2866 /*
2867 * doesn't really matter which of the child contexts the
2868 * events ends up in.
2869 */
2870 ctx = rcu_dereference(current->perf_counter_ctxp);
2871 if (ctx)
2872 perf_counter_mmap_ctx(ctx, mmap_event);
2873 rcu_read_unlock();
2874
2875 kfree(buf);
2876}
2877
2878void __perf_counter_mmap(struct vm_area_struct *vma)
2879{
2880 struct perf_mmap_event mmap_event;
2881
2882 if (!atomic_read(&nr_mmap_counters))
2883 return;
2884
2885 mmap_event = (struct perf_mmap_event){
2886 .vma = vma,
2887 .event = {
2888 .header = { .type = PERF_EVENT_MMAP, },
2889 .start = vma->vm_start,
2890 .len = vma->vm_end - vma->vm_start,
2891 .pgoff = vma->vm_pgoff,
2892 },
2893 };
2894
2895 perf_counter_mmap_event(&mmap_event);
2896}
2897
2898/*
2899 * Log sample_period changes so that analyzing tools can re-normalize the
2900 * event flow.
2901 */
2902
2903struct freq_event {
2904 struct perf_event_header header;
2905 u64 time;
2906 u64 id;
2907 u64 period;
2908};
2909
2910static void perf_log_period(struct perf_counter *counter, u64 period)
2911{
2912 struct perf_output_handle handle;
2913 struct freq_event event;
2914 int ret;
2915
2916 if (counter->hw.sample_period == period)
2917 return;
2918
2919 if (counter->attr.sample_type & PERF_SAMPLE_PERIOD)
2920 return;
2921
2922 event = (struct freq_event) {
2923 .header = {
2924 .type = PERF_EVENT_PERIOD,
2925 .misc = 0,
2926 .size = sizeof(event),
2927 },
2928 .time = sched_clock(),
2929 .id = counter->id,
2930 .period = period,
2931 };
2932
2933 ret = perf_output_begin(&handle, counter, sizeof(event), 1, 0);
2934 if (ret)
2935 return;
2936
2937 perf_output_put(&handle, event);
2938 perf_output_end(&handle);
2939}
2940
2941/*
2942 * IRQ throttle logging
2943 */
2944
2945static void perf_log_throttle(struct perf_counter *counter, int enable)
2946{
2947 struct perf_output_handle handle;
2948 int ret;
2949
2950 struct {
2951 struct perf_event_header header;
2952 u64 time;
2953 u64 id;
2954 } throttle_event = {
2955 .header = {
2956 .type = PERF_EVENT_THROTTLE + 1,
2957 .misc = 0,
2958 .size = sizeof(throttle_event),
2959 },
2960 .time = sched_clock(),
2961 .id = counter->id,
2962 };
2963
2964 ret = perf_output_begin(&handle, counter, sizeof(throttle_event), 1, 0);
2965 if (ret)
2966 return;
2967
2968 perf_output_put(&handle, throttle_event);
2969 perf_output_end(&handle);
2970}
2971
2972/*
2973 * Generic counter overflow handling.
2974 */
2975
2976int perf_counter_overflow(struct perf_counter *counter, int nmi,
2977 struct perf_sample_data *data)
2978{
2979 int events = atomic_read(&counter->event_limit);
2980 int throttle = counter->pmu->unthrottle != NULL;
2981 struct hw_perf_counter *hwc = &counter->hw;
2982 int ret = 0;
2983
2984 if (!throttle) {
2985 hwc->interrupts++;
2986 } else {
2987 if (hwc->interrupts != MAX_INTERRUPTS) {
2988 hwc->interrupts++;
2989 if (HZ * hwc->interrupts >
2990 (u64)sysctl_perf_counter_sample_rate) {
2991 hwc->interrupts = MAX_INTERRUPTS;
2992 perf_log_throttle(counter, 0);
2993 ret = 1;
2994 }
2995 } else {
2996 /*
2997 * Keep re-disabling counters even though on the previous
2998 * pass we disabled it - just in case we raced with a
2999 * sched-in and the counter got enabled again:
3000 */
3001 ret = 1;
3002 }
3003 }
3004
3005 if (counter->attr.freq) {
3006 u64 now = sched_clock();
3007 s64 delta = now - hwc->freq_stamp;
3008
3009 hwc->freq_stamp = now;
3010
3011 if (delta > 0 && delta < TICK_NSEC)
3012 perf_adjust_period(counter, NSEC_PER_SEC / (int)delta);
3013 }
3014
3015 /*
3016 * XXX event_limit might not quite work as expected on inherited
3017 * counters
3018 */
3019
3020 counter->pending_kill = POLL_IN;
3021 if (events && atomic_dec_and_test(&counter->event_limit)) {
3022 ret = 1;
3023 counter->pending_kill = POLL_HUP;
3024 if (nmi) {
3025 counter->pending_disable = 1;
3026 perf_pending_queue(&counter->pending,
3027 perf_pending_counter);
3028 } else
3029 perf_counter_disable(counter);
3030 }
3031
3032 perf_counter_output(counter, nmi, data);
3033 return ret;
3034}
3035
3036/*
3037 * Generic software counter infrastructure
3038 */
3039
3040static void perf_swcounter_update(struct perf_counter *counter)
3041{
3042 struct hw_perf_counter *hwc = &counter->hw;
3043 u64 prev, now;
3044 s64 delta;
3045
3046again:
3047 prev = atomic64_read(&hwc->prev_count);
3048 now = atomic64_read(&hwc->count);
3049 if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
3050 goto again;
3051
3052 delta = now - prev;
3053
3054 atomic64_add(delta, &counter->count);
3055 atomic64_sub(delta, &hwc->period_left);
3056}
3057
3058static void perf_swcounter_set_period(struct perf_counter *counter)
3059{
3060 struct hw_perf_counter *hwc = &counter->hw;
3061 s64 left = atomic64_read(&hwc->period_left);
3062 s64 period = hwc->sample_period;
3063
3064 if (unlikely(left <= -period)) {
3065 left = period;
3066 atomic64_set(&hwc->period_left, left);
3067 hwc->last_period = period;
3068 }
3069
3070 if (unlikely(left <= 0)) {
3071 left += period;
3072 atomic64_add(period, &hwc->period_left);
3073 hwc->last_period = period;
3074 }
3075
3076 atomic64_set(&hwc->prev_count, -left);
3077 atomic64_set(&hwc->count, -left);
3078}
3079
3080static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
3081{
3082 enum hrtimer_restart ret = HRTIMER_RESTART;
3083 struct perf_sample_data data;
3084 struct perf_counter *counter;
3085 u64 period;
3086
3087 counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
3088 counter->pmu->read(counter);
3089
3090 data.addr = 0;
3091 data.regs = get_irq_regs();
3092 /*
3093 * In case we exclude kernel IPs or are somehow not in interrupt
3094 * context, provide the next best thing, the user IP.
3095 */
3096 if ((counter->attr.exclude_kernel || !data.regs) &&
3097 !counter->attr.exclude_user)
3098 data.regs = task_pt_regs(current);
3099
3100 if (data.regs) {
3101 if (perf_counter_overflow(counter, 0, &data))
3102 ret = HRTIMER_NORESTART;
3103 }
3104
3105 period = max_t(u64, 10000, counter->hw.sample_period);
3106 hrtimer_forward_now(hrtimer, ns_to_ktime(period));
3107
3108 return ret;
3109}
3110
3111static void perf_swcounter_overflow(struct perf_counter *counter,
3112 int nmi, struct pt_regs *regs, u64 addr)
3113{
3114 struct perf_sample_data data = {
3115 .regs = regs,
3116 .addr = addr,
3117 .period = counter->hw.last_period,
3118 };
3119
3120 perf_swcounter_update(counter);
3121 perf_swcounter_set_period(counter);
3122 if (perf_counter_overflow(counter, nmi, &data))
3123 /* soft-disable the counter */
3124 ;
3125
3126}
3127
3128static int perf_swcounter_is_counting(struct perf_counter *counter)
3129{
3130 struct perf_counter_context *ctx;
3131 unsigned long flags;
3132 int count;
3133
3134 if (counter->state == PERF_COUNTER_STATE_ACTIVE)
3135 return 1;
3136
3137 if (counter->state != PERF_COUNTER_STATE_INACTIVE)
3138 return 0;
3139
3140 /*
3141 * If the counter is inactive, it could be just because
3142 * its task is scheduled out, or because it's in a group
3143 * which could not go on the PMU. We want to count in
3144 * the first case but not the second. If the context is
3145 * currently active then an inactive software counter must
3146 * be the second case. If it's not currently active then
3147 * we need to know whether the counter was active when the
3148 * context was last active, which we can determine by
3149 * comparing counter->tstamp_stopped with ctx->time.
3150 *
3151 * We are within an RCU read-side critical section,
3152 * which protects the existence of *ctx.
3153 */
3154 ctx = counter->ctx;
3155 spin_lock_irqsave(&ctx->lock, flags);
3156 count = 1;
3157 /* Re-check state now we have the lock */
3158 if (counter->state < PERF_COUNTER_STATE_INACTIVE ||
3159 counter->ctx->is_active ||
3160 counter->tstamp_stopped < ctx->time)
3161 count = 0;
3162 spin_unlock_irqrestore(&ctx->lock, flags);
3163 return count;
3164}
3165
3166static int perf_swcounter_match(struct perf_counter *counter,
3167 enum perf_type_id type,
3168 u32 event, struct pt_regs *regs)
3169{
3170 if (!perf_swcounter_is_counting(counter))
3171 return 0;
3172
3173 if (counter->attr.type != type)
3174 return 0;
3175 if (counter->attr.config != event)
3176 return 0;
3177
3178 if (regs) {
3179 if (counter->attr.exclude_user && user_mode(regs))
3180 return 0;
3181
3182 if (counter->attr.exclude_kernel && !user_mode(regs))
3183 return 0;
3184 }
3185
3186 return 1;
3187}
3188
3189static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
3190 int nmi, struct pt_regs *regs, u64 addr)
3191{
3192 int neg = atomic64_add_negative(nr, &counter->hw.count);
3193
3194 if (counter->hw.sample_period && !neg && regs)
3195 perf_swcounter_overflow(counter, nmi, regs, addr);
3196}
3197
3198static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
3199 enum perf_type_id type, u32 event,
3200 u64 nr, int nmi, struct pt_regs *regs,
3201 u64 addr)
3202{
3203 struct perf_counter *counter;
3204
3205 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
3206 return;
3207
3208 rcu_read_lock();
3209 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
3210 if (perf_swcounter_match(counter, type, event, regs))
3211 perf_swcounter_add(counter, nr, nmi, regs, addr);
3212 }
3213 rcu_read_unlock();
3214}
3215
3216static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
3217{
3218 if (in_nmi())
3219 return &cpuctx->recursion[3];
3220
3221 if (in_irq())
3222 return &cpuctx->recursion[2];
3223
3224 if (in_softirq())
3225 return &cpuctx->recursion[1];
3226
3227 return &cpuctx->recursion[0];
3228}
3229
3230static void __perf_swcounter_event(enum perf_type_id type, u32 event,
3231 u64 nr, int nmi, struct pt_regs *regs,
3232 u64 addr)
3233{
3234 struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
3235 int *recursion = perf_swcounter_recursion_context(cpuctx);
3236 struct perf_counter_context *ctx;
3237
3238 if (*recursion)
3239 goto out;
3240
3241 (*recursion)++;
3242 barrier();
3243
3244 perf_swcounter_ctx_event(&cpuctx->ctx, type, event,
3245 nr, nmi, regs, addr);
3246 rcu_read_lock();
3247 /*
3248 * doesn't really matter which of the child contexts the
3249 * events ends up in.
3250 */
3251 ctx = rcu_dereference(current->perf_counter_ctxp);
3252 if (ctx)
3253 perf_swcounter_ctx_event(ctx, type, event, nr, nmi, regs, addr);
3254 rcu_read_unlock();
3255
3256 barrier();
3257 (*recursion)--;
3258
3259out:
3260 put_cpu_var(perf_cpu_context);
3261}
3262
3263void
3264perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
3265{
3266 __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs, addr);
3267}
3268
3269static void perf_swcounter_read(struct perf_counter *counter)
3270{
3271 perf_swcounter_update(counter);
3272}
3273
3274static int perf_swcounter_enable(struct perf_counter *counter)
3275{
3276 perf_swcounter_set_period(counter);
3277 return 0;
3278}
3279
3280static void perf_swcounter_disable(struct perf_counter *counter)
3281{
3282 perf_swcounter_update(counter);
3283}
3284
3285static const struct pmu perf_ops_generic = {
3286 .enable = perf_swcounter_enable,
3287 .disable = perf_swcounter_disable,
3288 .read = perf_swcounter_read,
3289};
3290
3291/*
3292 * Software counter: cpu wall time clock
3293 */
3294
3295static void cpu_clock_perf_counter_update(struct perf_counter *counter)
3296{
3297 int cpu = raw_smp_processor_id();
3298 s64 prev;
3299 u64 now;
3300
3301 now = cpu_clock(cpu);
3302 prev = atomic64_read(&counter->hw.prev_count);
3303 atomic64_set(&counter->hw.prev_count, now);
3304 atomic64_add(now - prev, &counter->count);
3305}
3306
3307static int cpu_clock_perf_counter_enable(struct perf_counter *counter)
3308{
3309 struct hw_perf_counter *hwc = &counter->hw;
3310 int cpu = raw_smp_processor_id();
3311
3312 atomic64_set(&hwc->prev_count, cpu_clock(cpu));
3313 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3314 hwc->hrtimer.function = perf_swcounter_hrtimer;
3315 if (hwc->sample_period) {
3316 u64 period = max_t(u64, 10000, hwc->sample_period);
3317 __hrtimer_start_range_ns(&hwc->hrtimer,
3318 ns_to_ktime(period), 0,
3319 HRTIMER_MODE_REL, 0);
3320 }
3321
3322 return 0;
3323}
3324
3325static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
3326{
3327 if (counter->hw.sample_period)
3328 hrtimer_cancel(&counter->hw.hrtimer);
3329 cpu_clock_perf_counter_update(counter);
3330}
3331
3332static void cpu_clock_perf_counter_read(struct perf_counter *counter)
3333{
3334 cpu_clock_perf_counter_update(counter);
3335}
3336
3337static const struct pmu perf_ops_cpu_clock = {
3338 .enable = cpu_clock_perf_counter_enable,
3339 .disable = cpu_clock_perf_counter_disable,
3340 .read = cpu_clock_perf_counter_read,
3341};
3342
3343/*
3344 * Software counter: task time clock
3345 */
3346
3347static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
3348{
3349 u64 prev;
3350 s64 delta;
3351
3352 prev = atomic64_xchg(&counter->hw.prev_count, now);
3353 delta = now - prev;
3354 atomic64_add(delta, &counter->count);
3355}
3356
3357static int task_clock_perf_counter_enable(struct perf_counter *counter)
3358{
3359 struct hw_perf_counter *hwc = &counter->hw;
3360 u64 now;
3361
3362 now = counter->ctx->time;
3363
3364 atomic64_set(&hwc->prev_count, now);
3365 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3366 hwc->hrtimer.function = perf_swcounter_hrtimer;
3367 if (hwc->sample_period) {
3368 u64 period = max_t(u64, 10000, hwc->sample_period);
3369 __hrtimer_start_range_ns(&hwc->hrtimer,
3370 ns_to_ktime(period), 0,
3371 HRTIMER_MODE_REL, 0);
3372 }
3373
3374 return 0;
3375}
3376
3377static void task_clock_perf_counter_disable(struct perf_counter *counter)
3378{
3379 if (counter->hw.sample_period)
3380 hrtimer_cancel(&counter->hw.hrtimer);
3381 task_clock_perf_counter_update(counter, counter->ctx->time);
3382
3383}
3384
3385static void task_clock_perf_counter_read(struct perf_counter *counter)
3386{
3387 u64 time;
3388
3389 if (!in_nmi()) {
3390 update_context_time(counter->ctx);
3391 time = counter->ctx->time;
3392 } else {
3393 u64 now = perf_clock();
3394 u64 delta = now - counter->ctx->timestamp;
3395 time = counter->ctx->time + delta;
3396 }
3397
3398 task_clock_perf_counter_update(counter, time);
3399}
3400
3401static const struct pmu perf_ops_task_clock = {
3402 .enable = task_clock_perf_counter_enable,
3403 .disable = task_clock_perf_counter_disable,
3404 .read = task_clock_perf_counter_read,
3405};
3406
3407/*
3408 * Software counter: cpu migrations
3409 */
3410void perf_counter_task_migration(struct task_struct *task, int cpu)
3411{
3412 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
3413 struct perf_counter_context *ctx;
3414
3415 perf_swcounter_ctx_event(&cpuctx->ctx, PERF_TYPE_SOFTWARE,
3416 PERF_COUNT_SW_CPU_MIGRATIONS,
3417 1, 1, NULL, 0);
3418
3419 ctx = perf_pin_task_context(task);
3420 if (ctx) {
3421 perf_swcounter_ctx_event(ctx, PERF_TYPE_SOFTWARE,
3422 PERF_COUNT_SW_CPU_MIGRATIONS,
3423 1, 1, NULL, 0);
3424 perf_unpin_context(ctx);
3425 }
3426}
3427
3428#ifdef CONFIG_EVENT_PROFILE
3429void perf_tpcounter_event(int event_id)
3430{
3431 struct pt_regs *regs = get_irq_regs();
3432
3433 if (!regs)
3434 regs = task_pt_regs(current);
3435
3436 __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs, 0);
3437}
3438EXPORT_SYMBOL_GPL(perf_tpcounter_event);
3439
3440extern int ftrace_profile_enable(int);
3441extern void ftrace_profile_disable(int);
3442
3443static void tp_perf_counter_destroy(struct perf_counter *counter)
3444{
3445 ftrace_profile_disable(perf_event_id(&counter->attr));
3446}
3447
3448static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
3449{
3450 int event_id = perf_event_id(&counter->attr);
3451 int ret;
3452
3453 ret = ftrace_profile_enable(event_id);
3454 if (ret)
3455 return NULL;
3456
3457 counter->destroy = tp_perf_counter_destroy;
3458
3459 return &perf_ops_generic;
3460}
3461#else
3462static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
3463{
3464 return NULL;
3465}
3466#endif
3467
3468static const struct pmu *sw_perf_counter_init(struct perf_counter *counter)
3469{
3470 const struct pmu *pmu = NULL;
3471
3472 /*
3473 * Software counters (currently) can't in general distinguish
3474 * between user, kernel and hypervisor events.
3475 * However, context switches and cpu migrations are considered
3476 * to be kernel events, and page faults are never hypervisor
3477 * events.
3478 */
3479 switch (counter->attr.config) {
3480 case PERF_COUNT_SW_CPU_CLOCK:
3481 pmu = &perf_ops_cpu_clock;
3482
3483 break;
3484 case PERF_COUNT_SW_TASK_CLOCK:
3485 /*
3486 * If the user instantiates this as a per-cpu counter,
3487 * use the cpu_clock counter instead.
3488 */
3489 if (counter->ctx->task)
3490 pmu = &perf_ops_task_clock;
3491 else
3492 pmu = &perf_ops_cpu_clock;
3493
3494 break;
3495 case PERF_COUNT_SW_PAGE_FAULTS:
3496 case PERF_COUNT_SW_PAGE_FAULTS_MIN:
3497 case PERF_COUNT_SW_PAGE_FAULTS_MAJ:
3498 case PERF_COUNT_SW_CONTEXT_SWITCHES:
3499 case PERF_COUNT_SW_CPU_MIGRATIONS:
3500 pmu = &perf_ops_generic;
3501 break;
3502 }
3503
3504 return pmu;
3505}
3506
3507/*
3508 * Allocate and initialize a counter structure
3509 */
3510static struct perf_counter *
3511perf_counter_alloc(struct perf_counter_attr *attr,
3512 int cpu,
3513 struct perf_counter_context *ctx,
3514 struct perf_counter *group_leader,
3515 gfp_t gfpflags)
3516{
3517 const struct pmu *pmu;
3518 struct perf_counter *counter;
3519 struct hw_perf_counter *hwc;
3520 long err;
3521
3522 counter = kzalloc(sizeof(*counter), gfpflags);
3523 if (!counter)
3524 return ERR_PTR(-ENOMEM);
3525
3526 /*
3527 * Single counters are their own group leaders, with an
3528 * empty sibling list:
3529 */
3530 if (!group_leader)
3531 group_leader = counter;
3532
3533 mutex_init(&counter->child_mutex);
3534 INIT_LIST_HEAD(&counter->child_list);
3535
3536 INIT_LIST_HEAD(&counter->list_entry);
3537 INIT_LIST_HEAD(&counter->event_entry);
3538 INIT_LIST_HEAD(&counter->sibling_list);
3539 init_waitqueue_head(&counter->waitq);
3540
3541 mutex_init(&counter->mmap_mutex);
3542
3543 counter->cpu = cpu;
3544 counter->attr = *attr;
3545 counter->group_leader = group_leader;
3546 counter->pmu = NULL;
3547 counter->ctx = ctx;
3548 counter->oncpu = -1;
3549
3550 counter->ns = get_pid_ns(current->nsproxy->pid_ns);
3551 counter->id = atomic64_inc_return(&perf_counter_id);
3552
3553 counter->state = PERF_COUNTER_STATE_INACTIVE;
3554
3555 if (attr->disabled)
3556 counter->state = PERF_COUNTER_STATE_OFF;
3557
3558 pmu = NULL;
3559
3560 hwc = &counter->hw;
3561 hwc->sample_period = attr->sample_period;
3562 if (attr->freq && attr->sample_freq)
3563 hwc->sample_period = 1;
3564
3565 atomic64_set(&hwc->period_left, hwc->sample_period);
3566
3567 /*
3568 * we currently do not support PERF_SAMPLE_GROUP on inherited counters
3569 */
3570 if (attr->inherit && (attr->sample_type & PERF_SAMPLE_GROUP))
3571 goto done;
3572
3573 if (attr->type == PERF_TYPE_RAW) {
3574 pmu = hw_perf_counter_init(counter);
3575 goto done;
3576 }
3577
3578 switch (attr->type) {
3579 case PERF_TYPE_HARDWARE:
3580 case PERF_TYPE_HW_CACHE:
3581 pmu = hw_perf_counter_init(counter);
3582 break;
3583
3584 case PERF_TYPE_SOFTWARE:
3585 pmu = sw_perf_counter_init(counter);
3586 break;
3587
3588 case PERF_TYPE_TRACEPOINT:
3589 pmu = tp_perf_counter_init(counter);
3590 break;
3591 }
3592done:
3593 err = 0;
3594 if (!pmu)
3595 err = -EINVAL;
3596 else if (IS_ERR(pmu))
3597 err = PTR_ERR(pmu);
3598
3599 if (err) {
3600 if (counter->ns)
3601 put_pid_ns(counter->ns);
3602 kfree(counter);
3603 return ERR_PTR(err);
3604 }
3605
3606 counter->pmu = pmu;
3607
3608 atomic_inc(&nr_counters);
3609 if (counter->attr.mmap)
3610 atomic_inc(&nr_mmap_counters);
3611 if (counter->attr.comm)
3612 atomic_inc(&nr_comm_counters);
3613
3614 return counter;
3615}
3616
3617/**
3618 * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
3619 *
3620 * @attr_uptr: event type attributes for monitoring/sampling
3621 * @pid: target pid
3622 * @cpu: target cpu
3623 * @group_fd: group leader counter fd
3624 */
3625SYSCALL_DEFINE5(perf_counter_open,
3626 const struct perf_counter_attr __user *, attr_uptr,
3627 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
3628{
3629 struct perf_counter *counter, *group_leader;
3630 struct perf_counter_attr attr;
3631 struct perf_counter_context *ctx;
3632 struct file *counter_file = NULL;
3633 struct file *group_file = NULL;
3634 int fput_needed = 0;
3635 int fput_needed2 = 0;
3636 int ret;
3637
3638 /* for future expandability... */
3639 if (flags)
3640 return -EINVAL;
3641
3642 if (copy_from_user(&attr, attr_uptr, sizeof(attr)) != 0)
3643 return -EFAULT;
3644
3645 if (!attr.exclude_kernel) {
3646 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
3647 return -EACCES;
3648 }
3649
3650 if (attr.freq) {
3651 if (attr.sample_freq > sysctl_perf_counter_sample_rate)
3652 return -EINVAL;
3653 }
3654
3655 /*
3656 * Get the target context (task or percpu):
3657 */
3658 ctx = find_get_context(pid, cpu);
3659 if (IS_ERR(ctx))
3660 return PTR_ERR(ctx);
3661
3662 /*
3663 * Look up the group leader (we will attach this counter to it):
3664 */
3665 group_leader = NULL;
3666 if (group_fd != -1) {
3667 ret = -EINVAL;
3668 group_file = fget_light(group_fd, &fput_needed);
3669 if (!group_file)
3670 goto err_put_context;
3671 if (group_file->f_op != &perf_fops)
3672 goto err_put_context;
3673
3674 group_leader = group_file->private_data;
3675 /*
3676 * Do not allow a recursive hierarchy (this new sibling
3677 * becoming part of another group-sibling):
3678 */
3679 if (group_leader->group_leader != group_leader)
3680 goto err_put_context;
3681 /*
3682 * Do not allow to attach to a group in a different
3683 * task or CPU context:
3684 */
3685 if (group_leader->ctx != ctx)
3686 goto err_put_context;
3687 /*
3688 * Only a group leader can be exclusive or pinned
3689 */
3690 if (attr.exclusive || attr.pinned)
3691 goto err_put_context;
3692 }
3693
3694 counter = perf_counter_alloc(&attr, cpu, ctx, group_leader,
3695 GFP_KERNEL);
3696 ret = PTR_ERR(counter);
3697 if (IS_ERR(counter))
3698 goto err_put_context;
3699
3700 ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
3701 if (ret < 0)
3702 goto err_free_put_context;
3703
3704 counter_file = fget_light(ret, &fput_needed2);
3705 if (!counter_file)
3706 goto err_free_put_context;
3707
3708 counter->filp = counter_file;
3709 WARN_ON_ONCE(ctx->parent_ctx);
3710 mutex_lock(&ctx->mutex);
3711 perf_install_in_context(ctx, counter, cpu);
3712 ++ctx->generation;
3713 mutex_unlock(&ctx->mutex);
3714
3715 counter->owner = current;
3716 get_task_struct(current);
3717 mutex_lock(&current->perf_counter_mutex);
3718 list_add_tail(&counter->owner_entry, &current->perf_counter_list);
3719 mutex_unlock(&current->perf_counter_mutex);
3720
3721 fput_light(counter_file, fput_needed2);
3722
3723out_fput:
3724 fput_light(group_file, fput_needed);
3725
3726 return ret;
3727
3728err_free_put_context:
3729 kfree(counter);
3730
3731err_put_context:
3732 put_ctx(ctx);
3733
3734 goto out_fput;
3735}
3736
3737/*
3738 * inherit a counter from parent task to child task:
3739 */
3740static struct perf_counter *
3741inherit_counter(struct perf_counter *parent_counter,
3742 struct task_struct *parent,
3743 struct perf_counter_context *parent_ctx,
3744 struct task_struct *child,
3745 struct perf_counter *group_leader,
3746 struct perf_counter_context *child_ctx)
3747{
3748 struct perf_counter *child_counter;
3749
3750 /*
3751 * Instead of creating recursive hierarchies of counters,
3752 * we link inherited counters back to the original parent,
3753 * which has a filp for sure, which we use as the reference
3754 * count:
3755 */
3756 if (parent_counter->parent)
3757 parent_counter = parent_counter->parent;
3758
3759 child_counter = perf_counter_alloc(&parent_counter->attr,
3760 parent_counter->cpu, child_ctx,
3761 group_leader, GFP_KERNEL);
3762 if (IS_ERR(child_counter))
3763 return child_counter;
3764 get_ctx(child_ctx);
3765
3766 /*
3767 * Make the child state follow the state of the parent counter,
3768 * not its attr.disabled bit. We hold the parent's mutex,
3769 * so we won't race with perf_counter_{en, dis}able_family.
3770 */
3771 if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
3772 child_counter->state = PERF_COUNTER_STATE_INACTIVE;
3773 else
3774 child_counter->state = PERF_COUNTER_STATE_OFF;
3775
3776 if (parent_counter->attr.freq)
3777 child_counter->hw.sample_period = parent_counter->hw.sample_period;
3778
3779 /*
3780 * Link it up in the child's context:
3781 */
3782 add_counter_to_ctx(child_counter, child_ctx);
3783
3784 child_counter->parent = parent_counter;
3785 /*
3786 * inherit into child's child as well:
3787 */
3788 child_counter->attr.inherit = 1;
3789
3790 /*
3791 * Get a reference to the parent filp - we will fput it
3792 * when the child counter exits. This is safe to do because
3793 * we are in the parent and we know that the filp still
3794 * exists and has a nonzero count:
3795 */
3796 atomic_long_inc(&parent_counter->filp->f_count);
3797
3798 /*
3799 * Link this into the parent counter's child list
3800 */
3801 WARN_ON_ONCE(parent_counter->ctx->parent_ctx);
3802 mutex_lock(&parent_counter->child_mutex);
3803 list_add_tail(&child_counter->child_list, &parent_counter->child_list);
3804 mutex_unlock(&parent_counter->child_mutex);
3805
3806 return child_counter;
3807}
3808
3809static int inherit_group(struct perf_counter *parent_counter,
3810 struct task_struct *parent,
3811 struct perf_counter_context *parent_ctx,
3812 struct task_struct *child,
3813 struct perf_counter_context *child_ctx)
3814{
3815 struct perf_counter *leader;
3816 struct perf_counter *sub;
3817 struct perf_counter *child_ctr;
3818
3819 leader = inherit_counter(parent_counter, parent, parent_ctx,
3820 child, NULL, child_ctx);
3821 if (IS_ERR(leader))
3822 return PTR_ERR(leader);
3823 list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
3824 child_ctr = inherit_counter(sub, parent, parent_ctx,
3825 child, leader, child_ctx);
3826 if (IS_ERR(child_ctr))
3827 return PTR_ERR(child_ctr);
3828 }
3829 return 0;
3830}
3831
3832static void sync_child_counter(struct perf_counter *child_counter,
3833 struct perf_counter *parent_counter)
3834{
3835 u64 child_val;
3836
3837 child_val = atomic64_read(&child_counter->count);
3838
3839 /*
3840 * Add back the child's count to the parent's count:
3841 */
3842 atomic64_add(child_val, &parent_counter->count);
3843 atomic64_add(child_counter->total_time_enabled,
3844 &parent_counter->child_total_time_enabled);
3845 atomic64_add(child_counter->total_time_running,
3846 &parent_counter->child_total_time_running);
3847
3848 /*
3849 * Remove this counter from the parent's list
3850 */
3851 WARN_ON_ONCE(parent_counter->ctx->parent_ctx);
3852 mutex_lock(&parent_counter->child_mutex);
3853 list_del_init(&child_counter->child_list);
3854 mutex_unlock(&parent_counter->child_mutex);
3855
3856 /*
3857 * Release the parent counter, if this was the last
3858 * reference to it.
3859 */
3860 fput(parent_counter->filp);
3861}
3862
3863static void
3864__perf_counter_exit_task(struct perf_counter *child_counter,
3865 struct perf_counter_context *child_ctx)
3866{
3867 struct perf_counter *parent_counter;
3868
3869 update_counter_times(child_counter);
3870 perf_counter_remove_from_context(child_counter);
3871
3872 parent_counter = child_counter->parent;
3873 /*
3874 * It can happen that parent exits first, and has counters
3875 * that are still around due to the child reference. These
3876 * counters need to be zapped - but otherwise linger.
3877 */
3878 if (parent_counter) {
3879 sync_child_counter(child_counter, parent_counter);
3880 free_counter(child_counter);
3881 }
3882}
3883
3884/*
3885 * When a child task exits, feed back counter values to parent counters.
3886 */
3887void perf_counter_exit_task(struct task_struct *child)
3888{
3889 struct perf_counter *child_counter, *tmp;
3890 struct perf_counter_context *child_ctx;
3891 unsigned long flags;
3892
3893 if (likely(!child->perf_counter_ctxp))
3894 return;
3895
3896 local_irq_save(flags);
3897 /*
3898 * We can't reschedule here because interrupts are disabled,
3899 * and either child is current or it is a task that can't be
3900 * scheduled, so we are now safe from rescheduling changing
3901 * our context.
3902 */
3903 child_ctx = child->perf_counter_ctxp;
3904 __perf_counter_task_sched_out(child_ctx);
3905
3906 /*
3907 * Take the context lock here so that if find_get_context is
3908 * reading child->perf_counter_ctxp, we wait until it has
3909 * incremented the context's refcount before we do put_ctx below.
3910 */
3911 spin_lock(&child_ctx->lock);
3912 child->perf_counter_ctxp = NULL;
3913 if (child_ctx->parent_ctx) {
3914 /*
3915 * This context is a clone; unclone it so it can't get
3916 * swapped to another process while we're removing all
3917 * the counters from it.
3918 */
3919 put_ctx(child_ctx->parent_ctx);
3920 child_ctx->parent_ctx = NULL;
3921 }
3922 spin_unlock(&child_ctx->lock);
3923 local_irq_restore(flags);
3924
3925 /*
3926 * We can recurse on the same lock type through:
3927 *
3928 * __perf_counter_exit_task()
3929 * sync_child_counter()
3930 * fput(parent_counter->filp)
3931 * perf_release()
3932 * mutex_lock(&ctx->mutex)
3933 *
3934 * But since its the parent context it won't be the same instance.
3935 */
3936 mutex_lock_nested(&child_ctx->mutex, SINGLE_DEPTH_NESTING);
3937
3938again:
3939 list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
3940 list_entry)
3941 __perf_counter_exit_task(child_counter, child_ctx);
3942
3943 /*
3944 * If the last counter was a group counter, it will have appended all
3945 * its siblings to the list, but we obtained 'tmp' before that which
3946 * will still point to the list head terminating the iteration.
3947 */
3948 if (!list_empty(&child_ctx->counter_list))
3949 goto again;
3950
3951 mutex_unlock(&child_ctx->mutex);
3952
3953 put_ctx(child_ctx);
3954}
3955
3956/*
3957 * free an unexposed, unused context as created by inheritance by
3958 * init_task below, used by fork() in case of fail.
3959 */
3960void perf_counter_free_task(struct task_struct *task)
3961{
3962 struct perf_counter_context *ctx = task->perf_counter_ctxp;
3963 struct perf_counter *counter, *tmp;
3964
3965 if (!ctx)
3966 return;
3967
3968 mutex_lock(&ctx->mutex);
3969again:
3970 list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry) {
3971 struct perf_counter *parent = counter->parent;
3972
3973 if (WARN_ON_ONCE(!parent))
3974 continue;
3975
3976 mutex_lock(&parent->child_mutex);
3977 list_del_init(&counter->child_list);
3978 mutex_unlock(&parent->child_mutex);
3979
3980 fput(parent->filp);
3981
3982 list_del_counter(counter, ctx);
3983 free_counter(counter);
3984 }
3985
3986 if (!list_empty(&ctx->counter_list))
3987 goto again;
3988
3989 mutex_unlock(&ctx->mutex);
3990
3991 put_ctx(ctx);
3992}
3993
3994/*
3995 * Initialize the perf_counter context in task_struct
3996 */
3997int perf_counter_init_task(struct task_struct *child)
3998{
3999 struct perf_counter_context *child_ctx, *parent_ctx;
4000 struct perf_counter_context *cloned_ctx;
4001 struct perf_counter *counter;
4002 struct task_struct *parent = current;
4003 int inherited_all = 1;
4004 int ret = 0;
4005
4006 child->perf_counter_ctxp = NULL;
4007
4008 mutex_init(&child->perf_counter_mutex);
4009 INIT_LIST_HEAD(&child->perf_counter_list);
4010
4011 if (likely(!parent->perf_counter_ctxp))
4012 return 0;
4013
4014 /*
4015 * This is executed from the parent task context, so inherit
4016 * counters that have been marked for cloning.
4017 * First allocate and initialize a context for the child.
4018 */
4019
4020 child_ctx = kmalloc(sizeof(struct perf_counter_context), GFP_KERNEL);
4021 if (!child_ctx)
4022 return -ENOMEM;
4023
4024 __perf_counter_init_context(child_ctx, child);
4025 child->perf_counter_ctxp = child_ctx;
4026 get_task_struct(child);
4027
4028 /*
4029 * If the parent's context is a clone, pin it so it won't get
4030 * swapped under us.
4031 */
4032 parent_ctx = perf_pin_task_context(parent);
4033
4034 /*
4035 * No need to check if parent_ctx != NULL here; since we saw
4036 * it non-NULL earlier, the only reason for it to become NULL
4037 * is if we exit, and since we're currently in the middle of
4038 * a fork we can't be exiting at the same time.
4039 */
4040
4041 /*
4042 * Lock the parent list. No need to lock the child - not PID
4043 * hashed yet and not running, so nobody can access it.
4044 */
4045 mutex_lock(&parent_ctx->mutex);
4046
4047 /*
4048 * We dont have to disable NMIs - we are only looking at
4049 * the list, not manipulating it:
4050 */
4051 list_for_each_entry_rcu(counter, &parent_ctx->event_list, event_entry) {
4052 if (counter != counter->group_leader)
4053 continue;
4054
4055 if (!counter->attr.inherit) {
4056 inherited_all = 0;
4057 continue;
4058 }
4059
4060 ret = inherit_group(counter, parent, parent_ctx,
4061 child, child_ctx);
4062 if (ret) {
4063 inherited_all = 0;
4064 break;
4065 }
4066 }
4067
4068 if (inherited_all) {
4069 /*
4070 * Mark the child context as a clone of the parent
4071 * context, or of whatever the parent is a clone of.
4072 * Note that if the parent is a clone, it could get
4073 * uncloned at any point, but that doesn't matter
4074 * because the list of counters and the generation
4075 * count can't have changed since we took the mutex.
4076 */
4077 cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
4078 if (cloned_ctx) {
4079 child_ctx->parent_ctx = cloned_ctx;
4080 child_ctx->parent_gen = parent_ctx->parent_gen;
4081 } else {
4082 child_ctx->parent_ctx = parent_ctx;
4083 child_ctx->parent_gen = parent_ctx->generation;
4084 }
4085 get_ctx(child_ctx->parent_ctx);
4086 }
4087
4088 mutex_unlock(&parent_ctx->mutex);
4089
4090 perf_unpin_context(parent_ctx);
4091
4092 return ret;
4093}
4094
4095static void __cpuinit perf_counter_init_cpu(int cpu)
4096{
4097 struct perf_cpu_context *cpuctx;
4098
4099 cpuctx = &per_cpu(perf_cpu_context, cpu);
4100 __perf_counter_init_context(&cpuctx->ctx, NULL);
4101
4102 spin_lock(&perf_resource_lock);
4103 cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
4104 spin_unlock(&perf_resource_lock);
4105
4106 hw_perf_counter_setup(cpu);
4107}
4108
4109#ifdef CONFIG_HOTPLUG_CPU
4110static void __perf_counter_exit_cpu(void *info)
4111{
4112 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
4113 struct perf_counter_context *ctx = &cpuctx->ctx;
4114 struct perf_counter *counter, *tmp;
4115
4116 list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
4117 __perf_counter_remove_from_context(counter);
4118}
4119static void perf_counter_exit_cpu(int cpu)
4120{
4121 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
4122 struct perf_counter_context *ctx = &cpuctx->ctx;
4123
4124 mutex_lock(&ctx->mutex);
4125 smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
4126 mutex_unlock(&ctx->mutex);
4127}
4128#else
4129static inline void perf_counter_exit_cpu(int cpu) { }
4130#endif
4131
4132static int __cpuinit
4133perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
4134{
4135 unsigned int cpu = (long)hcpu;
4136
4137 switch (action) {
4138
4139 case CPU_UP_PREPARE:
4140 case CPU_UP_PREPARE_FROZEN:
4141 perf_counter_init_cpu(cpu);
4142 break;
4143
4144 case CPU_DOWN_PREPARE:
4145 case CPU_DOWN_PREPARE_FROZEN:
4146 perf_counter_exit_cpu(cpu);
4147 break;
4148
4149 default:
4150 break;
4151 }
4152
4153 return NOTIFY_OK;
4154}
4155
4156/*
4157 * This has to have a higher priority than migration_notifier in sched.c.
4158 */
4159static struct notifier_block __cpuinitdata perf_cpu_nb = {
4160 .notifier_call = perf_cpu_notify,
4161 .priority = 20,
4162};
4163
4164void __init perf_counter_init(void)
4165{
4166 perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
4167 (void *)(long)smp_processor_id());
4168 register_cpu_notifier(&perf_cpu_nb);
4169}
4170
4171static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
4172{
4173 return sprintf(buf, "%d\n", perf_reserved_percpu);
4174}
4175
4176static ssize_t
4177perf_set_reserve_percpu(struct sysdev_class *class,
4178 const char *buf,
4179 size_t count)
4180{
4181 struct perf_cpu_context *cpuctx;
4182 unsigned long val;
4183 int err, cpu, mpt;
4184
4185 err = strict_strtoul(buf, 10, &val);
4186 if (err)
4187 return err;
4188 if (val > perf_max_counters)
4189 return -EINVAL;
4190
4191 spin_lock(&perf_resource_lock);
4192 perf_reserved_percpu = val;
4193 for_each_online_cpu(cpu) {
4194 cpuctx = &per_cpu(perf_cpu_context, cpu);
4195 spin_lock_irq(&cpuctx->ctx.lock);
4196 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
4197 perf_max_counters - perf_reserved_percpu);
4198 cpuctx->max_pertask = mpt;
4199 spin_unlock_irq(&cpuctx->ctx.lock);
4200 }
4201 spin_unlock(&perf_resource_lock);
4202
4203 return count;
4204}
4205
4206static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
4207{
4208 return sprintf(buf, "%d\n", perf_overcommit);
4209}
4210
4211static ssize_t
4212perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
4213{
4214 unsigned long val;
4215 int err;
4216
4217 err = strict_strtoul(buf, 10, &val);
4218 if (err)
4219 return err;
4220 if (val > 1)
4221 return -EINVAL;
4222
4223 spin_lock(&perf_resource_lock);
4224 perf_overcommit = val;
4225 spin_unlock(&perf_resource_lock);
4226
4227 return count;
4228}
4229
4230static SYSDEV_CLASS_ATTR(
4231 reserve_percpu,
4232 0644,
4233 perf_show_reserve_percpu,
4234 perf_set_reserve_percpu
4235 );
4236
4237static SYSDEV_CLASS_ATTR(
4238 overcommit,
4239 0644,
4240 perf_show_overcommit,
4241 perf_set_overcommit
4242 );
4243
4244static struct attribute *perfclass_attrs[] = {
4245 &attr_reserve_percpu.attr,
4246 &attr_overcommit.attr,
4247 NULL
4248};
4249
4250static struct attribute_group perfclass_attr_group = {
4251 .attrs = perfclass_attrs,
4252 .name = "perf_counters",
4253};
4254
4255static int __init perf_counter_sysfs_init(void)
4256{
4257 return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
4258 &perfclass_attr_group);
4259}
4260device_initcall(perf_counter_sysfs_init);
diff --git a/kernel/sched.c b/kernel/sched.c
index 14c447ae5d53..5b3f6ec1b0b3 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -39,6 +39,7 @@
39#include <linux/completion.h> 39#include <linux/completion.h>
40#include <linux/kernel_stat.h> 40#include <linux/kernel_stat.h>
41#include <linux/debug_locks.h> 41#include <linux/debug_locks.h>
42#include <linux/perf_counter.h>
42#include <linux/security.h> 43#include <linux/security.h>
43#include <linux/notifier.h> 44#include <linux/notifier.h>
44#include <linux/profile.h> 45#include <linux/profile.h>
@@ -580,6 +581,7 @@ struct rq {
580 struct load_weight load; 581 struct load_weight load;
581 unsigned long nr_load_updates; 582 unsigned long nr_load_updates;
582 u64 nr_switches; 583 u64 nr_switches;
584 u64 nr_migrations_in;
583 585
584 struct cfs_rq cfs; 586 struct cfs_rq cfs;
585 struct rt_rq rt; 587 struct rt_rq rt;
@@ -692,7 +694,7 @@ static inline int cpu_of(struct rq *rq)
692#define task_rq(p) cpu_rq(task_cpu(p)) 694#define task_rq(p) cpu_rq(task_cpu(p))
693#define cpu_curr(cpu) (cpu_rq(cpu)->curr) 695#define cpu_curr(cpu) (cpu_rq(cpu)->curr)
694 696
695static inline void update_rq_clock(struct rq *rq) 697inline void update_rq_clock(struct rq *rq)
696{ 698{
697 rq->clock = sched_clock_cpu(cpu_of(rq)); 699 rq->clock = sched_clock_cpu(cpu_of(rq));
698} 700}
@@ -1969,12 +1971,16 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
1969 p->se.sleep_start -= clock_offset; 1971 p->se.sleep_start -= clock_offset;
1970 if (p->se.block_start) 1972 if (p->se.block_start)
1971 p->se.block_start -= clock_offset; 1973 p->se.block_start -= clock_offset;
1974#endif
1972 if (old_cpu != new_cpu) { 1975 if (old_cpu != new_cpu) {
1973 schedstat_inc(p, se.nr_migrations); 1976 p->se.nr_migrations++;
1977 new_rq->nr_migrations_in++;
1978#ifdef CONFIG_SCHEDSTATS
1974 if (task_hot(p, old_rq->clock, NULL)) 1979 if (task_hot(p, old_rq->clock, NULL))
1975 schedstat_inc(p, se.nr_forced2_migrations); 1980 schedstat_inc(p, se.nr_forced2_migrations);
1976 }
1977#endif 1981#endif
1982 perf_counter_task_migration(p, new_cpu);
1983 }
1978 p->se.vruntime -= old_cfsrq->min_vruntime - 1984 p->se.vruntime -= old_cfsrq->min_vruntime -
1979 new_cfsrq->min_vruntime; 1985 new_cfsrq->min_vruntime;
1980 1986
@@ -2369,6 +2375,27 @@ static int sched_balance_self(int cpu, int flag)
2369 2375
2370#endif /* CONFIG_SMP */ 2376#endif /* CONFIG_SMP */
2371 2377
2378/**
2379 * task_oncpu_function_call - call a function on the cpu on which a task runs
2380 * @p: the task to evaluate
2381 * @func: the function to be called
2382 * @info: the function call argument
2383 *
2384 * Calls the function @func when the task is currently running. This might
2385 * be on the current CPU, which just calls the function directly
2386 */
2387void task_oncpu_function_call(struct task_struct *p,
2388 void (*func) (void *info), void *info)
2389{
2390 int cpu;
2391
2392 preempt_disable();
2393 cpu = task_cpu(p);
2394 if (task_curr(p))
2395 smp_call_function_single(cpu, func, info, 1);
2396 preempt_enable();
2397}
2398
2372/*** 2399/***
2373 * try_to_wake_up - wake up a thread 2400 * try_to_wake_up - wake up a thread
2374 * @p: the to-be-woken-up thread 2401 * @p: the to-be-woken-up thread
@@ -2536,6 +2563,7 @@ static void __sched_fork(struct task_struct *p)
2536 p->se.exec_start = 0; 2563 p->se.exec_start = 0;
2537 p->se.sum_exec_runtime = 0; 2564 p->se.sum_exec_runtime = 0;
2538 p->se.prev_sum_exec_runtime = 0; 2565 p->se.prev_sum_exec_runtime = 0;
2566 p->se.nr_migrations = 0;
2539 p->se.last_wakeup = 0; 2567 p->se.last_wakeup = 0;
2540 p->se.avg_overlap = 0; 2568 p->se.avg_overlap = 0;
2541 p->se.start_runtime = 0; 2569 p->se.start_runtime = 0;
@@ -2766,6 +2794,7 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
2766 */ 2794 */
2767 prev_state = prev->state; 2795 prev_state = prev->state;
2768 finish_arch_switch(prev); 2796 finish_arch_switch(prev);
2797 perf_counter_task_sched_in(current, cpu_of(rq));
2769 finish_lock_switch(rq, prev); 2798 finish_lock_switch(rq, prev);
2770#ifdef CONFIG_SMP 2799#ifdef CONFIG_SMP
2771 if (post_schedule) 2800 if (post_schedule)
@@ -2981,6 +3010,15 @@ static void calc_load_account_active(struct rq *this_rq)
2981} 3010}
2982 3011
2983/* 3012/*
3013 * Externally visible per-cpu scheduler statistics:
3014 * cpu_nr_migrations(cpu) - number of migrations into that cpu
3015 */
3016u64 cpu_nr_migrations(int cpu)
3017{
3018 return cpu_rq(cpu)->nr_migrations_in;
3019}
3020
3021/*
2984 * Update rq->cpu_load[] statistics. This function is usually called every 3022 * Update rq->cpu_load[] statistics. This function is usually called every
2985 * scheduler tick (TICK_NSEC). 3023 * scheduler tick (TICK_NSEC).
2986 */ 3024 */
@@ -5078,6 +5116,8 @@ void scheduler_tick(void)
5078 curr->sched_class->task_tick(rq, curr, 0); 5116 curr->sched_class->task_tick(rq, curr, 0);
5079 spin_unlock(&rq->lock); 5117 spin_unlock(&rq->lock);
5080 5118
5119 perf_counter_task_tick(curr, cpu);
5120
5081#ifdef CONFIG_SMP 5121#ifdef CONFIG_SMP
5082 rq->idle_at_tick = idle_cpu(cpu); 5122 rq->idle_at_tick = idle_cpu(cpu);
5083 trigger_load_balance(rq, cpu); 5123 trigger_load_balance(rq, cpu);
@@ -5293,6 +5333,7 @@ need_resched_nonpreemptible:
5293 5333
5294 if (likely(prev != next)) { 5334 if (likely(prev != next)) {
5295 sched_info_switch(prev, next); 5335 sched_info_switch(prev, next);
5336 perf_counter_task_sched_out(prev, next, cpu);
5296 5337
5297 rq->nr_switches++; 5338 rq->nr_switches++;
5298 rq->curr = next; 5339 rq->curr = next;
@@ -7536,8 +7577,10 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
7536 return NOTIFY_OK; 7577 return NOTIFY_OK;
7537} 7578}
7538 7579
7539/* Register at highest priority so that task migration (migrate_all_tasks) 7580/*
7540 * happens before everything else. 7581 * Register at high priority so that task migration (migrate_all_tasks)
7582 * happens before everything else. This has to be lower priority than
7583 * the notifier in the perf_counter subsystem, though.
7541 */ 7584 */
7542static struct notifier_block __cpuinitdata migration_notifier = { 7585static struct notifier_block __cpuinitdata migration_notifier = {
7543 .notifier_call = migration_call, 7586 .notifier_call = migration_call,
@@ -9218,7 +9261,7 @@ void __init sched_init(void)
9218 * 1024) and two child groups A0 and A1 (of weight 1024 each), 9261 * 1024) and two child groups A0 and A1 (of weight 1024 each),
9219 * then A0's share of the cpu resource is: 9262 * then A0's share of the cpu resource is:
9220 * 9263 *
9221 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% 9264 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
9222 * 9265 *
9223 * We achieve this by letting init_task_group's tasks sit 9266 * We achieve this by letting init_task_group's tasks sit
9224 * directly in rq->cfs (i.e init_task_group->se[] = NULL). 9267 * directly in rq->cfs (i.e init_task_group->se[] = NULL).
@@ -9323,6 +9366,8 @@ void __init sched_init(void)
9323 alloc_bootmem_cpumask_var(&cpu_isolated_map); 9366 alloc_bootmem_cpumask_var(&cpu_isolated_map);
9324#endif /* SMP */ 9367#endif /* SMP */
9325 9368
9369 perf_counter_init();
9370
9326 scheduler_running = 1; 9371 scheduler_running = 1;
9327} 9372}
9328 9373
diff --git a/kernel/sys.c b/kernel/sys.c
index e7998cf31498..438d99a38c87 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -14,6 +14,7 @@
14#include <linux/prctl.h> 14#include <linux/prctl.h>
15#include <linux/highuid.h> 15#include <linux/highuid.h>
16#include <linux/fs.h> 16#include <linux/fs.h>
17#include <linux/perf_counter.h>
17#include <linux/resource.h> 18#include <linux/resource.h>
18#include <linux/kernel.h> 19#include <linux/kernel.h>
19#include <linux/kexec.h> 20#include <linux/kexec.h>
@@ -1793,6 +1794,12 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
1793 case PR_SET_TSC: 1794 case PR_SET_TSC:
1794 error = SET_TSC_CTL(arg2); 1795 error = SET_TSC_CTL(arg2);
1795 break; 1796 break;
1797 case PR_TASK_PERF_COUNTERS_DISABLE:
1798 error = perf_counter_task_disable();
1799 break;
1800 case PR_TASK_PERF_COUNTERS_ENABLE:
1801 error = perf_counter_task_enable();
1802 break;
1796 case PR_GET_TIMERSLACK: 1803 case PR_GET_TIMERSLACK:
1797 error = current->timer_slack_ns; 1804 error = current->timer_slack_ns;
1798 break; 1805 break;
diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c
index 27dad2967387..68320f6b07b5 100644
--- a/kernel/sys_ni.c
+++ b/kernel/sys_ni.c
@@ -175,3 +175,6 @@ cond_syscall(compat_sys_timerfd_settime);
175cond_syscall(compat_sys_timerfd_gettime); 175cond_syscall(compat_sys_timerfd_gettime);
176cond_syscall(sys_eventfd); 176cond_syscall(sys_eventfd);
177cond_syscall(sys_eventfd2); 177cond_syscall(sys_eventfd2);
178
179/* performance counters: */
180cond_syscall(sys_perf_counter_open);
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 6a463716ecbf..a7e4eb0525b1 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -49,6 +49,7 @@
49#include <linux/reboot.h> 49#include <linux/reboot.h>
50#include <linux/ftrace.h> 50#include <linux/ftrace.h>
51#include <linux/slow-work.h> 51#include <linux/slow-work.h>
52#include <linux/perf_counter.h>
52 53
53#include <asm/uaccess.h> 54#include <asm/uaccess.h>
54#include <asm/processor.h> 55#include <asm/processor.h>
@@ -920,6 +921,32 @@ static struct ctl_table kern_table[] = {
920 .child = slow_work_sysctls, 921 .child = slow_work_sysctls,
921 }, 922 },
922#endif 923#endif
924#ifdef CONFIG_PERF_COUNTERS
925 {
926 .ctl_name = CTL_UNNUMBERED,
927 .procname = "perf_counter_paranoid",
928 .data = &sysctl_perf_counter_paranoid,
929 .maxlen = sizeof(sysctl_perf_counter_paranoid),
930 .mode = 0644,
931 .proc_handler = &proc_dointvec,
932 },
933 {
934 .ctl_name = CTL_UNNUMBERED,
935 .procname = "perf_counter_mlock_kb",
936 .data = &sysctl_perf_counter_mlock,
937 .maxlen = sizeof(sysctl_perf_counter_mlock),
938 .mode = 0644,
939 .proc_handler = &proc_dointvec,
940 },
941 {
942 .ctl_name = CTL_UNNUMBERED,
943 .procname = "perf_counter_max_sample_rate",
944 .data = &sysctl_perf_counter_sample_rate,
945 .maxlen = sizeof(sysctl_perf_counter_sample_rate),
946 .mode = 0644,
947 .proc_handler = &proc_dointvec,
948 },
949#endif
923/* 950/*
924 * NOTE: do not add new entries to this table unless you have read 951 * NOTE: do not add new entries to this table unless you have read
925 * Documentation/sysctl/ctl_unnumbered.txt 952 * Documentation/sysctl/ctl_unnumbered.txt
diff --git a/kernel/timer.c b/kernel/timer.c
index a26ed294f938..c01e568935ea 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -37,6 +37,7 @@
37#include <linux/delay.h> 37#include <linux/delay.h>
38#include <linux/tick.h> 38#include <linux/tick.h>
39#include <linux/kallsyms.h> 39#include <linux/kallsyms.h>
40#include <linux/perf_counter.h>
40 41
41#include <asm/uaccess.h> 42#include <asm/uaccess.h>
42#include <asm/unistd.h> 43#include <asm/unistd.h>
@@ -1129,6 +1130,8 @@ static void run_timer_softirq(struct softirq_action *h)
1129{ 1130{
1130 struct tvec_base *base = __get_cpu_var(tvec_bases); 1131 struct tvec_base *base = __get_cpu_var(tvec_bases);
1131 1132
1133 perf_counter_do_pending();
1134
1132 hrtimer_run_pending(); 1135 hrtimer_run_pending();
1133 1136
1134 if (time_after_eq(jiffies, base->timer_jiffies)) 1137 if (time_after_eq(jiffies, base->timer_jiffies))
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-28 09:46:07 -0500