aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/events
diff options
context:
space:
mode:
authorPeter Zijlstra <a.p.zijlstra@chello.nl>2011-04-09 15:17:42 -0400
committerIngo Molnar <mingo@elte.hu>2011-05-28 12:01:12 -0400
commitfacc43071cc0d4821c176d7d34570714eb348df9 (patch)
tree0dd46f10d70f7814d622f1862ba6bf1c1cf4e18d /kernel/events
parent9137fb28ac74d05eb66d1d8e6778eaa14e6fed43 (diff)
perf: Optimize event scheduling locking
Currently we only hold one ctx->lock at a time, which results in us flipping back and forth between cpuctx->ctx.lock and task_ctx->lock. Avoid this and gain large atomic regions by holding both locks. We nest the task lock inside the cpu lock, since with task scheduling we might have to change task ctx while holding the cpu ctx lock. Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Link: http://lkml.kernel.org/r/20110409192141.769881865@chello.nl Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'kernel/events')
-rw-r--r--kernel/events/core.c61
1 files changed, 36 insertions, 25 deletions
diff --git a/kernel/events/core.c b/kernel/events/core.c
index d665ac4242f2..d243af954dcc 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -200,6 +200,22 @@ __get_cpu_context(struct perf_event_context *ctx)
200 return this_cpu_ptr(ctx->pmu->pmu_cpu_context); 200 return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
201} 201}
202 202
203static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
204 struct perf_event_context *ctx)
205{
206 raw_spin_lock(&cpuctx->ctx.lock);
207 if (ctx)
208 raw_spin_lock(&ctx->lock);
209}
210
211static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
212 struct perf_event_context *ctx)
213{
214 if (ctx)
215 raw_spin_unlock(&ctx->lock);
216 raw_spin_unlock(&cpuctx->ctx.lock);
217}
218
203#ifdef CONFIG_CGROUP_PERF 219#ifdef CONFIG_CGROUP_PERF
204 220
205/* 221/*
@@ -340,11 +356,8 @@ void perf_cgroup_switch(struct task_struct *task, int mode)
340 rcu_read_lock(); 356 rcu_read_lock();
341 357
342 list_for_each_entry_rcu(pmu, &pmus, entry) { 358 list_for_each_entry_rcu(pmu, &pmus, entry) {
343
344 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 359 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
345 360
346 perf_pmu_disable(cpuctx->ctx.pmu);
347
348 /* 361 /*
349 * perf_cgroup_events says at least one 362 * perf_cgroup_events says at least one
350 * context on this CPU has cgroup events. 363 * context on this CPU has cgroup events.
@@ -353,6 +366,8 @@ void perf_cgroup_switch(struct task_struct *task, int mode)
353 * events for a context. 366 * events for a context.
354 */ 367 */
355 if (cpuctx->ctx.nr_cgroups > 0) { 368 if (cpuctx->ctx.nr_cgroups > 0) {
369 perf_ctx_lock(cpuctx, cpuctx->task_ctx);
370 perf_pmu_disable(cpuctx->ctx.pmu);
356 371
357 if (mode & PERF_CGROUP_SWOUT) { 372 if (mode & PERF_CGROUP_SWOUT) {
358 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 373 cpu_ctx_sched_out(cpuctx, EVENT_ALL);
@@ -372,9 +387,9 @@ void perf_cgroup_switch(struct task_struct *task, int mode)
372 cpuctx->cgrp = perf_cgroup_from_task(task); 387 cpuctx->cgrp = perf_cgroup_from_task(task);
373 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task); 388 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
374 } 389 }
390 perf_pmu_enable(cpuctx->ctx.pmu);
391 perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
375 } 392 }
376
377 perf_pmu_enable(cpuctx->ctx.pmu);
378 } 393 }
379 394
380 rcu_read_unlock(); 395 rcu_read_unlock();
@@ -1759,15 +1774,14 @@ static void ctx_sched_out(struct perf_event_context *ctx,
1759{ 1774{
1760 struct perf_event *event; 1775 struct perf_event *event;
1761 1776
1762 raw_spin_lock(&ctx->lock);
1763 ctx->is_active = 0; 1777 ctx->is_active = 0;
1764 if (likely(!ctx->nr_events)) 1778 if (likely(!ctx->nr_events))
1765 goto out; 1779 return;
1780
1766 update_context_time(ctx); 1781 update_context_time(ctx);
1767 update_cgrp_time_from_cpuctx(cpuctx); 1782 update_cgrp_time_from_cpuctx(cpuctx);
1768
1769 if (!ctx->nr_active) 1783 if (!ctx->nr_active)
1770 goto out; 1784 return;
1771 1785
1772 perf_pmu_disable(ctx->pmu); 1786 perf_pmu_disable(ctx->pmu);
1773 if (event_type & EVENT_PINNED) { 1787 if (event_type & EVENT_PINNED) {
@@ -1780,8 +1794,6 @@ static void ctx_sched_out(struct perf_event_context *ctx,
1780 group_sched_out(event, cpuctx, ctx); 1794 group_sched_out(event, cpuctx, ctx);
1781 } 1795 }
1782 perf_pmu_enable(ctx->pmu); 1796 perf_pmu_enable(ctx->pmu);
1783out:
1784 raw_spin_unlock(&ctx->lock);
1785} 1797}
1786 1798
1787/* 1799/*
@@ -1929,8 +1941,10 @@ static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
1929 rcu_read_unlock(); 1941 rcu_read_unlock();
1930 1942
1931 if (do_switch) { 1943 if (do_switch) {
1944 raw_spin_lock(&ctx->lock);
1932 ctx_sched_out(ctx, cpuctx, EVENT_ALL); 1945 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
1933 cpuctx->task_ctx = NULL; 1946 cpuctx->task_ctx = NULL;
1947 raw_spin_unlock(&ctx->lock);
1934 } 1948 }
1935} 1949}
1936 1950
@@ -2056,10 +2070,9 @@ ctx_sched_in(struct perf_event_context *ctx,
2056{ 2070{
2057 u64 now; 2071 u64 now;
2058 2072
2059 raw_spin_lock(&ctx->lock);
2060 ctx->is_active = 1; 2073 ctx->is_active = 1;
2061 if (likely(!ctx->nr_events)) 2074 if (likely(!ctx->nr_events))
2062 goto out; 2075 return;
2063 2076
2064 now = perf_clock(); 2077 now = perf_clock();
2065 ctx->timestamp = now; 2078 ctx->timestamp = now;
@@ -2074,9 +2087,6 @@ ctx_sched_in(struct perf_event_context *ctx,
2074 /* Then walk through the lower prio flexible groups */ 2087 /* Then walk through the lower prio flexible groups */
2075 if (event_type & EVENT_FLEXIBLE) 2088 if (event_type & EVENT_FLEXIBLE)
2076 ctx_flexible_sched_in(ctx, cpuctx); 2089 ctx_flexible_sched_in(ctx, cpuctx);
2077
2078out:
2079 raw_spin_unlock(&ctx->lock);
2080} 2090}
2081 2091
2082static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 2092static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
@@ -2110,6 +2120,7 @@ static void perf_event_context_sched_in(struct perf_event_context *ctx,
2110 if (cpuctx->task_ctx == ctx) 2120 if (cpuctx->task_ctx == ctx)
2111 return; 2121 return;
2112 2122
2123 perf_ctx_lock(cpuctx, ctx);
2113 perf_pmu_disable(ctx->pmu); 2124 perf_pmu_disable(ctx->pmu);
2114 /* 2125 /*
2115 * We want to keep the following priority order: 2126 * We want to keep the following priority order:
@@ -2124,12 +2135,14 @@ static void perf_event_context_sched_in(struct perf_event_context *ctx,
2124 2135
2125 cpuctx->task_ctx = ctx; 2136 cpuctx->task_ctx = ctx;
2126 2137
2138 perf_pmu_enable(ctx->pmu);
2139 perf_ctx_unlock(cpuctx, ctx);
2140
2127 /* 2141 /*
2128 * Since these rotations are per-cpu, we need to ensure the 2142 * Since these rotations are per-cpu, we need to ensure the
2129 * cpu-context we got scheduled on is actually rotating. 2143 * cpu-context we got scheduled on is actually rotating.
2130 */ 2144 */
2131 perf_pmu_rotate_start(ctx->pmu); 2145 perf_pmu_rotate_start(ctx->pmu);
2132 perf_pmu_enable(ctx->pmu);
2133} 2146}
2134 2147
2135/* 2148/*
@@ -2269,7 +2282,6 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx, u64 period)
2269 u64 interrupts, now; 2282 u64 interrupts, now;
2270 s64 delta; 2283 s64 delta;
2271 2284
2272 raw_spin_lock(&ctx->lock);
2273 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 2285 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
2274 if (event->state != PERF_EVENT_STATE_ACTIVE) 2286 if (event->state != PERF_EVENT_STATE_ACTIVE)
2275 continue; 2287 continue;
@@ -2301,7 +2313,6 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx, u64 period)
2301 if (delta > 0) 2313 if (delta > 0)
2302 perf_adjust_period(event, period, delta); 2314 perf_adjust_period(event, period, delta);
2303 } 2315 }
2304 raw_spin_unlock(&ctx->lock);
2305} 2316}
2306 2317
2307/* 2318/*
@@ -2309,16 +2320,12 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx, u64 period)
2309 */ 2320 */
2310static void rotate_ctx(struct perf_event_context *ctx) 2321static void rotate_ctx(struct perf_event_context *ctx)
2311{ 2322{
2312 raw_spin_lock(&ctx->lock);
2313
2314 /* 2323 /*
2315 * Rotate the first entry last of non-pinned groups. Rotation might be 2324 * Rotate the first entry last of non-pinned groups. Rotation might be
2316 * disabled by the inheritance code. 2325 * disabled by the inheritance code.
2317 */ 2326 */
2318 if (!ctx->rotate_disable) 2327 if (!ctx->rotate_disable)
2319 list_rotate_left(&ctx->flexible_groups); 2328 list_rotate_left(&ctx->flexible_groups);
2320
2321 raw_spin_unlock(&ctx->lock);
2322} 2329}
2323 2330
2324/* 2331/*
@@ -2345,6 +2352,7 @@ static void perf_rotate_context(struct perf_cpu_context *cpuctx)
2345 rotate = 1; 2352 rotate = 1;
2346 } 2353 }
2347 2354
2355 perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2348 perf_pmu_disable(cpuctx->ctx.pmu); 2356 perf_pmu_disable(cpuctx->ctx.pmu);
2349 perf_ctx_adjust_freq(&cpuctx->ctx, interval); 2357 perf_ctx_adjust_freq(&cpuctx->ctx, interval);
2350 if (ctx) 2358 if (ctx)
@@ -2370,6 +2378,7 @@ done:
2370 list_del_init(&cpuctx->rotation_list); 2378 list_del_init(&cpuctx->rotation_list);
2371 2379
2372 perf_pmu_enable(cpuctx->ctx.pmu); 2380 perf_pmu_enable(cpuctx->ctx.pmu);
2381 perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2373} 2382}
2374 2383
2375void perf_event_task_tick(void) 2384void perf_event_task_tick(void)
@@ -2424,9 +2433,9 @@ static void perf_event_enable_on_exec(struct perf_event_context *ctx)
2424 * in. 2433 * in.
2425 */ 2434 */
2426 perf_cgroup_sched_out(current); 2435 perf_cgroup_sched_out(current);
2427 task_ctx_sched_out(ctx, EVENT_ALL);
2428 2436
2429 raw_spin_lock(&ctx->lock); 2437 raw_spin_lock(&ctx->lock);
2438 task_ctx_sched_out(ctx, EVENT_ALL);
2430 2439
2431 list_for_each_entry(event, &ctx->pinned_groups, group_entry) { 2440 list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2432 ret = event_enable_on_exec(event, ctx); 2441 ret = event_enable_on_exec(event, ctx);
@@ -5982,6 +5991,7 @@ free_dev:
5982} 5991}
5983 5992
5984static struct lock_class_key cpuctx_mutex; 5993static struct lock_class_key cpuctx_mutex;
5994static struct lock_class_key cpuctx_lock;
5985 5995
5986int perf_pmu_register(struct pmu *pmu, char *name, int type) 5996int perf_pmu_register(struct pmu *pmu, char *name, int type)
5987{ 5997{
@@ -6032,6 +6042,7 @@ skip_type:
6032 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 6042 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
6033 __perf_event_init_context(&cpuctx->ctx); 6043 __perf_event_init_context(&cpuctx->ctx);
6034 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex); 6044 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
6045 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
6035 cpuctx->ctx.type = cpu_context; 6046 cpuctx->ctx.type = cpu_context;
6036 cpuctx->ctx.pmu = pmu; 6047 cpuctx->ctx.pmu = pmu;
6037 cpuctx->jiffies_interval = 1; 6048 cpuctx->jiffies_interval = 1;
@@ -6776,7 +6787,6 @@ static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
6776 * our context. 6787 * our context.
6777 */ 6788 */
6778 child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]); 6789 child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
6779 task_ctx_sched_out(child_ctx, EVENT_ALL);
6780 6790
6781 /* 6791 /*
6782 * Take the context lock here so that if find_get_context is 6792 * Take the context lock here so that if find_get_context is
@@ -6784,6 +6794,7 @@ static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
6784 * incremented the context's refcount before we do put_ctx below. 6794 * incremented the context's refcount before we do put_ctx below.
6785 */ 6795 */
6786 raw_spin_lock(&child_ctx->lock); 6796 raw_spin_lock(&child_ctx->lock);
6797 task_ctx_sched_out(child_ctx, EVENT_ALL);
6787 child->perf_event_ctxp[ctxn] = NULL; 6798 child->perf_event_ctxp[ctxn] = NULL;
6788 /* 6799 /*
6789 * If this context is a clone; unclone it so it can't get 6800 * If this context is a clone; unclone it so it can't get