diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/exit.c | 1 | ||||
| -rw-r--r-- | kernel/fork.c | 1 | ||||
| -rw-r--r-- | kernel/freezer.c | 7 | ||||
| -rw-r--r-- | kernel/futex.c | 1 | ||||
| -rw-r--r-- | kernel/hrtimer.c | 110 | ||||
| -rw-r--r-- | kernel/irq/internals.h | 3 | ||||
| -rw-r--r-- | kernel/irq/manage.c | 50 | ||||
| -rw-r--r-- | kernel/irq/migration.c | 2 | ||||
| -rw-r--r-- | kernel/kmod.c | 1 | ||||
| -rw-r--r-- | kernel/kprobes.c | 6 | ||||
| -rw-r--r-- | kernel/module.c | 6 | ||||
| -rw-r--r-- | kernel/perf_counter.c | 2 | ||||
| -rw-r--r-- | kernel/pid.c | 7 | ||||
| -rw-r--r-- | kernel/power/user.c | 1 | ||||
| -rw-r--r-- | kernel/ptrace.c | 4 | ||||
| -rw-r--r-- | kernel/rcutree.c | 3 | ||||
| -rw-r--r-- | kernel/sched.c | 61 | ||||
| -rw-r--r-- | kernel/sched_fair.c | 13 | ||||
| -rw-r--r-- | kernel/sched_rt.c | 18 | ||||
| -rw-r--r-- | kernel/softirq.c | 64 | ||||
| -rw-r--r-- | kernel/time/clockevents.c | 11 | ||||
| -rw-r--r-- | kernel/time/clocksource.c | 2 | ||||
| -rw-r--r-- | kernel/timer.c | 2 | ||||
| -rw-r--r-- | kernel/trace/Kconfig | 6 | ||||
| -rw-r--r-- | kernel/trace/blktrace.c | 1 | ||||
| -rw-r--r-- | kernel/trace/ftrace.c | 9 | ||||
| -rw-r--r-- | kernel/trace/trace.c | 1 | ||||
| -rw-r--r-- | kernel/trace/trace_event_types.h | 3 | ||||
| -rw-r--r-- | kernel/trace/trace_functions.c | 2 | ||||
| -rw-r--r-- | kernel/trace/trace_output.c | 3 | ||||
| -rw-r--r-- | kernel/trace/trace_stack.c | 4 |
31 files changed, 278 insertions, 127 deletions
diff --git a/kernel/exit.c b/kernel/exit.c index 628d41f0dd54..869dc221733e 100644 --- a/kernel/exit.c +++ b/kernel/exit.c | |||
| @@ -12,7 +12,6 @@ | |||
| 12 | #include <linux/completion.h> | 12 | #include <linux/completion.h> |
| 13 | #include <linux/personality.h> | 13 | #include <linux/personality.h> |
| 14 | #include <linux/tty.h> | 14 | #include <linux/tty.h> |
| 15 | #include <linux/mnt_namespace.h> | ||
| 16 | #include <linux/iocontext.h> | 15 | #include <linux/iocontext.h> |
| 17 | #include <linux/key.h> | 16 | #include <linux/key.h> |
| 18 | #include <linux/security.h> | 17 | #include <linux/security.h> |
diff --git a/kernel/fork.c b/kernel/fork.c index 4812d60b29f8..9b42695f0d14 100644 --- a/kernel/fork.c +++ b/kernel/fork.c | |||
| @@ -17,7 +17,6 @@ | |||
| 17 | #include <linux/module.h> | 17 | #include <linux/module.h> |
| 18 | #include <linux/vmalloc.h> | 18 | #include <linux/vmalloc.h> |
| 19 | #include <linux/completion.h> | 19 | #include <linux/completion.h> |
| 20 | #include <linux/mnt_namespace.h> | ||
| 21 | #include <linux/personality.h> | 20 | #include <linux/personality.h> |
| 22 | #include <linux/mempolicy.h> | 21 | #include <linux/mempolicy.h> |
| 23 | #include <linux/sem.h> | 22 | #include <linux/sem.h> |
diff --git a/kernel/freezer.c b/kernel/freezer.c index 2f4936cf7083..bd1d42b17cb2 100644 --- a/kernel/freezer.c +++ b/kernel/freezer.c | |||
| @@ -44,12 +44,19 @@ void refrigerator(void) | |||
| 44 | recalc_sigpending(); /* We sent fake signal, clean it up */ | 44 | recalc_sigpending(); /* We sent fake signal, clean it up */ |
| 45 | spin_unlock_irq(¤t->sighand->siglock); | 45 | spin_unlock_irq(¤t->sighand->siglock); |
| 46 | 46 | ||
| 47 | /* prevent accounting of that task to load */ | ||
| 48 | current->flags |= PF_FREEZING; | ||
| 49 | |||
| 47 | for (;;) { | 50 | for (;;) { |
| 48 | set_current_state(TASK_UNINTERRUPTIBLE); | 51 | set_current_state(TASK_UNINTERRUPTIBLE); |
| 49 | if (!frozen(current)) | 52 | if (!frozen(current)) |
| 50 | break; | 53 | break; |
| 51 | schedule(); | 54 | schedule(); |
| 52 | } | 55 | } |
| 56 | |||
| 57 | /* Remove the accounting blocker */ | ||
| 58 | current->flags &= ~PF_FREEZING; | ||
| 59 | |||
| 53 | pr_debug("%s left refrigerator\n", current->comm); | 60 | pr_debug("%s left refrigerator\n", current->comm); |
| 54 | __set_current_state(save); | 61 | __set_current_state(save); |
| 55 | } | 62 | } |
diff --git a/kernel/futex.c b/kernel/futex.c index 794c862125fe..0672ff88f159 100644 --- a/kernel/futex.c +++ b/kernel/futex.c | |||
| @@ -247,6 +247,7 @@ again: | |||
| 247 | if (err < 0) | 247 | if (err < 0) |
| 248 | return err; | 248 | return err; |
| 249 | 249 | ||
| 250 | page = compound_head(page); | ||
| 250 | lock_page(page); | 251 | lock_page(page); |
| 251 | if (!page->mapping) { | 252 | if (!page->mapping) { |
| 252 | unlock_page(page); | 253 | unlock_page(page); |
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 9002958a96e7..49da79ab8486 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c | |||
| @@ -191,6 +191,46 @@ struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, | |||
| 191 | } | 191 | } |
| 192 | } | 192 | } |
| 193 | 193 | ||
| 194 | |||
| 195 | /* | ||
| 196 | * Get the preferred target CPU for NOHZ | ||
| 197 | */ | ||
| 198 | static int hrtimer_get_target(int this_cpu, int pinned) | ||
| 199 | { | ||
| 200 | #ifdef CONFIG_NO_HZ | ||
| 201 | if (!pinned && get_sysctl_timer_migration() && idle_cpu(this_cpu)) { | ||
| 202 | int preferred_cpu = get_nohz_load_balancer(); | ||
| 203 | |||
| 204 | if (preferred_cpu >= 0) | ||
| 205 | return preferred_cpu; | ||
| 206 | } | ||
| 207 | #endif | ||
| 208 | return this_cpu; | ||
| 209 | } | ||
| 210 | |||
| 211 | /* | ||
| 212 | * With HIGHRES=y we do not migrate the timer when it is expiring | ||
| 213 | * before the next event on the target cpu because we cannot reprogram | ||
| 214 | * the target cpu hardware and we would cause it to fire late. | ||
| 215 | * | ||
| 216 | * Called with cpu_base->lock of target cpu held. | ||
| 217 | */ | ||
| 218 | static int | ||
| 219 | hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base) | ||
| 220 | { | ||
| 221 | #ifdef CONFIG_HIGH_RES_TIMERS | ||
| 222 | ktime_t expires; | ||
| 223 | |||
| 224 | if (!new_base->cpu_base->hres_active) | ||
| 225 | return 0; | ||
| 226 | |||
| 227 | expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset); | ||
| 228 | return expires.tv64 <= new_base->cpu_base->expires_next.tv64; | ||
| 229 | #else | ||
| 230 | return 0; | ||
| 231 | #endif | ||
| 232 | } | ||
| 233 | |||
| 194 | /* | 234 | /* |
| 195 | * Switch the timer base to the current CPU when possible. | 235 | * Switch the timer base to the current CPU when possible. |
| 196 | */ | 236 | */ |
| @@ -200,16 +240,8 @@ switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base, | |||
| 200 | { | 240 | { |
| 201 | struct hrtimer_clock_base *new_base; | 241 | struct hrtimer_clock_base *new_base; |
| 202 | struct hrtimer_cpu_base *new_cpu_base; | 242 | struct hrtimer_cpu_base *new_cpu_base; |
| 203 | int cpu, preferred_cpu = -1; | 243 | int this_cpu = smp_processor_id(); |
| 204 | 244 | int cpu = hrtimer_get_target(this_cpu, pinned); | |
| 205 | cpu = smp_processor_id(); | ||
| 206 | #if defined(CONFIG_NO_HZ) && defined(CONFIG_SMP) | ||
| 207 | if (!pinned && get_sysctl_timer_migration() && idle_cpu(cpu)) { | ||
| 208 | preferred_cpu = get_nohz_load_balancer(); | ||
| 209 | if (preferred_cpu >= 0) | ||
| 210 | cpu = preferred_cpu; | ||
| 211 | } | ||
| 212 | #endif | ||
| 213 | 245 | ||
| 214 | again: | 246 | again: |
| 215 | new_cpu_base = &per_cpu(hrtimer_bases, cpu); | 247 | new_cpu_base = &per_cpu(hrtimer_bases, cpu); |
| @@ -217,7 +249,7 @@ again: | |||
| 217 | 249 | ||
| 218 | if (base != new_base) { | 250 | if (base != new_base) { |
| 219 | /* | 251 | /* |
| 220 | * We are trying to schedule the timer on the local CPU. | 252 | * We are trying to move timer to new_base. |
| 221 | * However we can't change timer's base while it is running, | 253 | * However we can't change timer's base while it is running, |
| 222 | * so we keep it on the same CPU. No hassle vs. reprogramming | 254 | * so we keep it on the same CPU. No hassle vs. reprogramming |
| 223 | * the event source in the high resolution case. The softirq | 255 | * the event source in the high resolution case. The softirq |
| @@ -233,38 +265,12 @@ again: | |||
| 233 | spin_unlock(&base->cpu_base->lock); | 265 | spin_unlock(&base->cpu_base->lock); |
| 234 | spin_lock(&new_base->cpu_base->lock); | 266 | spin_lock(&new_base->cpu_base->lock); |
| 235 | 267 | ||
| 236 | /* Optimized away for NOHZ=n SMP=n */ | 268 | if (cpu != this_cpu && hrtimer_check_target(timer, new_base)) { |
| 237 | if (cpu == preferred_cpu) { | 269 | cpu = this_cpu; |
| 238 | /* Calculate clock monotonic expiry time */ | 270 | spin_unlock(&new_base->cpu_base->lock); |
| 239 | #ifdef CONFIG_HIGH_RES_TIMERS | 271 | spin_lock(&base->cpu_base->lock); |
| 240 | ktime_t expires = ktime_sub(hrtimer_get_expires(timer), | 272 | timer->base = base; |
| 241 | new_base->offset); | 273 | goto again; |
| 242 | #else | ||
| 243 | ktime_t expires = hrtimer_get_expires(timer); | ||
| 244 | #endif | ||
| 245 | |||
| 246 | /* | ||
| 247 | * Get the next event on target cpu from the | ||
| 248 | * clock events layer. | ||
| 249 | * This covers the highres=off nohz=on case as well. | ||
| 250 | */ | ||
| 251 | ktime_t next = clockevents_get_next_event(cpu); | ||
| 252 | |||
| 253 | ktime_t delta = ktime_sub(expires, next); | ||
| 254 | |||
| 255 | /* | ||
| 256 | * We do not migrate the timer when it is expiring | ||
| 257 | * before the next event on the target cpu because | ||
| 258 | * we cannot reprogram the target cpu hardware and | ||
| 259 | * we would cause it to fire late. | ||
| 260 | */ | ||
| 261 | if (delta.tv64 < 0) { | ||
| 262 | cpu = smp_processor_id(); | ||
| 263 | spin_unlock(&new_base->cpu_base->lock); | ||
| 264 | spin_lock(&base->cpu_base->lock); | ||
| 265 | timer->base = base; | ||
| 266 | goto again; | ||
| 267 | } | ||
| 268 | } | 274 | } |
| 269 | timer->base = new_base; | 275 | timer->base = new_base; |
| 270 | } | 276 | } |
| @@ -1276,14 +1282,22 @@ void hrtimer_interrupt(struct clock_event_device *dev) | |||
| 1276 | 1282 | ||
| 1277 | expires_next.tv64 = KTIME_MAX; | 1283 | expires_next.tv64 = KTIME_MAX; |
| 1278 | 1284 | ||
| 1285 | spin_lock(&cpu_base->lock); | ||
| 1286 | /* | ||
| 1287 | * We set expires_next to KTIME_MAX here with cpu_base->lock | ||
| 1288 | * held to prevent that a timer is enqueued in our queue via | ||
| 1289 | * the migration code. This does not affect enqueueing of | ||
| 1290 | * timers which run their callback and need to be requeued on | ||
| 1291 | * this CPU. | ||
| 1292 | */ | ||
| 1293 | cpu_base->expires_next.tv64 = KTIME_MAX; | ||
| 1294 | |||
| 1279 | base = cpu_base->clock_base; | 1295 | base = cpu_base->clock_base; |
| 1280 | 1296 | ||
| 1281 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { | 1297 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { |
| 1282 | ktime_t basenow; | 1298 | ktime_t basenow; |
| 1283 | struct rb_node *node; | 1299 | struct rb_node *node; |
| 1284 | 1300 | ||
| 1285 | spin_lock(&cpu_base->lock); | ||
| 1286 | |||
| 1287 | basenow = ktime_add(now, base->offset); | 1301 | basenow = ktime_add(now, base->offset); |
| 1288 | 1302 | ||
| 1289 | while ((node = base->first)) { | 1303 | while ((node = base->first)) { |
| @@ -1316,11 +1330,15 @@ void hrtimer_interrupt(struct clock_event_device *dev) | |||
| 1316 | 1330 | ||
| 1317 | __run_hrtimer(timer); | 1331 | __run_hrtimer(timer); |
| 1318 | } | 1332 | } |
| 1319 | spin_unlock(&cpu_base->lock); | ||
| 1320 | base++; | 1333 | base++; |
| 1321 | } | 1334 | } |
| 1322 | 1335 | ||
| 1336 | /* | ||
| 1337 | * Store the new expiry value so the migration code can verify | ||
| 1338 | * against it. | ||
| 1339 | */ | ||
| 1323 | cpu_base->expires_next = expires_next; | 1340 | cpu_base->expires_next = expires_next; |
| 1341 | spin_unlock(&cpu_base->lock); | ||
| 1324 | 1342 | ||
| 1325 | /* Reprogramming necessary ? */ | 1343 | /* Reprogramming necessary ? */ |
| 1326 | if (expires_next.tv64 != KTIME_MAX) { | 1344 | if (expires_next.tv64 != KTIME_MAX) { |
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 73468253143b..e70ed5592eb9 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h | |||
| @@ -42,8 +42,7 @@ static inline void unregister_handler_proc(unsigned int irq, | |||
| 42 | 42 | ||
| 43 | extern int irq_select_affinity_usr(unsigned int irq); | 43 | extern int irq_select_affinity_usr(unsigned int irq); |
| 44 | 44 | ||
| 45 | extern void | 45 | extern void irq_set_thread_affinity(struct irq_desc *desc); |
| 46 | irq_set_thread_affinity(struct irq_desc *desc, const struct cpumask *cpumask); | ||
| 47 | 46 | ||
| 48 | /* | 47 | /* |
| 49 | * Debugging printout: | 48 | * Debugging printout: |
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 50da67672901..f0de36f13a44 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c | |||
| @@ -80,14 +80,22 @@ int irq_can_set_affinity(unsigned int irq) | |||
| 80 | return 1; | 80 | return 1; |
| 81 | } | 81 | } |
| 82 | 82 | ||
| 83 | void | 83 | /** |
| 84 | irq_set_thread_affinity(struct irq_desc *desc, const struct cpumask *cpumask) | 84 | * irq_set_thread_affinity - Notify irq threads to adjust affinity |
| 85 | * @desc: irq descriptor which has affitnity changed | ||
| 86 | * | ||
| 87 | * We just set IRQTF_AFFINITY and delegate the affinity setting | ||
| 88 | * to the interrupt thread itself. We can not call | ||
| 89 | * set_cpus_allowed_ptr() here as we hold desc->lock and this | ||
| 90 | * code can be called from hard interrupt context. | ||
| 91 | */ | ||
| 92 | void irq_set_thread_affinity(struct irq_desc *desc) | ||
| 85 | { | 93 | { |
| 86 | struct irqaction *action = desc->action; | 94 | struct irqaction *action = desc->action; |
| 87 | 95 | ||
| 88 | while (action) { | 96 | while (action) { |
| 89 | if (action->thread) | 97 | if (action->thread) |
| 90 | set_cpus_allowed_ptr(action->thread, cpumask); | 98 | set_bit(IRQTF_AFFINITY, &action->thread_flags); |
| 91 | action = action->next; | 99 | action = action->next; |
| 92 | } | 100 | } |
| 93 | } | 101 | } |
| @@ -112,7 +120,7 @@ int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask) | |||
| 112 | if (desc->status & IRQ_MOVE_PCNTXT) { | 120 | if (desc->status & IRQ_MOVE_PCNTXT) { |
| 113 | if (!desc->chip->set_affinity(irq, cpumask)) { | 121 | if (!desc->chip->set_affinity(irq, cpumask)) { |
| 114 | cpumask_copy(desc->affinity, cpumask); | 122 | cpumask_copy(desc->affinity, cpumask); |
| 115 | irq_set_thread_affinity(desc, cpumask); | 123 | irq_set_thread_affinity(desc); |
| 116 | } | 124 | } |
| 117 | } | 125 | } |
| 118 | else { | 126 | else { |
| @@ -122,7 +130,7 @@ int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask) | |||
| 122 | #else | 130 | #else |
| 123 | if (!desc->chip->set_affinity(irq, cpumask)) { | 131 | if (!desc->chip->set_affinity(irq, cpumask)) { |
| 124 | cpumask_copy(desc->affinity, cpumask); | 132 | cpumask_copy(desc->affinity, cpumask); |
| 125 | irq_set_thread_affinity(desc, cpumask); | 133 | irq_set_thread_affinity(desc); |
| 126 | } | 134 | } |
| 127 | #endif | 135 | #endif |
| 128 | desc->status |= IRQ_AFFINITY_SET; | 136 | desc->status |= IRQ_AFFINITY_SET; |
| @@ -176,7 +184,7 @@ int irq_select_affinity_usr(unsigned int irq) | |||
| 176 | spin_lock_irqsave(&desc->lock, flags); | 184 | spin_lock_irqsave(&desc->lock, flags); |
| 177 | ret = setup_affinity(irq, desc); | 185 | ret = setup_affinity(irq, desc); |
| 178 | if (!ret) | 186 | if (!ret) |
| 179 | irq_set_thread_affinity(desc, desc->affinity); | 187 | irq_set_thread_affinity(desc); |
| 180 | spin_unlock_irqrestore(&desc->lock, flags); | 188 | spin_unlock_irqrestore(&desc->lock, flags); |
| 181 | 189 | ||
| 182 | return ret; | 190 | return ret; |
| @@ -444,6 +452,34 @@ static int irq_wait_for_interrupt(struct irqaction *action) | |||
| 444 | } | 452 | } |
| 445 | 453 | ||
| 446 | /* | 454 | /* |
| 455 | * Check whether we need to change the affinity of the interrupt thread. | ||
| 456 | */ | ||
| 457 | static void | ||
| 458 | irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) | ||
| 459 | { | ||
| 460 | cpumask_var_t mask; | ||
| 461 | |||
| 462 | if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags)) | ||
| 463 | return; | ||
| 464 | |||
| 465 | /* | ||
| 466 | * In case we are out of memory we set IRQTF_AFFINITY again and | ||
| 467 | * try again next time | ||
| 468 | */ | ||
| 469 | if (!alloc_cpumask_var(&mask, GFP_KERNEL)) { | ||
| 470 | set_bit(IRQTF_AFFINITY, &action->thread_flags); | ||
| 471 | return; | ||
| 472 | } | ||
| 473 | |||
| 474 | spin_lock_irq(&desc->lock); | ||
| 475 | cpumask_copy(mask, desc->affinity); | ||
| 476 | spin_unlock_irq(&desc->lock); | ||
| 477 | |||
| 478 | set_cpus_allowed_ptr(current, mask); | ||
| 479 | free_cpumask_var(mask); | ||
| 480 | } | ||
| 481 | |||
| 482 | /* | ||
| 447 | * Interrupt handler thread | 483 | * Interrupt handler thread |
| 448 | */ | 484 | */ |
| 449 | static int irq_thread(void *data) | 485 | static int irq_thread(void *data) |
| @@ -458,6 +494,8 @@ static int irq_thread(void *data) | |||
| 458 | 494 | ||
| 459 | while (!irq_wait_for_interrupt(action)) { | 495 | while (!irq_wait_for_interrupt(action)) { |
| 460 | 496 | ||
| 497 | irq_thread_check_affinity(desc, action); | ||
| 498 | |||
| 461 | atomic_inc(&desc->threads_active); | 499 | atomic_inc(&desc->threads_active); |
| 462 | 500 | ||
| 463 | spin_lock_irq(&desc->lock); | 501 | spin_lock_irq(&desc->lock); |
diff --git a/kernel/irq/migration.c b/kernel/irq/migration.c index cfe767ca1545..fcb6c96f2627 100644 --- a/kernel/irq/migration.c +++ b/kernel/irq/migration.c | |||
| @@ -45,7 +45,7 @@ void move_masked_irq(int irq) | |||
| 45 | < nr_cpu_ids)) | 45 | < nr_cpu_ids)) |
| 46 | if (!desc->chip->set_affinity(irq, desc->pending_mask)) { | 46 | if (!desc->chip->set_affinity(irq, desc->pending_mask)) { |
| 47 | cpumask_copy(desc->affinity, desc->pending_mask); | 47 | cpumask_copy(desc->affinity, desc->pending_mask); |
| 48 | irq_set_thread_affinity(desc, desc->pending_mask); | 48 | irq_set_thread_affinity(desc); |
| 49 | } | 49 | } |
| 50 | 50 | ||
| 51 | cpumask_clear(desc->pending_mask); | 51 | cpumask_clear(desc->pending_mask); |
diff --git a/kernel/kmod.c b/kernel/kmod.c index 7e95bedb2bfc..385c31a1bdbf 100644 --- a/kernel/kmod.c +++ b/kernel/kmod.c | |||
| @@ -24,7 +24,6 @@ | |||
| 24 | #include <linux/unistd.h> | 24 | #include <linux/unistd.h> |
| 25 | #include <linux/kmod.h> | 25 | #include <linux/kmod.h> |
| 26 | #include <linux/slab.h> | 26 | #include <linux/slab.h> |
| 27 | #include <linux/mnt_namespace.h> | ||
| 28 | #include <linux/completion.h> | 27 | #include <linux/completion.h> |
| 29 | #include <linux/file.h> | 28 | #include <linux/file.h> |
| 30 | #include <linux/fdtable.h> | 29 | #include <linux/fdtable.h> |
diff --git a/kernel/kprobes.c b/kernel/kprobes.c index c0fa54b276d9..16b5739c516a 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c | |||
| @@ -237,13 +237,9 @@ static int __kprobes collect_garbage_slots(void) | |||
| 237 | { | 237 | { |
| 238 | struct kprobe_insn_page *kip; | 238 | struct kprobe_insn_page *kip; |
| 239 | struct hlist_node *pos, *next; | 239 | struct hlist_node *pos, *next; |
| 240 | int safety; | ||
| 241 | 240 | ||
| 242 | /* Ensure no-one is preepmted on the garbages */ | 241 | /* Ensure no-one is preepmted on the garbages */ |
| 243 | mutex_unlock(&kprobe_insn_mutex); | 242 | if (check_safety()) |
| 244 | safety = check_safety(); | ||
| 245 | mutex_lock(&kprobe_insn_mutex); | ||
| 246 | if (safety != 0) | ||
| 247 | return -EAGAIN; | 243 | return -EAGAIN; |
| 248 | 244 | ||
| 249 | hlist_for_each_entry_safe(kip, pos, next, &kprobe_insn_pages, hlist) { | 245 | hlist_for_each_entry_safe(kip, pos, next, &kprobe_insn_pages, hlist) { |
diff --git a/kernel/module.c b/kernel/module.c index 38928fcaff2b..0a049837008e 100644 --- a/kernel/module.c +++ b/kernel/module.c | |||
| @@ -2451,9 +2451,9 @@ SYSCALL_DEFINE3(init_module, void __user *, umod, | |||
| 2451 | return ret; | 2451 | return ret; |
| 2452 | } | 2452 | } |
| 2453 | if (ret > 0) { | 2453 | if (ret > 0) { |
| 2454 | printk(KERN_WARNING "%s: '%s'->init suspiciously returned %d, " | 2454 | printk(KERN_WARNING |
| 2455 | "it should follow 0/-E convention\n" | 2455 | "%s: '%s'->init suspiciously returned %d, it should follow 0/-E convention\n" |
| 2456 | KERN_WARNING "%s: loading module anyway...\n", | 2456 | "%s: loading module anyway...\n", |
| 2457 | __func__, mod->name, ret, | 2457 | __func__, mod->name, ret, |
| 2458 | __func__); | 2458 | __func__); |
| 2459 | dump_stack(); | 2459 | dump_stack(); |
diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c index 787d4daef185..950931041954 100644 --- a/kernel/perf_counter.c +++ b/kernel/perf_counter.c | |||
| @@ -2030,7 +2030,7 @@ fail: | |||
| 2030 | 2030 | ||
| 2031 | static void perf_mmap_free_page(unsigned long addr) | 2031 | static void perf_mmap_free_page(unsigned long addr) |
| 2032 | { | 2032 | { |
| 2033 | struct page *page = virt_to_page(addr); | 2033 | struct page *page = virt_to_page((void *)addr); |
| 2034 | 2034 | ||
| 2035 | page->mapping = NULL; | 2035 | page->mapping = NULL; |
| 2036 | __free_page(page); | 2036 | __free_page(page); |
diff --git a/kernel/pid.c b/kernel/pid.c index 5fa1db48d8b7..31310b5d3f50 100644 --- a/kernel/pid.c +++ b/kernel/pid.c | |||
| @@ -36,7 +36,6 @@ | |||
| 36 | #include <linux/pid_namespace.h> | 36 | #include <linux/pid_namespace.h> |
| 37 | #include <linux/init_task.h> | 37 | #include <linux/init_task.h> |
| 38 | #include <linux/syscalls.h> | 38 | #include <linux/syscalls.h> |
| 39 | #include <linux/kmemleak.h> | ||
| 40 | 39 | ||
| 41 | #define pid_hashfn(nr, ns) \ | 40 | #define pid_hashfn(nr, ns) \ |
| 42 | hash_long((unsigned long)nr + (unsigned long)ns, pidhash_shift) | 41 | hash_long((unsigned long)nr + (unsigned long)ns, pidhash_shift) |
| @@ -513,12 +512,6 @@ void __init pidhash_init(void) | |||
| 513 | pid_hash = alloc_bootmem(pidhash_size * sizeof(*(pid_hash))); | 512 | pid_hash = alloc_bootmem(pidhash_size * sizeof(*(pid_hash))); |
| 514 | if (!pid_hash) | 513 | if (!pid_hash) |
| 515 | panic("Could not alloc pidhash!\n"); | 514 | panic("Could not alloc pidhash!\n"); |
| 516 | /* | ||
| 517 | * pid_hash contains references to allocated struct pid objects and it | ||
| 518 | * must be scanned by kmemleak to avoid false positives. | ||
| 519 | */ | ||
| 520 | kmemleak_alloc(pid_hash, pidhash_size * sizeof(*(pid_hash)), 0, | ||
| 521 | GFP_KERNEL); | ||
| 522 | for (i = 0; i < pidhash_size; i++) | 515 | for (i = 0; i < pidhash_size; i++) |
| 523 | INIT_HLIST_HEAD(&pid_hash[i]); | 516 | INIT_HLIST_HEAD(&pid_hash[i]); |
| 524 | } | 517 | } |
diff --git a/kernel/power/user.c b/kernel/power/user.c index ed97375daae9..bf0014d6a5f0 100644 --- a/kernel/power/user.c +++ b/kernel/power/user.c | |||
| @@ -23,7 +23,6 @@ | |||
| 23 | #include <linux/console.h> | 23 | #include <linux/console.h> |
| 24 | #include <linux/cpu.h> | 24 | #include <linux/cpu.h> |
| 25 | #include <linux/freezer.h> | 25 | #include <linux/freezer.h> |
| 26 | #include <linux/smp_lock.h> | ||
| 27 | #include <scsi/scsi_scan.h> | 26 | #include <scsi/scsi_scan.h> |
| 28 | 27 | ||
| 29 | #include <asm/uaccess.h> | 28 | #include <asm/uaccess.h> |
diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 61c78b2c07ba..082c320e4dbf 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c | |||
| @@ -181,8 +181,8 @@ int ptrace_attach(struct task_struct *task) | |||
| 181 | * interference; SUID, SGID and LSM creds get determined differently | 181 | * interference; SUID, SGID and LSM creds get determined differently |
| 182 | * under ptrace. | 182 | * under ptrace. |
| 183 | */ | 183 | */ |
| 184 | retval = mutex_lock_interruptible(&task->cred_guard_mutex); | 184 | retval = -ERESTARTNOINTR; |
| 185 | if (retval < 0) | 185 | if (mutex_lock_interruptible(&task->cred_guard_mutex)) |
| 186 | goto out; | 186 | goto out; |
| 187 | 187 | ||
| 188 | task_lock(task); | 188 | task_lock(task); |
diff --git a/kernel/rcutree.c b/kernel/rcutree.c index 0dccfbba6d26..7717b95c2027 100644 --- a/kernel/rcutree.c +++ b/kernel/rcutree.c | |||
| @@ -1533,7 +1533,7 @@ void __init __rcu_init(void) | |||
| 1533 | int j; | 1533 | int j; |
| 1534 | struct rcu_node *rnp; | 1534 | struct rcu_node *rnp; |
| 1535 | 1535 | ||
| 1536 | printk(KERN_WARNING "Experimental hierarchical RCU implementation.\n"); | 1536 | printk(KERN_INFO "Hierarchical RCU implementation.\n"); |
| 1537 | #ifdef CONFIG_RCU_CPU_STALL_DETECTOR | 1537 | #ifdef CONFIG_RCU_CPU_STALL_DETECTOR |
| 1538 | printk(KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n"); | 1538 | printk(KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n"); |
| 1539 | #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ | 1539 | #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ |
| @@ -1546,7 +1546,6 @@ void __init __rcu_init(void) | |||
| 1546 | rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long)i); | 1546 | rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long)i); |
| 1547 | /* Register notifier for non-boot CPUs */ | 1547 | /* Register notifier for non-boot CPUs */ |
| 1548 | register_cpu_notifier(&rcu_nb); | 1548 | register_cpu_notifier(&rcu_nb); |
| 1549 | printk(KERN_WARNING "Experimental hierarchical RCU init done.\n"); | ||
| 1550 | } | 1549 | } |
| 1551 | 1550 | ||
| 1552 | module_param(blimit, int, 0); | 1551 | module_param(blimit, int, 0); |
diff --git a/kernel/sched.c b/kernel/sched.c index 7c9098d186e6..1b59e265273b 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
| @@ -493,6 +493,7 @@ struct rt_rq { | |||
| 493 | #endif | 493 | #endif |
| 494 | #ifdef CONFIG_SMP | 494 | #ifdef CONFIG_SMP |
| 495 | unsigned long rt_nr_migratory; | 495 | unsigned long rt_nr_migratory; |
| 496 | unsigned long rt_nr_total; | ||
| 496 | int overloaded; | 497 | int overloaded; |
| 497 | struct plist_head pushable_tasks; | 498 | struct plist_head pushable_tasks; |
| 498 | #endif | 499 | #endif |
| @@ -2571,15 +2572,37 @@ static void __sched_fork(struct task_struct *p) | |||
| 2571 | p->se.avg_wakeup = sysctl_sched_wakeup_granularity; | 2572 | p->se.avg_wakeup = sysctl_sched_wakeup_granularity; |
| 2572 | 2573 | ||
| 2573 | #ifdef CONFIG_SCHEDSTATS | 2574 | #ifdef CONFIG_SCHEDSTATS |
| 2574 | p->se.wait_start = 0; | 2575 | p->se.wait_start = 0; |
| 2575 | p->se.sum_sleep_runtime = 0; | 2576 | p->se.wait_max = 0; |
| 2576 | p->se.sleep_start = 0; | 2577 | p->se.wait_count = 0; |
| 2577 | p->se.block_start = 0; | 2578 | p->se.wait_sum = 0; |
| 2578 | p->se.sleep_max = 0; | 2579 | |
| 2579 | p->se.block_max = 0; | 2580 | p->se.sleep_start = 0; |
| 2580 | p->se.exec_max = 0; | 2581 | p->se.sleep_max = 0; |
| 2581 | p->se.slice_max = 0; | 2582 | p->se.sum_sleep_runtime = 0; |
| 2582 | p->se.wait_max = 0; | 2583 | |
| 2584 | p->se.block_start = 0; | ||
| 2585 | p->se.block_max = 0; | ||
| 2586 | p->se.exec_max = 0; | ||
| 2587 | p->se.slice_max = 0; | ||
| 2588 | |||
| 2589 | p->se.nr_migrations_cold = 0; | ||
| 2590 | p->se.nr_failed_migrations_affine = 0; | ||
| 2591 | p->se.nr_failed_migrations_running = 0; | ||
| 2592 | p->se.nr_failed_migrations_hot = 0; | ||
| 2593 | p->se.nr_forced_migrations = 0; | ||
| 2594 | p->se.nr_forced2_migrations = 0; | ||
| 2595 | |||
| 2596 | p->se.nr_wakeups = 0; | ||
| 2597 | p->se.nr_wakeups_sync = 0; | ||
| 2598 | p->se.nr_wakeups_migrate = 0; | ||
| 2599 | p->se.nr_wakeups_local = 0; | ||
| 2600 | p->se.nr_wakeups_remote = 0; | ||
| 2601 | p->se.nr_wakeups_affine = 0; | ||
| 2602 | p->se.nr_wakeups_affine_attempts = 0; | ||
| 2603 | p->se.nr_wakeups_passive = 0; | ||
| 2604 | p->se.nr_wakeups_idle = 0; | ||
| 2605 | |||
| 2583 | #endif | 2606 | #endif |
| 2584 | 2607 | ||
| 2585 | INIT_LIST_HEAD(&p->rt.run_list); | 2608 | INIT_LIST_HEAD(&p->rt.run_list); |
| @@ -6541,6 +6564,11 @@ SYSCALL_DEFINE0(sched_yield) | |||
| 6541 | return 0; | 6564 | return 0; |
| 6542 | } | 6565 | } |
| 6543 | 6566 | ||
| 6567 | static inline int should_resched(void) | ||
| 6568 | { | ||
| 6569 | return need_resched() && !(preempt_count() & PREEMPT_ACTIVE); | ||
| 6570 | } | ||
| 6571 | |||
| 6544 | static void __cond_resched(void) | 6572 | static void __cond_resched(void) |
| 6545 | { | 6573 | { |
| 6546 | #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP | 6574 | #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP |
| @@ -6560,8 +6588,7 @@ static void __cond_resched(void) | |||
| 6560 | 6588 | ||
| 6561 | int __sched _cond_resched(void) | 6589 | int __sched _cond_resched(void) |
| 6562 | { | 6590 | { |
| 6563 | if (need_resched() && !(preempt_count() & PREEMPT_ACTIVE) && | 6591 | if (should_resched()) { |
| 6564 | system_state == SYSTEM_RUNNING) { | ||
| 6565 | __cond_resched(); | 6592 | __cond_resched(); |
| 6566 | return 1; | 6593 | return 1; |
| 6567 | } | 6594 | } |
| @@ -6579,12 +6606,12 @@ EXPORT_SYMBOL(_cond_resched); | |||
| 6579 | */ | 6606 | */ |
| 6580 | int cond_resched_lock(spinlock_t *lock) | 6607 | int cond_resched_lock(spinlock_t *lock) |
| 6581 | { | 6608 | { |
| 6582 | int resched = need_resched() && system_state == SYSTEM_RUNNING; | 6609 | int resched = should_resched(); |
| 6583 | int ret = 0; | 6610 | int ret = 0; |
| 6584 | 6611 | ||
| 6585 | if (spin_needbreak(lock) || resched) { | 6612 | if (spin_needbreak(lock) || resched) { |
| 6586 | spin_unlock(lock); | 6613 | spin_unlock(lock); |
| 6587 | if (resched && need_resched()) | 6614 | if (resched) |
| 6588 | __cond_resched(); | 6615 | __cond_resched(); |
| 6589 | else | 6616 | else |
| 6590 | cpu_relax(); | 6617 | cpu_relax(); |
| @@ -6599,7 +6626,7 @@ int __sched cond_resched_softirq(void) | |||
| 6599 | { | 6626 | { |
| 6600 | BUG_ON(!in_softirq()); | 6627 | BUG_ON(!in_softirq()); |
| 6601 | 6628 | ||
| 6602 | if (need_resched() && system_state == SYSTEM_RUNNING) { | 6629 | if (should_resched()) { |
| 6603 | local_bh_enable(); | 6630 | local_bh_enable(); |
| 6604 | __cond_resched(); | 6631 | __cond_resched(); |
| 6605 | local_bh_disable(); | 6632 | local_bh_disable(); |
| @@ -7262,6 +7289,7 @@ static void migrate_dead_tasks(unsigned int dead_cpu) | |||
| 7262 | static void calc_global_load_remove(struct rq *rq) | 7289 | static void calc_global_load_remove(struct rq *rq) |
| 7263 | { | 7290 | { |
| 7264 | atomic_long_sub(rq->calc_load_active, &calc_load_tasks); | 7291 | atomic_long_sub(rq->calc_load_active, &calc_load_tasks); |
| 7292 | rq->calc_load_active = 0; | ||
| 7265 | } | 7293 | } |
| 7266 | #endif /* CONFIG_HOTPLUG_CPU */ | 7294 | #endif /* CONFIG_HOTPLUG_CPU */ |
| 7267 | 7295 | ||
| @@ -7488,6 +7516,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) | |||
| 7488 | task_rq_unlock(rq, &flags); | 7516 | task_rq_unlock(rq, &flags); |
| 7489 | get_task_struct(p); | 7517 | get_task_struct(p); |
| 7490 | cpu_rq(cpu)->migration_thread = p; | 7518 | cpu_rq(cpu)->migration_thread = p; |
| 7519 | rq->calc_load_update = calc_load_update; | ||
| 7491 | break; | 7520 | break; |
| 7492 | 7521 | ||
| 7493 | case CPU_ONLINE: | 7522 | case CPU_ONLINE: |
| @@ -7498,8 +7527,6 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) | |||
| 7498 | /* Update our root-domain */ | 7527 | /* Update our root-domain */ |
| 7499 | rq = cpu_rq(cpu); | 7528 | rq = cpu_rq(cpu); |
| 7500 | spin_lock_irqsave(&rq->lock, flags); | 7529 | spin_lock_irqsave(&rq->lock, flags); |
| 7501 | rq->calc_load_update = calc_load_update; | ||
| 7502 | rq->calc_load_active = 0; | ||
| 7503 | if (rq->rd) { | 7530 | if (rq->rd) { |
| 7504 | BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); | 7531 | BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); |
| 7505 | 7532 | ||
| @@ -9070,7 +9097,7 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq) | |||
| 9070 | #ifdef CONFIG_SMP | 9097 | #ifdef CONFIG_SMP |
| 9071 | rt_rq->rt_nr_migratory = 0; | 9098 | rt_rq->rt_nr_migratory = 0; |
| 9072 | rt_rq->overloaded = 0; | 9099 | rt_rq->overloaded = 0; |
| 9073 | plist_head_init(&rq->rt.pushable_tasks, &rq->lock); | 9100 | plist_head_init(&rt_rq->pushable_tasks, &rq->lock); |
| 9074 | #endif | 9101 | #endif |
| 9075 | 9102 | ||
| 9076 | rt_rq->rt_time = 0; | 9103 | rt_rq->rt_time = 0; |
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index ba7fd6e9556f..9ffb2b2ceba4 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c | |||
| @@ -266,6 +266,12 @@ static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) | |||
| 266 | return min_vruntime; | 266 | return min_vruntime; |
| 267 | } | 267 | } |
| 268 | 268 | ||
| 269 | static inline int entity_before(struct sched_entity *a, | ||
| 270 | struct sched_entity *b) | ||
| 271 | { | ||
| 272 | return (s64)(a->vruntime - b->vruntime) < 0; | ||
| 273 | } | ||
| 274 | |||
| 269 | static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) | 275 | static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 270 | { | 276 | { |
| 271 | return se->vruntime - cfs_rq->min_vruntime; | 277 | return se->vruntime - cfs_rq->min_vruntime; |
| @@ -687,7 +693,8 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) | |||
| 687 | * all of which have the same weight. | 693 | * all of which have the same weight. |
| 688 | */ | 694 | */ |
| 689 | if (sched_feat(NORMALIZED_SLEEPER) && | 695 | if (sched_feat(NORMALIZED_SLEEPER) && |
| 690 | task_of(se)->policy != SCHED_IDLE) | 696 | (!entity_is_task(se) || |
| 697 | task_of(se)->policy != SCHED_IDLE)) | ||
| 691 | thresh = calc_delta_fair(thresh, se); | 698 | thresh = calc_delta_fair(thresh, se); |
| 692 | 699 | ||
| 693 | vruntime -= thresh; | 700 | vruntime -= thresh; |
| @@ -1016,7 +1023,7 @@ static void yield_task_fair(struct rq *rq) | |||
| 1016 | /* | 1023 | /* |
| 1017 | * Already in the rightmost position? | 1024 | * Already in the rightmost position? |
| 1018 | */ | 1025 | */ |
| 1019 | if (unlikely(!rightmost || rightmost->vruntime < se->vruntime)) | 1026 | if (unlikely(!rightmost || entity_before(rightmost, se))) |
| 1020 | return; | 1027 | return; |
| 1021 | 1028 | ||
| 1022 | /* | 1029 | /* |
| @@ -1712,7 +1719,7 @@ static void task_new_fair(struct rq *rq, struct task_struct *p) | |||
| 1712 | 1719 | ||
| 1713 | /* 'curr' will be NULL if the child belongs to a different group */ | 1720 | /* 'curr' will be NULL if the child belongs to a different group */ |
| 1714 | if (sysctl_sched_child_runs_first && this_cpu == task_cpu(p) && | 1721 | if (sysctl_sched_child_runs_first && this_cpu == task_cpu(p) && |
| 1715 | curr && curr->vruntime < se->vruntime) { | 1722 | curr && entity_before(curr, se)) { |
| 1716 | /* | 1723 | /* |
| 1717 | * Upon rescheduling, sched_class::put_prev_task() will place | 1724 | * Upon rescheduling, sched_class::put_prev_task() will place |
| 1718 | * 'current' within the tree based on its new key value. | 1725 | * 'current' within the tree based on its new key value. |
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 9bf0d2a73045..3918e01994e0 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c | |||
| @@ -10,6 +10,8 @@ static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) | |||
| 10 | 10 | ||
| 11 | #ifdef CONFIG_RT_GROUP_SCHED | 11 | #ifdef CONFIG_RT_GROUP_SCHED |
| 12 | 12 | ||
| 13 | #define rt_entity_is_task(rt_se) (!(rt_se)->my_q) | ||
| 14 | |||
| 13 | static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) | 15 | static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) |
| 14 | { | 16 | { |
| 15 | return rt_rq->rq; | 17 | return rt_rq->rq; |
| @@ -22,6 +24,8 @@ static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) | |||
| 22 | 24 | ||
| 23 | #else /* CONFIG_RT_GROUP_SCHED */ | 25 | #else /* CONFIG_RT_GROUP_SCHED */ |
| 24 | 26 | ||
| 27 | #define rt_entity_is_task(rt_se) (1) | ||
| 28 | |||
| 25 | static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) | 29 | static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) |
| 26 | { | 30 | { |
| 27 | return container_of(rt_rq, struct rq, rt); | 31 | return container_of(rt_rq, struct rq, rt); |
| @@ -73,7 +77,7 @@ static inline void rt_clear_overload(struct rq *rq) | |||
| 73 | 77 | ||
| 74 | static void update_rt_migration(struct rt_rq *rt_rq) | 78 | static void update_rt_migration(struct rt_rq *rt_rq) |
| 75 | { | 79 | { |
| 76 | if (rt_rq->rt_nr_migratory && (rt_rq->rt_nr_running > 1)) { | 80 | if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) { |
| 77 | if (!rt_rq->overloaded) { | 81 | if (!rt_rq->overloaded) { |
| 78 | rt_set_overload(rq_of_rt_rq(rt_rq)); | 82 | rt_set_overload(rq_of_rt_rq(rt_rq)); |
| 79 | rt_rq->overloaded = 1; | 83 | rt_rq->overloaded = 1; |
| @@ -86,6 +90,12 @@ static void update_rt_migration(struct rt_rq *rt_rq) | |||
| 86 | 90 | ||
| 87 | static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) | 91 | static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) |
| 88 | { | 92 | { |
| 93 | if (!rt_entity_is_task(rt_se)) | ||
| 94 | return; | ||
| 95 | |||
| 96 | rt_rq = &rq_of_rt_rq(rt_rq)->rt; | ||
| 97 | |||
| 98 | rt_rq->rt_nr_total++; | ||
| 89 | if (rt_se->nr_cpus_allowed > 1) | 99 | if (rt_se->nr_cpus_allowed > 1) |
| 90 | rt_rq->rt_nr_migratory++; | 100 | rt_rq->rt_nr_migratory++; |
| 91 | 101 | ||
| @@ -94,6 +104,12 @@ static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) | |||
| 94 | 104 | ||
| 95 | static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) | 105 | static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) |
| 96 | { | 106 | { |
| 107 | if (!rt_entity_is_task(rt_se)) | ||
| 108 | return; | ||
| 109 | |||
| 110 | rt_rq = &rq_of_rt_rq(rt_rq)->rt; | ||
| 111 | |||
| 112 | rt_rq->rt_nr_total--; | ||
| 97 | if (rt_se->nr_cpus_allowed > 1) | 113 | if (rt_se->nr_cpus_allowed > 1) |
| 98 | rt_rq->rt_nr_migratory--; | 114 | rt_rq->rt_nr_migratory--; |
| 99 | 115 | ||
diff --git a/kernel/softirq.c b/kernel/softirq.c index 3a94905fa5d2..eb5e131a0485 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c | |||
| @@ -345,7 +345,9 @@ void open_softirq(int nr, void (*action)(struct softirq_action *)) | |||
| 345 | softirq_vec[nr].action = action; | 345 | softirq_vec[nr].action = action; |
| 346 | } | 346 | } |
| 347 | 347 | ||
| 348 | /* Tasklets */ | 348 | /* |
| 349 | * Tasklets | ||
| 350 | */ | ||
| 349 | struct tasklet_head | 351 | struct tasklet_head |
| 350 | { | 352 | { |
| 351 | struct tasklet_struct *head; | 353 | struct tasklet_struct *head; |
| @@ -493,6 +495,66 @@ void tasklet_kill(struct tasklet_struct *t) | |||
| 493 | 495 | ||
| 494 | EXPORT_SYMBOL(tasklet_kill); | 496 | EXPORT_SYMBOL(tasklet_kill); |
| 495 | 497 | ||
| 498 | /* | ||
| 499 | * tasklet_hrtimer | ||
| 500 | */ | ||
| 501 | |||
| 502 | /* | ||
| 503 | * The trampoline is called when the hrtimer expires. If this is | ||
| 504 | * called from the hrtimer interrupt then we schedule the tasklet as | ||
| 505 | * the timer callback function expects to run in softirq context. If | ||
| 506 | * it's called in softirq context anyway (i.e. high resolution timers | ||
| 507 | * disabled) then the hrtimer callback is called right away. | ||
| 508 | */ | ||
| 509 | static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer) | ||
| 510 | { | ||
| 511 | struct tasklet_hrtimer *ttimer = | ||
| 512 | container_of(timer, struct tasklet_hrtimer, timer); | ||
| 513 | |||
| 514 | if (hrtimer_is_hres_active(timer)) { | ||
| 515 | tasklet_hi_schedule(&ttimer->tasklet); | ||
| 516 | return HRTIMER_NORESTART; | ||
| 517 | } | ||
| 518 | return ttimer->function(timer); | ||
| 519 | } | ||
| 520 | |||
| 521 | /* | ||
| 522 | * Helper function which calls the hrtimer callback from | ||
| 523 | * tasklet/softirq context | ||
| 524 | */ | ||
| 525 | static void __tasklet_hrtimer_trampoline(unsigned long data) | ||
| 526 | { | ||
| 527 | struct tasklet_hrtimer *ttimer = (void *)data; | ||
| 528 | enum hrtimer_restart restart; | ||
| 529 | |||
| 530 | restart = ttimer->function(&ttimer->timer); | ||
| 531 | if (restart != HRTIMER_NORESTART) | ||
| 532 | hrtimer_restart(&ttimer->timer); | ||
| 533 | } | ||
| 534 | |||
| 535 | /** | ||
| 536 | * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks | ||
| 537 | * @ttimer: tasklet_hrtimer which is initialized | ||
| 538 | * @function: hrtimer callback funtion which gets called from softirq context | ||
| 539 | * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME) | ||
| 540 | * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL) | ||
| 541 | */ | ||
| 542 | void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer, | ||
| 543 | enum hrtimer_restart (*function)(struct hrtimer *), | ||
| 544 | clockid_t which_clock, enum hrtimer_mode mode) | ||
| 545 | { | ||
| 546 | hrtimer_init(&ttimer->timer, which_clock, mode); | ||
| 547 | ttimer->timer.function = __hrtimer_tasklet_trampoline; | ||
| 548 | tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline, | ||
| 549 | (unsigned long)ttimer); | ||
| 550 | ttimer->function = function; | ||
| 551 | } | ||
| 552 | EXPORT_SYMBOL_GPL(tasklet_hrtimer_init); | ||
| 553 | |||
| 554 | /* | ||
| 555 | * Remote softirq bits | ||
| 556 | */ | ||
| 557 | |||
| 496 | DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list); | 558 | DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list); |
| 497 | EXPORT_PER_CPU_SYMBOL(softirq_work_list); | 559 | EXPORT_PER_CPU_SYMBOL(softirq_work_list); |
| 498 | 560 | ||
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 1ad6dd461119..a6dcd67b041d 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c | |||
| @@ -254,15 +254,4 @@ void clockevents_notify(unsigned long reason, void *arg) | |||
| 254 | spin_unlock(&clockevents_lock); | 254 | spin_unlock(&clockevents_lock); |
| 255 | } | 255 | } |
| 256 | EXPORT_SYMBOL_GPL(clockevents_notify); | 256 | EXPORT_SYMBOL_GPL(clockevents_notify); |
| 257 | |||
| 258 | ktime_t clockevents_get_next_event(int cpu) | ||
| 259 | { | ||
| 260 | struct tick_device *td; | ||
| 261 | struct clock_event_device *dev; | ||
| 262 | |||
| 263 | td = &per_cpu(tick_cpu_device, cpu); | ||
| 264 | dev = td->evtdev; | ||
| 265 | |||
| 266 | return dev->next_event; | ||
| 267 | } | ||
| 268 | #endif | 257 | #endif |
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 592bf584d1d2..7466cb811251 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c | |||
| @@ -513,7 +513,7 @@ static ssize_t sysfs_override_clocksource(struct sys_device *dev, | |||
| 513 | * Check to make sure we don't switch to a non-highres capable | 513 | * Check to make sure we don't switch to a non-highres capable |
| 514 | * clocksource if the tick code is in oneshot mode (highres or nohz) | 514 | * clocksource if the tick code is in oneshot mode (highres or nohz) |
| 515 | */ | 515 | */ |
| 516 | if (tick_oneshot_mode_active() && | 516 | if (tick_oneshot_mode_active() && ovr && |
| 517 | !(ovr->flags & CLOCK_SOURCE_VALID_FOR_HRES)) { | 517 | !(ovr->flags & CLOCK_SOURCE_VALID_FOR_HRES)) { |
| 518 | printk(KERN_WARNING "%s clocksource is not HRT compatible. " | 518 | printk(KERN_WARNING "%s clocksource is not HRT compatible. " |
| 519 | "Cannot switch while in HRT/NOHZ mode\n", ovr->name); | 519 | "Cannot switch while in HRT/NOHZ mode\n", ovr->name); |
diff --git a/kernel/timer.c b/kernel/timer.c index 0b36b9e5cc8b..a7f07d5a6241 100644 --- a/kernel/timer.c +++ b/kernel/timer.c | |||
| @@ -714,7 +714,7 @@ int mod_timer(struct timer_list *timer, unsigned long expires) | |||
| 714 | * networking code - if the timer is re-modified | 714 | * networking code - if the timer is re-modified |
| 715 | * to be the same thing then just return: | 715 | * to be the same thing then just return: |
| 716 | */ | 716 | */ |
| 717 | if (timer->expires == expires && timer_pending(timer)) | 717 | if (timer_pending(timer) && timer->expires == expires) |
| 718 | return 1; | 718 | return 1; |
| 719 | 719 | ||
| 720 | return __mod_timer(timer, expires, false, TIMER_NOT_PINNED); | 720 | return __mod_timer(timer, expires, false, TIMER_NOT_PINNED); |
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 1551f47e7669..019f380fd764 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig | |||
| @@ -226,13 +226,13 @@ config BOOT_TRACER | |||
| 226 | the timings of the initcalls and traces key events and the identity | 226 | the timings of the initcalls and traces key events and the identity |
| 227 | of tasks that can cause boot delays, such as context-switches. | 227 | of tasks that can cause boot delays, such as context-switches. |
| 228 | 228 | ||
| 229 | Its aim is to be parsed by the /scripts/bootgraph.pl tool to | 229 | Its aim is to be parsed by the scripts/bootgraph.pl tool to |
| 230 | produce pretty graphics about boot inefficiencies, giving a visual | 230 | produce pretty graphics about boot inefficiencies, giving a visual |
| 231 | representation of the delays during initcalls - but the raw | 231 | representation of the delays during initcalls - but the raw |
| 232 | /debug/tracing/trace text output is readable too. | 232 | /debug/tracing/trace text output is readable too. |
| 233 | 233 | ||
| 234 | You must pass in ftrace=initcall to the kernel command line | 234 | You must pass in initcall_debug and ftrace=initcall to the kernel |
| 235 | to enable this on bootup. | 235 | command line to enable this on bootup. |
| 236 | 236 | ||
| 237 | config TRACE_BRANCH_PROFILING | 237 | config TRACE_BRANCH_PROFILING |
| 238 | bool | 238 | bool |
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index 39af8af6fc30..1090b0aed9ba 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c | |||
| @@ -22,6 +22,7 @@ | |||
| 22 | #include <linux/init.h> | 22 | #include <linux/init.h> |
| 23 | #include <linux/mutex.h> | 23 | #include <linux/mutex.h> |
| 24 | #include <linux/debugfs.h> | 24 | #include <linux/debugfs.h> |
| 25 | #include <linux/smp_lock.h> | ||
| 25 | #include <linux/time.h> | 26 | #include <linux/time.h> |
| 26 | #include <linux/uaccess.h> | 27 | #include <linux/uaccess.h> |
| 27 | 28 | ||
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index f3716bf04df6..4521c77d1a1a 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
| @@ -768,7 +768,7 @@ static struct tracer_stat function_stats __initdata = { | |||
| 768 | .stat_show = function_stat_show | 768 | .stat_show = function_stat_show |
| 769 | }; | 769 | }; |
| 770 | 770 | ||
| 771 | static void ftrace_profile_debugfs(struct dentry *d_tracer) | 771 | static __init void ftrace_profile_debugfs(struct dentry *d_tracer) |
| 772 | { | 772 | { |
| 773 | struct ftrace_profile_stat *stat; | 773 | struct ftrace_profile_stat *stat; |
| 774 | struct dentry *entry; | 774 | struct dentry *entry; |
| @@ -786,7 +786,6 @@ static void ftrace_profile_debugfs(struct dentry *d_tracer) | |||
| 786 | * The files created are permanent, if something happens | 786 | * The files created are permanent, if something happens |
| 787 | * we still do not free memory. | 787 | * we still do not free memory. |
| 788 | */ | 788 | */ |
| 789 | kfree(stat); | ||
| 790 | WARN(1, | 789 | WARN(1, |
| 791 | "Could not allocate stat file for cpu %d\n", | 790 | "Could not allocate stat file for cpu %d\n", |
| 792 | cpu); | 791 | cpu); |
| @@ -813,7 +812,7 @@ static void ftrace_profile_debugfs(struct dentry *d_tracer) | |||
| 813 | } | 812 | } |
| 814 | 813 | ||
| 815 | #else /* CONFIG_FUNCTION_PROFILER */ | 814 | #else /* CONFIG_FUNCTION_PROFILER */ |
| 816 | static void ftrace_profile_debugfs(struct dentry *d_tracer) | 815 | static __init void ftrace_profile_debugfs(struct dentry *d_tracer) |
| 817 | { | 816 | { |
| 818 | } | 817 | } |
| 819 | #endif /* CONFIG_FUNCTION_PROFILER */ | 818 | #endif /* CONFIG_FUNCTION_PROFILER */ |
| @@ -3160,10 +3159,10 @@ ftrace_enable_sysctl(struct ctl_table *table, int write, | |||
| 3160 | 3159 | ||
| 3161 | ret = proc_dointvec(table, write, file, buffer, lenp, ppos); | 3160 | ret = proc_dointvec(table, write, file, buffer, lenp, ppos); |
| 3162 | 3161 | ||
| 3163 | if (ret || !write || (last_ftrace_enabled == ftrace_enabled)) | 3162 | if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled)) |
| 3164 | goto out; | 3163 | goto out; |
| 3165 | 3164 | ||
| 3166 | last_ftrace_enabled = ftrace_enabled; | 3165 | last_ftrace_enabled = !!ftrace_enabled; |
| 3167 | 3166 | ||
| 3168 | if (ftrace_enabled) { | 3167 | if (ftrace_enabled) { |
| 3169 | 3168 | ||
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 3aa0a0dfdfa8..8bc8d8afea6a 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
| @@ -17,6 +17,7 @@ | |||
| 17 | #include <linux/writeback.h> | 17 | #include <linux/writeback.h> |
| 18 | #include <linux/kallsyms.h> | 18 | #include <linux/kallsyms.h> |
| 19 | #include <linux/seq_file.h> | 19 | #include <linux/seq_file.h> |
| 20 | #include <linux/smp_lock.h> | ||
| 20 | #include <linux/notifier.h> | 21 | #include <linux/notifier.h> |
| 21 | #include <linux/irqflags.h> | 22 | #include <linux/irqflags.h> |
| 22 | #include <linux/debugfs.h> | 23 | #include <linux/debugfs.h> |
diff --git a/kernel/trace/trace_event_types.h b/kernel/trace/trace_event_types.h index 5e32e375134d..6db005e12487 100644 --- a/kernel/trace/trace_event_types.h +++ b/kernel/trace/trace_event_types.h | |||
| @@ -26,6 +26,9 @@ TRACE_EVENT_FORMAT(funcgraph_exit, TRACE_GRAPH_RET, | |||
| 26 | ftrace_graph_ret_entry, ignore, | 26 | ftrace_graph_ret_entry, ignore, |
| 27 | TRACE_STRUCT( | 27 | TRACE_STRUCT( |
| 28 | TRACE_FIELD(unsigned long, ret.func, func) | 28 | TRACE_FIELD(unsigned long, ret.func, func) |
| 29 | TRACE_FIELD(unsigned long long, ret.calltime, calltime) | ||
| 30 | TRACE_FIELD(unsigned long long, ret.rettime, rettime) | ||
| 31 | TRACE_FIELD(unsigned long, ret.overrun, overrun) | ||
| 29 | TRACE_FIELD(int, ret.depth, depth) | 32 | TRACE_FIELD(int, ret.depth, depth) |
| 30 | ), | 33 | ), |
| 31 | TP_RAW_FMT("<-- %lx (%d)") | 34 | TP_RAW_FMT("<-- %lx (%d)") |
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index 7402144bff21..75ef000613c3 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c | |||
| @@ -363,7 +363,7 @@ ftrace_trace_onoff_callback(char *glob, char *cmd, char *param, int enable) | |||
| 363 | out_reg: | 363 | out_reg: |
| 364 | ret = register_ftrace_function_probe(glob, ops, count); | 364 | ret = register_ftrace_function_probe(glob, ops, count); |
| 365 | 365 | ||
| 366 | return ret; | 366 | return ret < 0 ? ret : 0; |
| 367 | } | 367 | } |
| 368 | 368 | ||
| 369 | static struct ftrace_func_command ftrace_traceon_cmd = { | 369 | static struct ftrace_func_command ftrace_traceon_cmd = { |
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c index 7938f3ae93e3..e0c2545622e8 100644 --- a/kernel/trace/trace_output.c +++ b/kernel/trace/trace_output.c | |||
| @@ -27,8 +27,7 @@ void trace_print_seq(struct seq_file *m, struct trace_seq *s) | |||
| 27 | { | 27 | { |
| 28 | int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len; | 28 | int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len; |
| 29 | 29 | ||
| 30 | s->buffer[len] = 0; | 30 | seq_write(m, s->buffer, len); |
| 31 | seq_puts(m, s->buffer); | ||
| 32 | 31 | ||
| 33 | trace_seq_init(s); | 32 | trace_seq_init(s); |
| 34 | } | 33 | } |
diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c index 2d7aebd71dbd..e644af910124 100644 --- a/kernel/trace/trace_stack.c +++ b/kernel/trace/trace_stack.c | |||
| @@ -326,10 +326,10 @@ stack_trace_sysctl(struct ctl_table *table, int write, | |||
| 326 | ret = proc_dointvec(table, write, file, buffer, lenp, ppos); | 326 | ret = proc_dointvec(table, write, file, buffer, lenp, ppos); |
| 327 | 327 | ||
| 328 | if (ret || !write || | 328 | if (ret || !write || |
| 329 | (last_stack_tracer_enabled == stack_tracer_enabled)) | 329 | (last_stack_tracer_enabled == !!stack_tracer_enabled)) |
| 330 | goto out; | 330 | goto out; |
| 331 | 331 | ||
| 332 | last_stack_tracer_enabled = stack_tracer_enabled; | 332 | last_stack_tracer_enabled = !!stack_tracer_enabled; |
| 333 | 333 | ||
| 334 | if (stack_tracer_enabled) | 334 | if (stack_tracer_enabled) |
| 335 | register_ftrace_function(&trace_ops); | 335 | register_ftrace_function(&trace_ops); |
