aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/auditsc.c22
-rw-r--r--kernel/irq/chip.c5
-rw-r--r--kernel/irq/resend.c12
-rw-r--r--kernel/kprobes.c5
-rw-r--r--kernel/power/snapshot.c3
-rw-r--r--kernel/profile.c4
-rw-r--r--kernel/sched.c402
-rw-r--r--kernel/sched_debug.c30
-rw-r--r--kernel/sched_fair.c225
-rw-r--r--kernel/sched_idletask.c10
-rw-r--r--kernel/sched_rt.c48
-rw-r--r--kernel/sysctl.c2
-rw-r--r--kernel/time/clockevents.c10
13 files changed, 379 insertions, 399 deletions
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index a777d37614..3401293359 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -1992,19 +1992,19 @@ int __audit_signal_info(int sig, struct task_struct *t)
1992 extern uid_t audit_sig_uid; 1992 extern uid_t audit_sig_uid;
1993 extern u32 audit_sig_sid; 1993 extern u32 audit_sig_sid;
1994 1994
1995 if (audit_pid && t->tgid == audit_pid && 1995 if (audit_pid && t->tgid == audit_pid) {
1996 (sig == SIGTERM || sig == SIGHUP || sig == SIGUSR1)) { 1996 if (sig == SIGTERM || sig == SIGHUP || sig == SIGUSR1) {
1997 audit_sig_pid = tsk->pid; 1997 audit_sig_pid = tsk->pid;
1998 if (ctx) 1998 if (ctx)
1999 audit_sig_uid = ctx->loginuid; 1999 audit_sig_uid = ctx->loginuid;
2000 else 2000 else
2001 audit_sig_uid = tsk->uid; 2001 audit_sig_uid = tsk->uid;
2002 selinux_get_task_sid(tsk, &audit_sig_sid); 2002 selinux_get_task_sid(tsk, &audit_sig_sid);
2003 }
2004 if (!audit_signals || audit_dummy_context())
2005 return 0;
2003 } 2006 }
2004 2007
2005 if (!audit_signals) /* audit_context checked in wrapper */
2006 return 0;
2007
2008 /* optimize the common case by putting first signal recipient directly 2008 /* optimize the common case by putting first signal recipient directly
2009 * in audit_context */ 2009 * in audit_context */
2010 if (!ctx->target_pid) { 2010 if (!ctx->target_pid) {
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index 615ce97c6c..f1a73f0b54 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -352,13 +352,10 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc)
352 * keep it masked and get out of here 352 * keep it masked and get out of here
353 */ 353 */
354 action = desc->action; 354 action = desc->action;
355 if (unlikely(!action || (desc->status & IRQ_DISABLED))) { 355 if (unlikely(!action || (desc->status & IRQ_DISABLED)))
356 desc->status |= IRQ_PENDING;
357 goto out_unlock; 356 goto out_unlock;
358 }
359 357
360 desc->status |= IRQ_INPROGRESS; 358 desc->status |= IRQ_INPROGRESS;
361 desc->status &= ~IRQ_PENDING;
362 spin_unlock(&desc->lock); 359 spin_unlock(&desc->lock);
363 360
364 action_ret = handle_IRQ_event(irq, action); 361 action_ret = handle_IRQ_event(irq, action);
diff --git a/kernel/irq/resend.c b/kernel/irq/resend.c
index c382727468..a8046791ba 100644
--- a/kernel/irq/resend.c
+++ b/kernel/irq/resend.c
@@ -63,15 +63,11 @@ void check_irq_resend(struct irq_desc *desc, unsigned int irq)
63 desc->chip->enable(irq); 63 desc->chip->enable(irq);
64 64
65 /* 65 /*
66 * Temporary hack to figure out more about the problem, which 66 * We do not resend level type interrupts. Level type
67 * is causing the ancient network cards to die. 67 * interrupts are resent by hardware when they are still
68 * active.
68 */ 69 */
69 if (desc->handle_irq != handle_edge_irq) { 70 if ((status & (IRQ_LEVEL | IRQ_PENDING | IRQ_REPLAY)) == IRQ_PENDING) {
70 WARN_ON_ONCE(1);
71 return;
72 }
73
74 if ((status & (IRQ_PENDING | IRQ_REPLAY)) == IRQ_PENDING) {
75 desc->status = (status & ~IRQ_PENDING) | IRQ_REPLAY; 71 desc->status = (status & ~IRQ_PENDING) | IRQ_REPLAY;
76 72
77 if (!desc->chip || !desc->chip->retrigger || 73 if (!desc->chip || !desc->chip->retrigger ||
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 3e9f513a72..4b8a4493c5 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -1063,6 +1063,11 @@ EXPORT_SYMBOL_GPL(register_kprobe);
1063EXPORT_SYMBOL_GPL(unregister_kprobe); 1063EXPORT_SYMBOL_GPL(unregister_kprobe);
1064EXPORT_SYMBOL_GPL(register_jprobe); 1064EXPORT_SYMBOL_GPL(register_jprobe);
1065EXPORT_SYMBOL_GPL(unregister_jprobe); 1065EXPORT_SYMBOL_GPL(unregister_jprobe);
1066#ifdef CONFIG_KPROBES
1066EXPORT_SYMBOL_GPL(jprobe_return); 1067EXPORT_SYMBOL_GPL(jprobe_return);
1068#endif
1069
1070#ifdef CONFIG_KPROBES
1067EXPORT_SYMBOL_GPL(register_kretprobe); 1071EXPORT_SYMBOL_GPL(register_kretprobe);
1068EXPORT_SYMBOL_GPL(unregister_kretprobe); 1072EXPORT_SYMBOL_GPL(unregister_kretprobe);
1073#endif
diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c
index a3b7854b8f..a686590d88 100644
--- a/kernel/power/snapshot.c
+++ b/kernel/power/snapshot.c
@@ -709,7 +709,8 @@ static void mark_nosave_pages(struct memory_bitmap *bm)
709 region->end_pfn << PAGE_SHIFT); 709 region->end_pfn << PAGE_SHIFT);
710 710
711 for (pfn = region->start_pfn; pfn < region->end_pfn; pfn++) 711 for (pfn = region->start_pfn; pfn < region->end_pfn; pfn++)
712 memory_bm_set_bit(bm, pfn); 712 if (pfn_valid(pfn))
713 memory_bm_set_bit(bm, pfn);
713 } 714 }
714} 715}
715 716
diff --git a/kernel/profile.c b/kernel/profile.c
index 5b20fe977b..cb1e37d2da 100644
--- a/kernel/profile.c
+++ b/kernel/profile.c
@@ -199,11 +199,11 @@ EXPORT_SYMBOL_GPL(register_timer_hook);
199EXPORT_SYMBOL_GPL(unregister_timer_hook); 199EXPORT_SYMBOL_GPL(unregister_timer_hook);
200EXPORT_SYMBOL_GPL(task_handoff_register); 200EXPORT_SYMBOL_GPL(task_handoff_register);
201EXPORT_SYMBOL_GPL(task_handoff_unregister); 201EXPORT_SYMBOL_GPL(task_handoff_unregister);
202EXPORT_SYMBOL_GPL(profile_event_register);
203EXPORT_SYMBOL_GPL(profile_event_unregister);
202 204
203#endif /* CONFIG_PROFILING */ 205#endif /* CONFIG_PROFILING */
204 206
205EXPORT_SYMBOL_GPL(profile_event_register);
206EXPORT_SYMBOL_GPL(profile_event_unregister);
207 207
208#ifdef CONFIG_SMP 208#ifdef CONFIG_SMP
209/* 209/*
diff --git a/kernel/sched.c b/kernel/sched.c
index 72bb9483d9..45e17b83b7 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -263,6 +263,7 @@ struct rq {
263 263
264 unsigned int clock_warps, clock_overflows; 264 unsigned int clock_warps, clock_overflows;
265 unsigned int clock_unstable_events; 265 unsigned int clock_unstable_events;
266 u64 tick_timestamp;
266 267
267 atomic_t nr_iowait; 268 atomic_t nr_iowait;
268 269
@@ -318,15 +319,19 @@ static inline int cpu_of(struct rq *rq)
318} 319}
319 320
320/* 321/*
321 * Per-runqueue clock, as finegrained as the platform can give us: 322 * Update the per-runqueue clock, as finegrained as the platform can give
323 * us, but without assuming monotonicity, etc.:
322 */ 324 */
323static unsigned long long __rq_clock(struct rq *rq) 325static void __update_rq_clock(struct rq *rq)
324{ 326{
325 u64 prev_raw = rq->prev_clock_raw; 327 u64 prev_raw = rq->prev_clock_raw;
326 u64 now = sched_clock(); 328 u64 now = sched_clock();
327 s64 delta = now - prev_raw; 329 s64 delta = now - prev_raw;
328 u64 clock = rq->clock; 330 u64 clock = rq->clock;
329 331
332#ifdef CONFIG_SCHED_DEBUG
333 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
334#endif
330 /* 335 /*
331 * Protect against sched_clock() occasionally going backwards: 336 * Protect against sched_clock() occasionally going backwards:
332 */ 337 */
@@ -337,8 +342,11 @@ static unsigned long long __rq_clock(struct rq *rq)
337 /* 342 /*
338 * Catch too large forward jumps too: 343 * Catch too large forward jumps too:
339 */ 344 */
340 if (unlikely(delta > 2*TICK_NSEC)) { 345 if (unlikely(clock + delta > rq->tick_timestamp + TICK_NSEC)) {
341 clock++; 346 if (clock < rq->tick_timestamp + TICK_NSEC)
347 clock = rq->tick_timestamp + TICK_NSEC;
348 else
349 clock++;
342 rq->clock_overflows++; 350 rq->clock_overflows++;
343 } else { 351 } else {
344 if (unlikely(delta > rq->clock_max_delta)) 352 if (unlikely(delta > rq->clock_max_delta))
@@ -349,18 +357,12 @@ static unsigned long long __rq_clock(struct rq *rq)
349 357
350 rq->prev_clock_raw = now; 358 rq->prev_clock_raw = now;
351 rq->clock = clock; 359 rq->clock = clock;
352
353 return clock;
354} 360}
355 361
356static inline unsigned long long rq_clock(struct rq *rq) 362static void update_rq_clock(struct rq *rq)
357{ 363{
358 int this_cpu = smp_processor_id(); 364 if (likely(smp_processor_id() == cpu_of(rq)))
359 365 __update_rq_clock(rq);
360 if (this_cpu == cpu_of(rq))
361 return __rq_clock(rq);
362
363 return rq->clock;
364} 366}
365 367
366/* 368/*
@@ -386,9 +388,12 @@ unsigned long long cpu_clock(int cpu)
386{ 388{
387 unsigned long long now; 389 unsigned long long now;
388 unsigned long flags; 390 unsigned long flags;
391 struct rq *rq;
389 392
390 local_irq_save(flags); 393 local_irq_save(flags);
391 now = rq_clock(cpu_rq(cpu)); 394 rq = cpu_rq(cpu);
395 update_rq_clock(rq);
396 now = rq->clock;
392 local_irq_restore(flags); 397 local_irq_restore(flags);
393 398
394 return now; 399 return now;
@@ -637,6 +642,11 @@ static u64 div64_likely32(u64 divident, unsigned long divisor)
637 642
638#define WMULT_SHIFT 32 643#define WMULT_SHIFT 32
639 644
645/*
646 * Shift right and round:
647 */
648#define RSR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
649
640static unsigned long 650static unsigned long
641calc_delta_mine(unsigned long delta_exec, unsigned long weight, 651calc_delta_mine(unsigned long delta_exec, unsigned long weight,
642 struct load_weight *lw) 652 struct load_weight *lw)
@@ -644,18 +654,17 @@ calc_delta_mine(unsigned long delta_exec, unsigned long weight,
644 u64 tmp; 654 u64 tmp;
645 655
646 if (unlikely(!lw->inv_weight)) 656 if (unlikely(!lw->inv_weight))
647 lw->inv_weight = WMULT_CONST / lw->weight; 657 lw->inv_weight = (WMULT_CONST - lw->weight/2) / lw->weight + 1;
648 658
649 tmp = (u64)delta_exec * weight; 659 tmp = (u64)delta_exec * weight;
650 /* 660 /*
651 * Check whether we'd overflow the 64-bit multiplication: 661 * Check whether we'd overflow the 64-bit multiplication:
652 */ 662 */
653 if (unlikely(tmp > WMULT_CONST)) { 663 if (unlikely(tmp > WMULT_CONST))
654 tmp = ((tmp >> WMULT_SHIFT/2) * lw->inv_weight) 664 tmp = RSR(RSR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
655 >> (WMULT_SHIFT/2); 665 WMULT_SHIFT/2);
656 } else { 666 else
657 tmp = (tmp * lw->inv_weight) >> WMULT_SHIFT; 667 tmp = RSR(tmp * lw->inv_weight, WMULT_SHIFT);
658 }
659 668
660 return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX); 669 return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
661} 670}
@@ -703,11 +712,14 @@ static void update_load_sub(struct load_weight *lw, unsigned long dec)
703 * the relative distance between them is ~25%.) 712 * the relative distance between them is ~25%.)
704 */ 713 */
705static const int prio_to_weight[40] = { 714static const int prio_to_weight[40] = {
706/* -20 */ 88818, 71054, 56843, 45475, 36380, 29104, 23283, 18626, 14901, 11921, 715 /* -20 */ 88761, 71755, 56483, 46273, 36291,
707/* -10 */ 9537, 7629, 6103, 4883, 3906, 3125, 2500, 2000, 1600, 1280, 716 /* -15 */ 29154, 23254, 18705, 14949, 11916,
708/* 0 */ NICE_0_LOAD /* 1024 */, 717 /* -10 */ 9548, 7620, 6100, 4904, 3906,
709/* 1 */ 819, 655, 524, 419, 336, 268, 215, 172, 137, 718 /* -5 */ 3121, 2501, 1991, 1586, 1277,
710/* 10 */ 110, 87, 70, 56, 45, 36, 29, 23, 18, 15, 719 /* 0 */ 1024, 820, 655, 526, 423,
720 /* 5 */ 335, 272, 215, 172, 137,
721 /* 10 */ 110, 87, 70, 56, 45,
722 /* 15 */ 36, 29, 23, 18, 15,
711}; 723};
712 724
713/* 725/*
@@ -718,14 +730,14 @@ static const int prio_to_weight[40] = {
718 * into multiplications: 730 * into multiplications:
719 */ 731 */
720static const u32 prio_to_wmult[40] = { 732static const u32 prio_to_wmult[40] = {
721/* -20 */ 48356, 60446, 75558, 94446, 118058, 733 /* -20 */ 48388, 59856, 76040, 92818, 118348,
722/* -15 */ 147573, 184467, 230589, 288233, 360285, 734 /* -15 */ 147320, 184698, 229616, 287308, 360437,
723/* -10 */ 450347, 562979, 703746, 879575, 1099582, 735 /* -10 */ 449829, 563644, 704093, 875809, 1099582,
724/* -5 */ 1374389, 1717986, 2147483, 2684354, 3355443, 736 /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326,
725/* 0 */ 4194304, 5244160, 6557201, 8196502, 10250518, 737 /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587,
726/* 5 */ 12782640, 16025997, 19976592, 24970740, 31350126, 738 /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126,
727/* 10 */ 39045157, 49367440, 61356675, 76695844, 95443717, 739 /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717,
728/* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, 740 /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
729}; 741};
730 742
731static void activate_task(struct rq *rq, struct task_struct *p, int wakeup); 743static void activate_task(struct rq *rq, struct task_struct *p, int wakeup);
@@ -745,8 +757,7 @@ static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
745 unsigned long max_nr_move, unsigned long max_load_move, 757 unsigned long max_nr_move, unsigned long max_load_move,
746 struct sched_domain *sd, enum cpu_idle_type idle, 758 struct sched_domain *sd, enum cpu_idle_type idle,
747 int *all_pinned, unsigned long *load_moved, 759 int *all_pinned, unsigned long *load_moved,
748 int this_best_prio, int best_prio, int best_prio_seen, 760 int *this_best_prio, struct rq_iterator *iterator);
749 struct rq_iterator *iterator);
750 761
751#include "sched_stats.h" 762#include "sched_stats.h"
752#include "sched_rt.c" 763#include "sched_rt.c"
@@ -782,14 +793,14 @@ static void __update_curr_load(struct rq *rq, struct load_stat *ls)
782 * This function is called /before/ updating rq->ls.load 793 * This function is called /before/ updating rq->ls.load
783 * and when switching tasks. 794 * and when switching tasks.
784 */ 795 */
785static void update_curr_load(struct rq *rq, u64 now) 796static void update_curr_load(struct rq *rq)
786{ 797{
787 struct load_stat *ls = &rq->ls; 798 struct load_stat *ls = &rq->ls;
788 u64 start; 799 u64 start;
789 800
790 start = ls->load_update_start; 801 start = ls->load_update_start;
791 ls->load_update_start = now; 802 ls->load_update_start = rq->clock;
792 ls->delta_stat += now - start; 803 ls->delta_stat += rq->clock - start;
793 /* 804 /*
794 * Stagger updates to ls->delta_fair. Very frequent updates 805 * Stagger updates to ls->delta_fair. Very frequent updates
795 * can be expensive. 806 * can be expensive.
@@ -798,30 +809,28 @@ static void update_curr_load(struct rq *rq, u64 now)
798 __update_curr_load(rq, ls); 809 __update_curr_load(rq, ls);
799} 810}
800 811
801static inline void 812static inline void inc_load(struct rq *rq, const struct task_struct *p)
802inc_load(struct rq *rq, const struct task_struct *p, u64 now)
803{ 813{
804 update_curr_load(rq, now); 814 update_curr_load(rq);
805 update_load_add(&rq->ls.load, p->se.load.weight); 815 update_load_add(&rq->ls.load, p->se.load.weight);
806} 816}
807 817
808static inline void 818static inline void dec_load(struct rq *rq, const struct task_struct *p)
809dec_load(struct rq *rq, const struct task_struct *p, u64 now)
810{ 819{
811 update_curr_load(rq, now); 820 update_curr_load(rq);
812 update_load_sub(&rq->ls.load, p->se.load.weight); 821 update_load_sub(&rq->ls.load, p->se.load.weight);
813} 822}
814 823
815static void inc_nr_running(struct task_struct *p, struct rq *rq, u64 now) 824static void inc_nr_running(struct task_struct *p, struct rq *rq)
816{ 825{
817 rq->nr_running++; 826 rq->nr_running++;
818 inc_load(rq, p, now); 827 inc_load(rq, p);
819} 828}
820 829
821static void dec_nr_running(struct task_struct *p, struct rq *rq, u64 now) 830static void dec_nr_running(struct task_struct *p, struct rq *rq)
822{ 831{
823 rq->nr_running--; 832 rq->nr_running--;
824 dec_load(rq, p, now); 833 dec_load(rq, p);
825} 834}
826 835
827static void set_load_weight(struct task_struct *p) 836static void set_load_weight(struct task_struct *p)
@@ -848,18 +857,16 @@ static void set_load_weight(struct task_struct *p)
848 p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO]; 857 p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
849} 858}
850 859
851static void 860static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
852enqueue_task(struct rq *rq, struct task_struct *p, int wakeup, u64 now)
853{ 861{
854 sched_info_queued(p); 862 sched_info_queued(p);
855 p->sched_class->enqueue_task(rq, p, wakeup, now); 863 p->sched_class->enqueue_task(rq, p, wakeup);
856 p->se.on_rq = 1; 864 p->se.on_rq = 1;
857} 865}
858 866
859static void 867static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep)
860dequeue_task(struct rq *rq, struct task_struct *p, int sleep, u64 now)
861{ 868{
862 p->sched_class->dequeue_task(rq, p, sleep, now); 869 p->sched_class->dequeue_task(rq, p, sleep);
863 p->se.on_rq = 0; 870 p->se.on_rq = 0;
864} 871}
865 872
@@ -914,13 +921,11 @@ static int effective_prio(struct task_struct *p)
914 */ 921 */
915static void activate_task(struct rq *rq, struct task_struct *p, int wakeup) 922static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
916{ 923{
917 u64 now = rq_clock(rq);
918
919 if (p->state == TASK_UNINTERRUPTIBLE) 924 if (p->state == TASK_UNINTERRUPTIBLE)
920 rq->nr_uninterruptible--; 925 rq->nr_uninterruptible--;
921 926
922 enqueue_task(rq, p, wakeup, now); 927 enqueue_task(rq, p, wakeup);
923 inc_nr_running(p, rq, now); 928 inc_nr_running(p, rq);
924} 929}
925 930
926/* 931/*
@@ -928,13 +933,13 @@ static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
928 */ 933 */
929static inline void activate_idle_task(struct task_struct *p, struct rq *rq) 934static inline void activate_idle_task(struct task_struct *p, struct rq *rq)
930{ 935{
931 u64 now = rq_clock(rq); 936 update_rq_clock(rq);
932 937
933 if (p->state == TASK_UNINTERRUPTIBLE) 938 if (p->state == TASK_UNINTERRUPTIBLE)
934 rq->nr_uninterruptible--; 939 rq->nr_uninterruptible--;
935 940
936 enqueue_task(rq, p, 0, now); 941 enqueue_task(rq, p, 0);
937 inc_nr_running(p, rq, now); 942 inc_nr_running(p, rq);
938} 943}
939 944
940/* 945/*
@@ -942,13 +947,11 @@ static inline void activate_idle_task(struct task_struct *p, struct rq *rq)
942 */ 947 */
943static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep) 948static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep)
944{ 949{
945 u64 now = rq_clock(rq);
946
947 if (p->state == TASK_UNINTERRUPTIBLE) 950 if (p->state == TASK_UNINTERRUPTIBLE)
948 rq->nr_uninterruptible++; 951 rq->nr_uninterruptible++;
949 952
950 dequeue_task(rq, p, sleep, now); 953 dequeue_task(rq, p, sleep);
951 dec_nr_running(p, rq, now); 954 dec_nr_running(p, rq);
952} 955}
953 956
954/** 957/**
@@ -1516,6 +1519,7 @@ out_set_cpu:
1516 1519
1517out_activate: 1520out_activate:
1518#endif /* CONFIG_SMP */ 1521#endif /* CONFIG_SMP */
1522 update_rq_clock(rq);
1519 activate_task(rq, p, 1); 1523 activate_task(rq, p, 1);
1520 /* 1524 /*
1521 * Sync wakeups (i.e. those types of wakeups where the waker 1525 * Sync wakeups (i.e. those types of wakeups where the waker
@@ -1647,12 +1651,11 @@ void fastcall wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
1647 unsigned long flags; 1651 unsigned long flags;
1648 struct rq *rq; 1652 struct rq *rq;
1649 int this_cpu; 1653 int this_cpu;
1650 u64 now;
1651 1654
1652 rq = task_rq_lock(p, &flags); 1655 rq = task_rq_lock(p, &flags);
1653 BUG_ON(p->state != TASK_RUNNING); 1656 BUG_ON(p->state != TASK_RUNNING);
1654 this_cpu = smp_processor_id(); /* parent's CPU */ 1657 this_cpu = smp_processor_id(); /* parent's CPU */
1655 now = rq_clock(rq); 1658 update_rq_clock(rq);
1656 1659
1657 p->prio = effective_prio(p); 1660 p->prio = effective_prio(p);
1658 1661
@@ -1666,8 +1669,8 @@ void fastcall wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
1666 * Let the scheduling class do new task startup 1669 * Let the scheduling class do new task startup
1667 * management (if any): 1670 * management (if any):
1668 */ 1671 */
1669 p->sched_class->task_new(rq, p, now); 1672 p->sched_class->task_new(rq, p);
1670 inc_nr_running(p, rq, now); 1673 inc_nr_running(p, rq);
1671 } 1674 }
1672 check_preempt_curr(rq, p); 1675 check_preempt_curr(rq, p);
1673 task_rq_unlock(rq, &flags); 1676 task_rq_unlock(rq, &flags);
@@ -1954,7 +1957,6 @@ static void update_cpu_load(struct rq *this_rq)
1954 unsigned long total_load = this_rq->ls.load.weight; 1957 unsigned long total_load = this_rq->ls.load.weight;
1955 unsigned long this_load = total_load; 1958 unsigned long this_load = total_load;
1956 struct load_stat *ls = &this_rq->ls; 1959 struct load_stat *ls = &this_rq->ls;
1957 u64 now = __rq_clock(this_rq);
1958 int i, scale; 1960 int i, scale;
1959 1961
1960 this_rq->nr_load_updates++; 1962 this_rq->nr_load_updates++;
@@ -1962,7 +1964,7 @@ static void update_cpu_load(struct rq *this_rq)
1962 goto do_avg; 1964 goto do_avg;
1963 1965
1964 /* Update delta_fair/delta_exec fields first */ 1966 /* Update delta_fair/delta_exec fields first */
1965 update_curr_load(this_rq, now); 1967 update_curr_load(this_rq);
1966 1968
1967 fair_delta64 = ls->delta_fair + 1; 1969 fair_delta64 = ls->delta_fair + 1;
1968 ls->delta_fair = 0; 1970 ls->delta_fair = 0;
@@ -1970,8 +1972,8 @@ static void update_cpu_load(struct rq *this_rq)
1970 exec_delta64 = ls->delta_exec + 1; 1972 exec_delta64 = ls->delta_exec + 1;
1971 ls->delta_exec = 0; 1973 ls->delta_exec = 0;
1972 1974
1973 sample_interval64 = now - ls->load_update_last; 1975 sample_interval64 = this_rq->clock - ls->load_update_last;
1974 ls->load_update_last = now; 1976 ls->load_update_last = this_rq->clock;
1975 1977
1976 if ((s64)sample_interval64 < (s64)TICK_NSEC) 1978 if ((s64)sample_interval64 < (s64)TICK_NSEC)
1977 sample_interval64 = TICK_NSEC; 1979 sample_interval64 = TICK_NSEC;
@@ -2026,6 +2028,8 @@ static void double_rq_lock(struct rq *rq1, struct rq *rq2)
2026 spin_lock(&rq1->lock); 2028 spin_lock(&rq1->lock);
2027 } 2029 }
2028 } 2030 }
2031 update_rq_clock(rq1);
2032 update_rq_clock(rq2);
2029} 2033}
2030 2034
2031/* 2035/*
@@ -2166,8 +2170,7 @@ static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
2166 unsigned long max_nr_move, unsigned long max_load_move, 2170 unsigned long max_nr_move, unsigned long max_load_move,
2167 struct sched_domain *sd, enum cpu_idle_type idle, 2171 struct sched_domain *sd, enum cpu_idle_type idle,
2168 int *all_pinned, unsigned long *load_moved, 2172 int *all_pinned, unsigned long *load_moved,
2169 int this_best_prio, int best_prio, int best_prio_seen, 2173 int *this_best_prio, struct rq_iterator *iterator)
2170 struct rq_iterator *iterator)
2171{ 2174{
2172 int pulled = 0, pinned = 0, skip_for_load; 2175 int pulled = 0, pinned = 0, skip_for_load;
2173 struct task_struct *p; 2176 struct task_struct *p;
@@ -2192,12 +2195,8 @@ next:
2192 */ 2195 */
2193 skip_for_load = (p->se.load.weight >> 1) > rem_load_move + 2196 skip_for_load = (p->se.load.weight >> 1) > rem_load_move +
2194 SCHED_LOAD_SCALE_FUZZ; 2197 SCHED_LOAD_SCALE_FUZZ;
2195 if (skip_for_load && p->prio < this_best_prio) 2198 if ((skip_for_load && p->prio >= *this_best_prio) ||
2196 skip_for_load = !best_prio_seen && p->prio == best_prio;
2197 if (skip_for_load ||
2198 !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) { 2199 !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
2199
2200 best_prio_seen |= p->prio == best_prio;
2201 p = iterator->next(iterator->arg); 2200 p = iterator->next(iterator->arg);
2202 goto next; 2201 goto next;
2203 } 2202 }
@@ -2211,8 +2210,8 @@ next:
2211 * and the prescribed amount of weighted load. 2210 * and the prescribed amount of weighted load.
2212 */ 2211 */
2213 if (pulled < max_nr_move && rem_load_move > 0) { 2212 if (pulled < max_nr_move && rem_load_move > 0) {
2214 if (p->prio < this_best_prio) 2213 if (p->prio < *this_best_prio)
2215 this_best_prio = p->prio; 2214 *this_best_prio = p->prio;
2216 p = iterator->next(iterator->arg); 2215 p = iterator->next(iterator->arg);
2217 goto next; 2216 goto next;
2218 } 2217 }
@@ -2231,32 +2230,52 @@ out:
2231} 2230}
2232 2231
2233/* 2232/*
2234 * move_tasks tries to move up to max_nr_move tasks and max_load_move weighted 2233 * move_tasks tries to move up to max_load_move weighted load from busiest to
2235 * load from busiest to this_rq, as part of a balancing operation within 2234 * this_rq, as part of a balancing operation within domain "sd".
2236 * "domain". Returns the number of tasks moved. 2235 * Returns 1 if successful and 0 otherwise.
2237 * 2236 *
2238 * Called with both runqueues locked. 2237 * Called with both runqueues locked.
2239 */ 2238 */
2240static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, 2239static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
2241 unsigned long max_nr_move, unsigned long max_load_move, 2240 unsigned long max_load_move,
2242 struct sched_domain *sd, enum cpu_idle_type idle, 2241 struct sched_domain *sd, enum cpu_idle_type idle,
2243 int *all_pinned) 2242 int *all_pinned)
2244{ 2243{
2245 struct sched_class *class = sched_class_highest; 2244 struct sched_class *class = sched_class_highest;
2246 unsigned long load_moved, total_nr_moved = 0, nr_moved; 2245 unsigned long total_load_moved = 0;
2247 long rem_load_move = max_load_move; 2246 int this_best_prio = this_rq->curr->prio;
2248 2247
2249 do { 2248 do {
2250 nr_moved = class->load_balance(this_rq, this_cpu, busiest, 2249 total_load_moved +=
2251 max_nr_move, (unsigned long)rem_load_move, 2250 class->load_balance(this_rq, this_cpu, busiest,
2252 sd, idle, all_pinned, &load_moved); 2251 ULONG_MAX, max_load_move - total_load_moved,
2253 total_nr_moved += nr_moved; 2252 sd, idle, all_pinned, &this_best_prio);
2254 max_nr_move -= nr_moved;
2255 rem_load_move -= load_moved;
2256 class = class->next; 2253 class = class->next;
2257 } while (class && max_nr_move && rem_load_move > 0); 2254 } while (class && max_load_move > total_load_moved);
2258 2255
2259 return total_nr_moved; 2256 return total_load_moved > 0;
2257}
2258
2259/*
2260 * move_one_task tries to move exactly one task from busiest to this_rq, as
2261 * part of active balancing operations within "domain".
2262 * Returns 1 if successful and 0 otherwise.
2263 *
2264 * Called with both runqueues locked.
2265 */
2266static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
2267 struct sched_domain *sd, enum cpu_idle_type idle)
2268{
2269 struct sched_class *class;
2270 int this_best_prio = MAX_PRIO;
2271
2272 for (class = sched_class_highest; class; class = class->next)
2273 if (class->load_balance(this_rq, this_cpu, busiest,
2274 1, ULONG_MAX, sd, idle, NULL,
2275 &this_best_prio))
2276 return 1;
2277
2278 return 0;
2260} 2279}
2261 2280
2262/* 2281/*
@@ -2588,11 +2607,6 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
2588 */ 2607 */
2589#define MAX_PINNED_INTERVAL 512 2608#define MAX_PINNED_INTERVAL 512
2590 2609
2591static inline unsigned long minus_1_or_zero(unsigned long n)
2592{
2593 return n > 0 ? n - 1 : 0;
2594}
2595
2596/* 2610/*
2597 * Check this_cpu to ensure it is balanced within domain. Attempt to move 2611 * Check this_cpu to ensure it is balanced within domain. Attempt to move
2598 * tasks if there is an imbalance. 2612 * tasks if there is an imbalance.
@@ -2601,7 +2615,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
2601 struct sched_domain *sd, enum cpu_idle_type idle, 2615 struct sched_domain *sd, enum cpu_idle_type idle,
2602 int *balance) 2616 int *balance)
2603{ 2617{
2604 int nr_moved, all_pinned = 0, active_balance = 0, sd_idle = 0; 2618 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
2605 struct sched_group *group; 2619 struct sched_group *group;
2606 unsigned long imbalance; 2620 unsigned long imbalance;
2607 struct rq *busiest; 2621 struct rq *busiest;
@@ -2642,18 +2656,17 @@ redo:
2642 2656
2643 schedstat_add(sd, lb_imbalance[idle], imbalance); 2657 schedstat_add(sd, lb_imbalance[idle], imbalance);
2644 2658
2645 nr_moved = 0; 2659 ld_moved = 0;
2646 if (busiest->nr_running > 1) { 2660 if (busiest->nr_running > 1) {
2647 /* 2661 /*
2648 * Attempt to move tasks. If find_busiest_group has found 2662 * Attempt to move tasks. If find_busiest_group has found
2649 * an imbalance but busiest->nr_running <= 1, the group is 2663 * an imbalance but busiest->nr_running <= 1, the group is
2650 * still unbalanced. nr_moved simply stays zero, so it is 2664 * still unbalanced. ld_moved simply stays zero, so it is
2651 * correctly treated as an imbalance. 2665 * correctly treated as an imbalance.
2652 */ 2666 */
2653 local_irq_save(flags); 2667 local_irq_save(flags);
2654 double_rq_lock(this_rq, busiest); 2668 double_rq_lock(this_rq, busiest);
2655 nr_moved = move_tasks(this_rq, this_cpu, busiest, 2669 ld_moved = move_tasks(this_rq, this_cpu, busiest,
2656 minus_1_or_zero(busiest->nr_running),
2657 imbalance, sd, idle, &all_pinned); 2670 imbalance, sd, idle, &all_pinned);
2658 double_rq_unlock(this_rq, busiest); 2671 double_rq_unlock(this_rq, busiest);
2659 local_irq_restore(flags); 2672 local_irq_restore(flags);
@@ -2661,7 +2674,7 @@ redo:
2661 /* 2674 /*
2662 * some other cpu did the load balance for us. 2675 * some other cpu did the load balance for us.
2663 */ 2676 */
2664 if (nr_moved && this_cpu != smp_processor_id()) 2677 if (ld_moved && this_cpu != smp_processor_id())
2665 resched_cpu(this_cpu); 2678 resched_cpu(this_cpu);
2666 2679
2667 /* All tasks on this runqueue were pinned by CPU affinity */ 2680 /* All tasks on this runqueue were pinned by CPU affinity */
@@ -2673,7 +2686,7 @@ redo:
2673 } 2686 }
2674 } 2687 }
2675 2688
2676 if (!nr_moved) { 2689 if (!ld_moved) {
2677 schedstat_inc(sd, lb_failed[idle]); 2690 schedstat_inc(sd, lb_failed[idle]);
2678 sd->nr_balance_failed++; 2691 sd->nr_balance_failed++;
2679 2692
@@ -2722,10 +2735,10 @@ redo:
2722 sd->balance_interval *= 2; 2735 sd->balance_interval *= 2;
2723 } 2736 }
2724 2737
2725 if (!nr_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER && 2738 if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2726 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE)) 2739 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
2727 return -1; 2740 return -1;
2728 return nr_moved; 2741 return ld_moved;
2729 2742
2730out_balanced: 2743out_balanced:
2731 schedstat_inc(sd, lb_balanced[idle]); 2744 schedstat_inc(sd, lb_balanced[idle]);
@@ -2757,7 +2770,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd)
2757 struct sched_group *group; 2770 struct sched_group *group;
2758 struct rq *busiest = NULL; 2771 struct rq *busiest = NULL;
2759 unsigned long imbalance; 2772 unsigned long imbalance;
2760 int nr_moved = 0; 2773 int ld_moved = 0;
2761 int sd_idle = 0; 2774 int sd_idle = 0;
2762 int all_pinned = 0; 2775 int all_pinned = 0;
2763 cpumask_t cpus = CPU_MASK_ALL; 2776 cpumask_t cpus = CPU_MASK_ALL;
@@ -2792,12 +2805,13 @@ redo:
2792 2805
2793 schedstat_add(sd, lb_imbalance[CPU_NEWLY_IDLE], imbalance); 2806 schedstat_add(sd, lb_imbalance[CPU_NEWLY_IDLE], imbalance);
2794 2807
2795 nr_moved = 0; 2808 ld_moved = 0;
2796 if (busiest->nr_running > 1) { 2809 if (busiest->nr_running > 1) {
2797 /* Attempt to move tasks */ 2810 /* Attempt to move tasks */
2798 double_lock_balance(this_rq, busiest); 2811 double_lock_balance(this_rq, busiest);
2799 nr_moved = move_tasks(this_rq, this_cpu, busiest, 2812 /* this_rq->clock is already updated */
2800 minus_1_or_zero(busiest->nr_running), 2813 update_rq_clock(busiest);
2814 ld_moved = move_tasks(this_rq, this_cpu, busiest,
2801 imbalance, sd, CPU_NEWLY_IDLE, 2815 imbalance, sd, CPU_NEWLY_IDLE,
2802 &all_pinned); 2816 &all_pinned);
2803 spin_unlock(&busiest->lock); 2817 spin_unlock(&busiest->lock);
@@ -2809,7 +2823,7 @@ redo:
2809 } 2823 }
2810 } 2824 }
2811 2825
2812 if (!nr_moved) { 2826 if (!ld_moved) {
2813 schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]); 2827 schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
2814 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER && 2828 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2815 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE)) 2829 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
@@ -2817,7 +2831,7 @@ redo:
2817 } else 2831 } else
2818 sd->nr_balance_failed = 0; 2832 sd->nr_balance_failed = 0;
2819 2833
2820 return nr_moved; 2834 return ld_moved;
2821 2835
2822out_balanced: 2836out_balanced:
2823 schedstat_inc(sd, lb_balanced[CPU_NEWLY_IDLE]); 2837 schedstat_inc(sd, lb_balanced[CPU_NEWLY_IDLE]);
@@ -2894,6 +2908,8 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
2894 2908
2895 /* move a task from busiest_rq to target_rq */ 2909 /* move a task from busiest_rq to target_rq */
2896 double_lock_balance(busiest_rq, target_rq); 2910 double_lock_balance(busiest_rq, target_rq);
2911 update_rq_clock(busiest_rq);
2912 update_rq_clock(target_rq);
2897 2913
2898 /* Search for an sd spanning us and the target CPU. */ 2914 /* Search for an sd spanning us and the target CPU. */
2899 for_each_domain(target_cpu, sd) { 2915 for_each_domain(target_cpu, sd) {
@@ -2905,8 +2921,8 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
2905 if (likely(sd)) { 2921 if (likely(sd)) {
2906 schedstat_inc(sd, alb_cnt); 2922 schedstat_inc(sd, alb_cnt);
2907 2923
2908 if (move_tasks(target_rq, target_cpu, busiest_rq, 1, 2924 if (move_one_task(target_rq, target_cpu, busiest_rq,
2909 ULONG_MAX, sd, CPU_IDLE, NULL)) 2925 sd, CPU_IDLE))
2910 schedstat_inc(sd, alb_pushed); 2926 schedstat_inc(sd, alb_pushed);
2911 else 2927 else
2912 schedstat_inc(sd, alb_failed); 2928 schedstat_inc(sd, alb_failed);
@@ -3090,7 +3106,7 @@ static void run_rebalance_domains(struct softirq_action *h)
3090 if (need_resched()) 3106 if (need_resched())
3091 break; 3107 break;
3092 3108
3093 rebalance_domains(balance_cpu, SCHED_IDLE); 3109 rebalance_domains(balance_cpu, CPU_IDLE);
3094 3110
3095 rq = cpu_rq(balance_cpu); 3111 rq = cpu_rq(balance_cpu);
3096 if (time_after(this_rq->next_balance, rq->next_balance)) 3112 if (time_after(this_rq->next_balance, rq->next_balance))
@@ -3175,8 +3191,7 @@ static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
3175 unsigned long max_nr_move, unsigned long max_load_move, 3191 unsigned long max_nr_move, unsigned long max_load_move,
3176 struct sched_domain *sd, enum cpu_idle_type idle, 3192 struct sched_domain *sd, enum cpu_idle_type idle,
3177 int *all_pinned, unsigned long *load_moved, 3193 int *all_pinned, unsigned long *load_moved,
3178 int this_best_prio, int best_prio, int best_prio_seen, 3194 int *this_best_prio, struct rq_iterator *iterator)
3179 struct rq_iterator *iterator)
3180{ 3195{
3181 *load_moved = 0; 3196 *load_moved = 0;
3182 3197
@@ -3202,7 +3217,8 @@ unsigned long long task_sched_runtime(struct task_struct *p)
3202 rq = task_rq_lock(p, &flags); 3217 rq = task_rq_lock(p, &flags);
3203 ns = p->se.sum_exec_runtime; 3218 ns = p->se.sum_exec_runtime;
3204 if (rq->curr == p) { 3219 if (rq->curr == p) {
3205 delta_exec = rq_clock(rq) - p->se.exec_start; 3220 update_rq_clock(rq);
3221 delta_exec = rq->clock - p->se.exec_start;
3206 if ((s64)delta_exec > 0) 3222 if ((s64)delta_exec > 0)
3207 ns += delta_exec; 3223 ns += delta_exec;
3208 } 3224 }
@@ -3296,11 +3312,19 @@ void scheduler_tick(void)
3296 int cpu = smp_processor_id(); 3312 int cpu = smp_processor_id();
3297 struct rq *rq = cpu_rq(cpu); 3313 struct rq *rq = cpu_rq(cpu);
3298 struct task_struct *curr = rq->curr; 3314 struct task_struct *curr = rq->curr;
3315 u64 next_tick = rq->tick_timestamp + TICK_NSEC;
3299 3316
3300 spin_lock(&rq->lock); 3317 spin_lock(&rq->lock);
3318 __update_rq_clock(rq);
3319 /*
3320 * Let rq->clock advance by at least TICK_NSEC:
3321 */
3322 if (unlikely(rq->clock < next_tick))
3323 rq->clock = next_tick;
3324 rq->tick_timestamp = rq->clock;
3325 update_cpu_load(rq);
3301 if (curr != rq->idle) /* FIXME: needed? */ 3326 if (curr != rq->idle) /* FIXME: needed? */
3302 curr->sched_class->task_tick(rq, curr); 3327 curr->sched_class->task_tick(rq, curr);
3303 update_cpu_load(rq);
3304 spin_unlock(&rq->lock); 3328 spin_unlock(&rq->lock);
3305 3329
3306#ifdef CONFIG_SMP 3330#ifdef CONFIG_SMP
@@ -3382,7 +3406,7 @@ static inline void schedule_debug(struct task_struct *prev)
3382 * Pick up the highest-prio task: 3406 * Pick up the highest-prio task:
3383 */ 3407 */
3384static inline struct task_struct * 3408static inline struct task_struct *
3385pick_next_task(struct rq *rq, struct task_struct *prev, u64 now) 3409pick_next_task(struct rq *rq, struct task_struct *prev)
3386{ 3410{
3387 struct sched_class *class; 3411 struct sched_class *class;
3388 struct task_struct *p; 3412 struct task_struct *p;
@@ -3392,14 +3416,14 @@ pick_next_task(struct rq *rq, struct task_struct *prev, u64 now)
3392 * the fair class we can call that function directly: 3416 * the fair class we can call that function directly:
3393 */ 3417 */
3394 if (likely(rq->nr_running == rq->cfs.nr_running)) { 3418 if (likely(rq->nr_running == rq->cfs.nr_running)) {
3395 p = fair_sched_class.pick_next_task(rq, now); 3419 p = fair_sched_class.pick_next_task(rq);
3396 if (likely(p)) 3420 if (likely(p))
3397 return p; 3421 return p;
3398 } 3422 }
3399 3423
3400 class = sched_class_highest; 3424 class = sched_class_highest;
3401 for ( ; ; ) { 3425 for ( ; ; ) {
3402 p = class->pick_next_task(rq, now); 3426 p = class->pick_next_task(rq);
3403 if (p) 3427 if (p)
3404 return p; 3428 return p;
3405 /* 3429 /*
@@ -3418,7 +3442,6 @@ asmlinkage void __sched schedule(void)
3418 struct task_struct *prev, *next; 3442 struct task_struct *prev, *next;
3419 long *switch_count; 3443 long *switch_count;
3420 struct rq *rq; 3444 struct rq *rq;
3421 u64 now;
3422 int cpu; 3445 int cpu;
3423 3446
3424need_resched: 3447need_resched:
@@ -3436,6 +3459,7 @@ need_resched_nonpreemptible:
3436 3459
3437 spin_lock_irq(&rq->lock); 3460 spin_lock_irq(&rq->lock);
3438 clear_tsk_need_resched(prev); 3461 clear_tsk_need_resched(prev);
3462 __update_rq_clock(rq);
3439 3463
3440 if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { 3464 if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
3441 if (unlikely((prev->state & TASK_INTERRUPTIBLE) && 3465 if (unlikely((prev->state & TASK_INTERRUPTIBLE) &&
@@ -3450,9 +3474,8 @@ need_resched_nonpreemptible:
3450 if (unlikely(!rq->nr_running)) 3474 if (unlikely(!rq->nr_running))
3451 idle_balance(cpu, rq); 3475 idle_balance(cpu, rq);
3452 3476
3453 now = __rq_clock(rq); 3477 prev->sched_class->put_prev_task(rq, prev);
3454 prev->sched_class->put_prev_task(rq, prev, now); 3478 next = pick_next_task(rq, prev);
3455 next = pick_next_task(rq, prev, now);
3456 3479
3457 sched_info_switch(prev, next); 3480 sched_info_switch(prev, next);
3458 3481
@@ -3895,17 +3918,16 @@ void rt_mutex_setprio(struct task_struct *p, int prio)
3895 unsigned long flags; 3918 unsigned long flags;
3896 int oldprio, on_rq; 3919 int oldprio, on_rq;
3897 struct rq *rq; 3920 struct rq *rq;
3898 u64 now;
3899 3921
3900 BUG_ON(prio < 0 || prio > MAX_PRIO); 3922 BUG_ON(prio < 0 || prio > MAX_PRIO);
3901 3923
3902 rq = task_rq_lock(p, &flags); 3924 rq = task_rq_lock(p, &flags);
3903 now = rq_clock(rq); 3925 update_rq_clock(rq);
3904 3926
3905 oldprio = p->prio; 3927 oldprio = p->prio;
3906 on_rq = p->se.on_rq; 3928 on_rq = p->se.on_rq;
3907 if (on_rq) 3929 if (on_rq)
3908 dequeue_task(rq, p, 0, now); 3930 dequeue_task(rq, p, 0);
3909 3931
3910 if (rt_prio(prio)) 3932 if (rt_prio(prio))
3911 p->sched_class = &rt_sched_class; 3933 p->sched_class = &rt_sched_class;
@@ -3915,7 +3937,7 @@ void rt_mutex_setprio(struct task_struct *p, int prio)
3915 p->prio = prio; 3937 p->prio = prio;
3916 3938
3917 if (on_rq) { 3939 if (on_rq) {
3918 enqueue_task(rq, p, 0, now); 3940 enqueue_task(rq, p, 0);
3919 /* 3941 /*
3920 * Reschedule if we are currently running on this runqueue and 3942 * Reschedule if we are currently running on this runqueue and
3921 * our priority decreased, or if we are not currently running on 3943 * our priority decreased, or if we are not currently running on
@@ -3938,7 +3960,6 @@ void set_user_nice(struct task_struct *p, long nice)
3938 int old_prio, delta, on_rq; 3960 int old_prio, delta, on_rq;
3939 unsigned long flags; 3961 unsigned long flags;
3940 struct rq *rq; 3962 struct rq *rq;
3941 u64 now;
3942 3963
3943 if (TASK_NICE(p) == nice || nice < -20 || nice > 19) 3964 if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
3944 return; 3965 return;
@@ -3947,7 +3968,7 @@ void set_user_nice(struct task_struct *p, long nice)
3947 * the task might be in the middle of scheduling on another CPU. 3968 * the task might be in the middle of scheduling on another CPU.
3948 */ 3969 */
3949 rq = task_rq_lock(p, &flags); 3970 rq = task_rq_lock(p, &flags);
3950 now = rq_clock(rq); 3971 update_rq_clock(rq);
3951 /* 3972 /*
3952 * The RT priorities are set via sched_setscheduler(), but we still 3973 * The RT priorities are set via sched_setscheduler(), but we still
3953 * allow the 'normal' nice value to be set - but as expected 3974 * allow the 'normal' nice value to be set - but as expected
@@ -3960,8 +3981,8 @@ void set_user_nice(struct task_struct *p, long nice)
3960 } 3981 }
3961 on_rq = p->se.on_rq; 3982 on_rq = p->se.on_rq;
3962 if (on_rq) { 3983 if (on_rq) {
3963 dequeue_task(rq, p, 0, now); 3984 dequeue_task(rq, p, 0);
3964 dec_load(rq, p, now); 3985 dec_load(rq, p);
3965 } 3986 }
3966 3987
3967 p->static_prio = NICE_TO_PRIO(nice); 3988 p->static_prio = NICE_TO_PRIO(nice);
@@ -3971,8 +3992,8 @@ void set_user_nice(struct task_struct *p, long nice)
3971 delta = p->prio - old_prio; 3992 delta = p->prio - old_prio;
3972 3993
3973 if (on_rq) { 3994 if (on_rq) {
3974 enqueue_task(rq, p, 0, now); 3995 enqueue_task(rq, p, 0);
3975 inc_load(rq, p, now); 3996 inc_load(rq, p);
3976 /* 3997 /*
3977 * If the task increased its priority or is running and 3998 * If the task increased its priority or is running and
3978 * lowered its priority, then reschedule its CPU: 3999 * lowered its priority, then reschedule its CPU:
@@ -4208,6 +4229,7 @@ recheck:
4208 spin_unlock_irqrestore(&p->pi_lock, flags); 4229 spin_unlock_irqrestore(&p->pi_lock, flags);
4209 goto recheck; 4230 goto recheck;
4210 } 4231 }
4232 update_rq_clock(rq);
4211 on_rq = p->se.on_rq; 4233 on_rq = p->se.on_rq;
4212 if (on_rq) 4234 if (on_rq)
4213 deactivate_task(rq, p, 0); 4235 deactivate_task(rq, p, 0);
@@ -4463,10 +4485,8 @@ long sched_getaffinity(pid_t pid, cpumask_t *mask)
4463out_unlock: 4485out_unlock:
4464 read_unlock(&tasklist_lock); 4486 read_unlock(&tasklist_lock);
4465 mutex_unlock(&sched_hotcpu_mutex); 4487 mutex_unlock(&sched_hotcpu_mutex);
4466 if (retval)
4467 return retval;
4468 4488
4469 return 0; 4489 return retval;
4470} 4490}
4471 4491
4472/** 4492/**
@@ -4966,6 +4986,7 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
4966 on_rq = p->se.on_rq; 4986 on_rq = p->se.on_rq;
4967 if (on_rq) 4987 if (on_rq)
4968 deactivate_task(rq_src, p, 0); 4988 deactivate_task(rq_src, p, 0);
4989
4969 set_task_cpu(p, dest_cpu); 4990 set_task_cpu(p, dest_cpu);
4970 if (on_rq) { 4991 if (on_rq) {
4971 activate_task(rq_dest, p, 0); 4992 activate_task(rq_dest, p, 0);
@@ -5198,7 +5219,8 @@ static void migrate_dead_tasks(unsigned int dead_cpu)
5198 for ( ; ; ) { 5219 for ( ; ; ) {
5199 if (!rq->nr_running) 5220 if (!rq->nr_running)
5200 break; 5221 break;
5201 next = pick_next_task(rq, rq->curr, rq_clock(rq)); 5222 update_rq_clock(rq);
5223 next = pick_next_task(rq, rq->curr);
5202 if (!next) 5224 if (!next)
5203 break; 5225 break;
5204 migrate_dead(dead_cpu, next); 5226 migrate_dead(dead_cpu, next);
@@ -5210,12 +5232,19 @@ static void migrate_dead_tasks(unsigned int dead_cpu)
5210#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL) 5232#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
5211 5233
5212static struct ctl_table sd_ctl_dir[] = { 5234static struct ctl_table sd_ctl_dir[] = {
5213 {CTL_UNNUMBERED, "sched_domain", NULL, 0, 0755, NULL, }, 5235 {
5236 .procname = "sched_domain",
5237 .mode = 0755,
5238 },
5214 {0,}, 5239 {0,},
5215}; 5240};
5216 5241
5217static struct ctl_table sd_ctl_root[] = { 5242static struct ctl_table sd_ctl_root[] = {
5218 {CTL_UNNUMBERED, "kernel", NULL, 0, 0755, sd_ctl_dir, }, 5243 {
5244 .procname = "kernel",
5245 .mode = 0755,
5246 .child = sd_ctl_dir,
5247 },
5219 {0,}, 5248 {0,},
5220}; 5249};
5221 5250
@@ -5231,11 +5260,10 @@ static struct ctl_table *sd_alloc_ctl_entry(int n)
5231} 5260}
5232 5261
5233static void 5262static void
5234set_table_entry(struct ctl_table *entry, int ctl_name, 5263set_table_entry(struct ctl_table *entry,
5235 const char *procname, void *data, int maxlen, 5264 const char *procname, void *data, int maxlen,
5236 mode_t mode, proc_handler *proc_handler) 5265 mode_t mode, proc_handler *proc_handler)
5237{ 5266{
5238 entry->ctl_name = ctl_name;
5239 entry->procname = procname; 5267 entry->procname = procname;
5240 entry->data = data; 5268 entry->data = data;
5241 entry->maxlen = maxlen; 5269 entry->maxlen = maxlen;
@@ -5248,28 +5276,28 @@ sd_alloc_ctl_domain_table(struct sched_domain *sd)
5248{ 5276{
5249 struct ctl_table *table = sd_alloc_ctl_entry(14); 5277 struct ctl_table *table = sd_alloc_ctl_entry(14);
5250 5278
5251 set_table_entry(&table[0], 1, "min_interval", &sd->min_interval, 5279 set_table_entry(&table[0], "min_interval", &sd->min_interval,
5252 sizeof(long), 0644, proc_doulongvec_minmax); 5280 sizeof(long), 0644, proc_doulongvec_minmax);
5253 set_table_entry(&table[1], 2, "max_interval", &sd->max_interval, 5281 set_table_entry(&table[1], "max_interval", &sd->max_interval,
5254 sizeof(long), 0644, proc_doulongvec_minmax); 5282 sizeof(long), 0644, proc_doulongvec_minmax);
5255 set_table_entry(&table[2], 3, "busy_idx", &sd->busy_idx, 5283 set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
5256 sizeof(int), 0644, proc_dointvec_minmax); 5284 sizeof(int), 0644, proc_dointvec_minmax);
5257 set_table_entry(&table[3], 4, "idle_idx", &sd->idle_idx, 5285 set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
5258 sizeof(int), 0644, proc_dointvec_minmax); 5286 sizeof(int), 0644, proc_dointvec_minmax);
5259 set_table_entry(&table[4], 5, "newidle_idx", &sd->newidle_idx, 5287 set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
5260 sizeof(int), 0644, proc_dointvec_minmax); 5288 sizeof(int), 0644, proc_dointvec_minmax);
5261 set_table_entry(&table[5], 6, "wake_idx", &sd->wake_idx, 5289 set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
5262 sizeof(int), 0644, proc_dointvec_minmax); 5290 sizeof(int), 0644, proc_dointvec_minmax);
5263 set_table_entry(&table[6], 7, "forkexec_idx", &sd->forkexec_idx, 5291 set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
5264 sizeof(int), 0644, proc_dointvec_minmax); 5292 sizeof(int), 0644, proc_dointvec_minmax);
5265 set_table_entry(&table[7], 8, "busy_factor", &sd->busy_factor, 5293 set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
5266 sizeof(int), 0644, proc_dointvec_minmax); 5294 sizeof(int), 0644, proc_dointvec_minmax);
5267 set_table_entry(&table[8], 9, "imbalance_pct", &sd->imbalance_pct, 5295 set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
5268 sizeof(int), 0644, proc_dointvec_minmax); 5296 sizeof(int), 0644, proc_dointvec_minmax);
5269 set_table_entry(&table[10], 11, "cache_nice_tries", 5297 set_table_entry(&table[10], "cache_nice_tries",
5270 &sd->cache_nice_tries, 5298 &sd->cache_nice_tries,
5271 sizeof(int), 0644, proc_dointvec_minmax); 5299 sizeof(int), 0644, proc_dointvec_minmax);
5272 set_table_entry(&table[12], 13, "flags", &sd->flags, 5300 set_table_entry(&table[12], "flags", &sd->flags,
5273 sizeof(int), 0644, proc_dointvec_minmax); 5301 sizeof(int), 0644, proc_dointvec_minmax);
5274 5302
5275 return table; 5303 return table;
@@ -5289,7 +5317,6 @@ static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
5289 i = 0; 5317 i = 0;
5290 for_each_domain(cpu, sd) { 5318 for_each_domain(cpu, sd) {
5291 snprintf(buf, 32, "domain%d", i); 5319 snprintf(buf, 32, "domain%d", i);
5292 entry->ctl_name = i + 1;
5293 entry->procname = kstrdup(buf, GFP_KERNEL); 5320 entry->procname = kstrdup(buf, GFP_KERNEL);
5294 entry->mode = 0755; 5321 entry->mode = 0755;
5295 entry->child = sd_alloc_ctl_domain_table(sd); 5322 entry->child = sd_alloc_ctl_domain_table(sd);
@@ -5310,7 +5337,6 @@ static void init_sched_domain_sysctl(void)
5310 5337
5311 for (i = 0; i < cpu_num; i++, entry++) { 5338 for (i = 0; i < cpu_num; i++, entry++) {
5312 snprintf(buf, 32, "cpu%d", i); 5339 snprintf(buf, 32, "cpu%d", i);
5313 entry->ctl_name = i + 1;
5314 entry->procname = kstrdup(buf, GFP_KERNEL); 5340 entry->procname = kstrdup(buf, GFP_KERNEL);
5315 entry->mode = 0755; 5341 entry->mode = 0755;
5316 entry->child = sd_alloc_ctl_cpu_table(i); 5342 entry->child = sd_alloc_ctl_cpu_table(i);
@@ -5379,6 +5405,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
5379 rq->migration_thread = NULL; 5405 rq->migration_thread = NULL;
5380 /* Idle task back to normal (off runqueue, low prio) */ 5406 /* Idle task back to normal (off runqueue, low prio) */
5381 rq = task_rq_lock(rq->idle, &flags); 5407 rq = task_rq_lock(rq->idle, &flags);
5408 update_rq_clock(rq);
5382 deactivate_task(rq, rq->idle, 0); 5409 deactivate_task(rq, rq->idle, 0);
5383 rq->idle->static_prio = MAX_PRIO; 5410 rq->idle->static_prio = MAX_PRIO;
5384 __setscheduler(rq, rq->idle, SCHED_NORMAL, 0); 5411 __setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
@@ -6301,7 +6328,7 @@ int partition_sched_domains(cpumask_t *partition1, cpumask_t *partition2)
6301} 6328}
6302 6329
6303#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT) 6330#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
6304int arch_reinit_sched_domains(void) 6331static int arch_reinit_sched_domains(void)
6305{ 6332{
6306 int err; 6333 int err;
6307 6334
@@ -6330,24 +6357,6 @@ static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
6330 return ret ? ret : count; 6357 return ret ? ret : count;
6331} 6358}
6332 6359
6333int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
6334{
6335 int err = 0;
6336
6337#ifdef CONFIG_SCHED_SMT
6338 if (smt_capable())
6339 err = sysfs_create_file(&cls->kset.kobj,
6340 &attr_sched_smt_power_savings.attr);
6341#endif
6342#ifdef CONFIG_SCHED_MC
6343 if (!err && mc_capable())
6344 err = sysfs_create_file(&cls->kset.kobj,
6345 &attr_sched_mc_power_savings.attr);
6346#endif
6347 return err;
6348}
6349#endif
6350
6351#ifdef CONFIG_SCHED_MC 6360#ifdef CONFIG_SCHED_MC
6352static ssize_t sched_mc_power_savings_show(struct sys_device *dev, char *page) 6361static ssize_t sched_mc_power_savings_show(struct sys_device *dev, char *page)
6353{ 6362{
@@ -6358,8 +6367,8 @@ static ssize_t sched_mc_power_savings_store(struct sys_device *dev,
6358{ 6367{
6359 return sched_power_savings_store(buf, count, 0); 6368 return sched_power_savings_store(buf, count, 0);
6360} 6369}
6361SYSDEV_ATTR(sched_mc_power_savings, 0644, sched_mc_power_savings_show, 6370static SYSDEV_ATTR(sched_mc_power_savings, 0644, sched_mc_power_savings_show,
6362 sched_mc_power_savings_store); 6371 sched_mc_power_savings_store);
6363#endif 6372#endif
6364 6373
6365#ifdef CONFIG_SCHED_SMT 6374#ifdef CONFIG_SCHED_SMT
@@ -6372,8 +6381,26 @@ static ssize_t sched_smt_power_savings_store(struct sys_device *dev,
6372{ 6381{
6373 return sched_power_savings_store(buf, count, 1); 6382 return sched_power_savings_store(buf, count, 1);
6374} 6383}
6375SYSDEV_ATTR(sched_smt_power_savings, 0644, sched_smt_power_savings_show, 6384static SYSDEV_ATTR(sched_smt_power_savings, 0644, sched_smt_power_savings_show,
6376 sched_smt_power_savings_store); 6385 sched_smt_power_savings_store);
6386#endif
6387
6388int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
6389{
6390 int err = 0;
6391
6392#ifdef CONFIG_SCHED_SMT
6393 if (smt_capable())
6394 err = sysfs_create_file(&cls->kset.kobj,
6395 &attr_sched_smt_power_savings.attr);
6396#endif
6397#ifdef CONFIG_SCHED_MC
6398 if (!err && mc_capable())
6399 err = sysfs_create_file(&cls->kset.kobj,
6400 &attr_sched_mc_power_savings.attr);
6401#endif
6402 return err;
6403}
6377#endif 6404#endif
6378 6405
6379/* 6406/*
@@ -6616,12 +6643,13 @@ void normalize_rt_tasks(void)
6616 goto out_unlock; 6643 goto out_unlock;
6617#endif 6644#endif
6618 6645
6646 update_rq_clock(rq);
6619 on_rq = p->se.on_rq; 6647 on_rq = p->se.on_rq;
6620 if (on_rq) 6648 if (on_rq)
6621 deactivate_task(task_rq(p), p, 0); 6649 deactivate_task(rq, p, 0);
6622 __setscheduler(rq, p, SCHED_NORMAL, 0); 6650 __setscheduler(rq, p, SCHED_NORMAL, 0);
6623 if (on_rq) { 6651 if (on_rq) {
6624 activate_task(task_rq(p), p, 0); 6652 activate_task(rq, p, 0);
6625 resched_task(rq->curr); 6653 resched_task(rq->curr);
6626 } 6654 }
6627#ifdef CONFIG_SMP 6655#ifdef CONFIG_SMP
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c
index 1c61e5315a..87e524762b 100644
--- a/kernel/sched_debug.c
+++ b/kernel/sched_debug.c
@@ -29,34 +29,34 @@
29 } while (0) 29 } while (0)
30 30
31static void 31static void
32print_task(struct seq_file *m, struct rq *rq, struct task_struct *p, u64 now) 32print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)
33{ 33{
34 if (rq->curr == p) 34 if (rq->curr == p)
35 SEQ_printf(m, "R"); 35 SEQ_printf(m, "R");
36 else 36 else
37 SEQ_printf(m, " "); 37 SEQ_printf(m, " ");
38 38
39 SEQ_printf(m, "%15s %5d %15Ld %13Ld %13Ld %9Ld %5d " 39 SEQ_printf(m, "%15s %5d %15Ld %13Ld %13Ld %9Ld %5d ",
40 "%15Ld %15Ld %15Ld %15Ld %15Ld\n",
41 p->comm, p->pid, 40 p->comm, p->pid,
42 (long long)p->se.fair_key, 41 (long long)p->se.fair_key,
43 (long long)(p->se.fair_key - rq->cfs.fair_clock), 42 (long long)(p->se.fair_key - rq->cfs.fair_clock),
44 (long long)p->se.wait_runtime, 43 (long long)p->se.wait_runtime,
45 (long long)(p->nvcsw + p->nivcsw), 44 (long long)(p->nvcsw + p->nivcsw),
46 p->prio, 45 p->prio);
47#ifdef CONFIG_SCHEDSTATS 46#ifdef CONFIG_SCHEDSTATS
47 SEQ_printf(m, "%15Ld %15Ld %15Ld %15Ld %15Ld\n",
48 (long long)p->se.sum_exec_runtime, 48 (long long)p->se.sum_exec_runtime,
49 (long long)p->se.sum_wait_runtime, 49 (long long)p->se.sum_wait_runtime,
50 (long long)p->se.sum_sleep_runtime, 50 (long long)p->se.sum_sleep_runtime,
51 (long long)p->se.wait_runtime_overruns, 51 (long long)p->se.wait_runtime_overruns,
52 (long long)p->se.wait_runtime_underruns 52 (long long)p->se.wait_runtime_underruns);
53#else 53#else
54 0LL, 0LL, 0LL, 0LL, 0LL 54 SEQ_printf(m, "%15Ld %15Ld %15Ld %15Ld %15Ld\n",
55 0LL, 0LL, 0LL, 0LL, 0LL);
55#endif 56#endif
56 );
57} 57}
58 58
59static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu, u64 now) 59static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu)
60{ 60{
61 struct task_struct *g, *p; 61 struct task_struct *g, *p;
62 62
@@ -77,7 +77,7 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu, u64 now)
77 if (!p->se.on_rq || task_cpu(p) != rq_cpu) 77 if (!p->se.on_rq || task_cpu(p) != rq_cpu)
78 continue; 78 continue;
79 79
80 print_task(m, rq, p, now); 80 print_task(m, rq, p);
81 } while_each_thread(g, p); 81 } while_each_thread(g, p);
82 82
83 read_unlock_irq(&tasklist_lock); 83 read_unlock_irq(&tasklist_lock);
@@ -106,9 +106,9 @@ print_cfs_rq_runtime_sum(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
106 (long long)wait_runtime_rq_sum); 106 (long long)wait_runtime_rq_sum);
107} 107}
108 108
109void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq, u64 now) 109void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
110{ 110{
111 SEQ_printf(m, "\ncfs_rq %p\n", cfs_rq); 111 SEQ_printf(m, "\ncfs_rq\n");
112 112
113#define P(x) \ 113#define P(x) \
114 SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(cfs_rq->x)) 114 SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(cfs_rq->x))
@@ -124,7 +124,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq, u64 now)
124 print_cfs_rq_runtime_sum(m, cpu, cfs_rq); 124 print_cfs_rq_runtime_sum(m, cpu, cfs_rq);
125} 125}
126 126
127static void print_cpu(struct seq_file *m, int cpu, u64 now) 127static void print_cpu(struct seq_file *m, int cpu)
128{ 128{
129 struct rq *rq = &per_cpu(runqueues, cpu); 129 struct rq *rq = &per_cpu(runqueues, cpu);
130 130
@@ -166,9 +166,9 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now)
166 P(cpu_load[4]); 166 P(cpu_load[4]);
167#undef P 167#undef P
168 168
169 print_cfs_stats(m, cpu, now); 169 print_cfs_stats(m, cpu);
170 170
171 print_rq(m, rq, cpu, now); 171 print_rq(m, rq, cpu);
172} 172}
173 173
174static int sched_debug_show(struct seq_file *m, void *v) 174static int sched_debug_show(struct seq_file *m, void *v)
@@ -184,7 +184,7 @@ static int sched_debug_show(struct seq_file *m, void *v)
184 SEQ_printf(m, "now at %Lu nsecs\n", (unsigned long long)now); 184 SEQ_printf(m, "now at %Lu nsecs\n", (unsigned long long)now);
185 185
186 for_each_online_cpu(cpu) 186 for_each_online_cpu(cpu)
187 print_cpu(m, cpu, now); 187 print_cpu(m, cpu);
188 188
189 SEQ_printf(m, "\n"); 189 SEQ_printf(m, "\n");
190 190
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 6f579ff5a9..fedbb51bba 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -75,7 +75,7 @@ enum {
75 75
76unsigned int sysctl_sched_features __read_mostly = 76unsigned int sysctl_sched_features __read_mostly =
77 SCHED_FEAT_FAIR_SLEEPERS *1 | 77 SCHED_FEAT_FAIR_SLEEPERS *1 |
78 SCHED_FEAT_SLEEPER_AVG *1 | 78 SCHED_FEAT_SLEEPER_AVG *0 |
79 SCHED_FEAT_SLEEPER_LOAD_AVG *1 | 79 SCHED_FEAT_SLEEPER_LOAD_AVG *1 |
80 SCHED_FEAT_PRECISE_CPU_LOAD *1 | 80 SCHED_FEAT_PRECISE_CPU_LOAD *1 |
81 SCHED_FEAT_START_DEBIT *1 | 81 SCHED_FEAT_START_DEBIT *1 |
@@ -222,21 +222,25 @@ niced_granularity(struct sched_entity *curr, unsigned long granularity)
222{ 222{
223 u64 tmp; 223 u64 tmp;
224 224
225 if (likely(curr->load.weight == NICE_0_LOAD))
226 return granularity;
225 /* 227 /*
226 * Negative nice levels get the same granularity as nice-0: 228 * Positive nice levels get the same granularity as nice-0:
227 */ 229 */
228 if (likely(curr->load.weight >= NICE_0_LOAD)) 230 if (likely(curr->load.weight < NICE_0_LOAD)) {
229 return granularity; 231 tmp = curr->load.weight * (u64)granularity;
232 return (long) (tmp >> NICE_0_SHIFT);
233 }
230 /* 234 /*
231 * Positive nice level tasks get linearly finer 235 * Negative nice level tasks get linearly finer
232 * granularity: 236 * granularity:
233 */ 237 */
234 tmp = curr->load.weight * (u64)granularity; 238 tmp = curr->load.inv_weight * (u64)granularity;
235 239
236 /* 240 /*
237 * It will always fit into 'long': 241 * It will always fit into 'long':
238 */ 242 */
239 return (long) (tmp >> NICE_0_SHIFT); 243 return (long) (tmp >> WMULT_SHIFT);
240} 244}
241 245
242static inline void 246static inline void
@@ -281,31 +285,28 @@ add_wait_runtime(struct cfs_rq *cfs_rq, struct sched_entity *se, long delta)
281 * are not in our scheduling class. 285 * are not in our scheduling class.
282 */ 286 */
283static inline void 287static inline void
284__update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr, u64 now) 288__update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr)
285{ 289{
286 unsigned long delta, delta_exec, delta_fair; 290 unsigned long delta, delta_exec, delta_fair, delta_mine;
287 long delta_mine;
288 struct load_weight *lw = &cfs_rq->load; 291 struct load_weight *lw = &cfs_rq->load;
289 unsigned long load = lw->weight; 292 unsigned long load = lw->weight;
290 293
291 if (unlikely(!load))
292 return;
293
294 delta_exec = curr->delta_exec; 294 delta_exec = curr->delta_exec;
295 schedstat_set(curr->exec_max, max((u64)delta_exec, curr->exec_max)); 295 schedstat_set(curr->exec_max, max((u64)delta_exec, curr->exec_max));
296 296
297 curr->sum_exec_runtime += delta_exec; 297 curr->sum_exec_runtime += delta_exec;
298 cfs_rq->exec_clock += delta_exec; 298 cfs_rq->exec_clock += delta_exec;
299 299
300 if (unlikely(!load))
301 return;
302
300 delta_fair = calc_delta_fair(delta_exec, lw); 303 delta_fair = calc_delta_fair(delta_exec, lw);
301 delta_mine = calc_delta_mine(delta_exec, curr->load.weight, lw); 304 delta_mine = calc_delta_mine(delta_exec, curr->load.weight, lw);
302 305
303 if (cfs_rq->sleeper_bonus > sysctl_sched_stat_granularity) { 306 if (cfs_rq->sleeper_bonus > sysctl_sched_granularity) {
304 delta = calc_delta_mine(cfs_rq->sleeper_bonus, 307 delta = min(cfs_rq->sleeper_bonus, (u64)delta_exec);
305 curr->load.weight, lw); 308 delta = calc_delta_mine(delta, curr->load.weight, lw);
306 if (unlikely(delta > cfs_rq->sleeper_bonus)) 309 delta = min((u64)delta, cfs_rq->sleeper_bonus);
307 delta = cfs_rq->sleeper_bonus;
308
309 cfs_rq->sleeper_bonus -= delta; 310 cfs_rq->sleeper_bonus -= delta;
310 delta_mine -= delta; 311 delta_mine -= delta;
311 } 312 }
@@ -321,7 +322,7 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr, u64 now)
321 add_wait_runtime(cfs_rq, curr, delta_mine - delta_exec); 322 add_wait_runtime(cfs_rq, curr, delta_mine - delta_exec);
322} 323}
323 324
324static void update_curr(struct cfs_rq *cfs_rq, u64 now) 325static void update_curr(struct cfs_rq *cfs_rq)
325{ 326{
326 struct sched_entity *curr = cfs_rq_curr(cfs_rq); 327 struct sched_entity *curr = cfs_rq_curr(cfs_rq);
327 unsigned long delta_exec; 328 unsigned long delta_exec;
@@ -334,22 +335,22 @@ static void update_curr(struct cfs_rq *cfs_rq, u64 now)
334 * since the last time we changed load (this cannot 335 * since the last time we changed load (this cannot
335 * overflow on 32 bits): 336 * overflow on 32 bits):
336 */ 337 */
337 delta_exec = (unsigned long)(now - curr->exec_start); 338 delta_exec = (unsigned long)(rq_of(cfs_rq)->clock - curr->exec_start);
338 339
339 curr->delta_exec += delta_exec; 340 curr->delta_exec += delta_exec;
340 341
341 if (unlikely(curr->delta_exec > sysctl_sched_stat_granularity)) { 342 if (unlikely(curr->delta_exec > sysctl_sched_stat_granularity)) {
342 __update_curr(cfs_rq, curr, now); 343 __update_curr(cfs_rq, curr);
343 curr->delta_exec = 0; 344 curr->delta_exec = 0;
344 } 345 }
345 curr->exec_start = now; 346 curr->exec_start = rq_of(cfs_rq)->clock;
346} 347}
347 348
348static inline void 349static inline void
349update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now) 350update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
350{ 351{
351 se->wait_start_fair = cfs_rq->fair_clock; 352 se->wait_start_fair = cfs_rq->fair_clock;
352 schedstat_set(se->wait_start, now); 353 schedstat_set(se->wait_start, rq_of(cfs_rq)->clock);
353} 354}
354 355
355/* 356/*
@@ -377,8 +378,7 @@ calc_weighted(unsigned long delta, unsigned long weight, int shift)
377/* 378/*
378 * Task is being enqueued - update stats: 379 * Task is being enqueued - update stats:
379 */ 380 */
380static void 381static void update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
381update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
382{ 382{
383 s64 key; 383 s64 key;
384 384
@@ -387,7 +387,7 @@ update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
387 * a dequeue/enqueue event is a NOP) 387 * a dequeue/enqueue event is a NOP)
388 */ 388 */
389 if (se != cfs_rq_curr(cfs_rq)) 389 if (se != cfs_rq_curr(cfs_rq))
390 update_stats_wait_start(cfs_rq, se, now); 390 update_stats_wait_start(cfs_rq, se);
391 /* 391 /*
392 * Update the key: 392 * Update the key:
393 */ 393 */
@@ -407,7 +407,8 @@ update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
407 (WMULT_SHIFT - NICE_0_SHIFT); 407 (WMULT_SHIFT - NICE_0_SHIFT);
408 } else { 408 } else {
409 tmp = se->wait_runtime; 409 tmp = se->wait_runtime;
410 key -= (tmp * se->load.weight) >> NICE_0_SHIFT; 410 key -= (tmp * se->load.inv_weight) >>
411 (WMULT_SHIFT - NICE_0_SHIFT);
411 } 412 }
412 } 413 }
413 414
@@ -418,11 +419,12 @@ update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
418 * Note: must be called with a freshly updated rq->fair_clock. 419 * Note: must be called with a freshly updated rq->fair_clock.
419 */ 420 */
420static inline void 421static inline void
421__update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now) 422__update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
422{ 423{
423 unsigned long delta_fair = se->delta_fair_run; 424 unsigned long delta_fair = se->delta_fair_run;
424 425
425 schedstat_set(se->wait_max, max(se->wait_max, now - se->wait_start)); 426 schedstat_set(se->wait_max, max(se->wait_max,
427 rq_of(cfs_rq)->clock - se->wait_start));
426 428
427 if (unlikely(se->load.weight != NICE_0_LOAD)) 429 if (unlikely(se->load.weight != NICE_0_LOAD))
428 delta_fair = calc_weighted(delta_fair, se->load.weight, 430 delta_fair = calc_weighted(delta_fair, se->load.weight,
@@ -432,7 +434,7 @@ __update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
432} 434}
433 435
434static void 436static void
435update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now) 437update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
436{ 438{
437 unsigned long delta_fair; 439 unsigned long delta_fair;
438 440
@@ -442,7 +444,7 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
442 se->delta_fair_run += delta_fair; 444 se->delta_fair_run += delta_fair;
443 if (unlikely(abs(se->delta_fair_run) >= 445 if (unlikely(abs(se->delta_fair_run) >=
444 sysctl_sched_stat_granularity)) { 446 sysctl_sched_stat_granularity)) {
445 __update_stats_wait_end(cfs_rq, se, now); 447 __update_stats_wait_end(cfs_rq, se);
446 se->delta_fair_run = 0; 448 se->delta_fair_run = 0;
447 } 449 }
448 450
@@ -451,34 +453,34 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
451} 453}
452 454
453static inline void 455static inline void
454update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now) 456update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
455{ 457{
456 update_curr(cfs_rq, now); 458 update_curr(cfs_rq);
457 /* 459 /*
458 * Mark the end of the wait period if dequeueing a 460 * Mark the end of the wait period if dequeueing a
459 * waiting task: 461 * waiting task:
460 */ 462 */
461 if (se != cfs_rq_curr(cfs_rq)) 463 if (se != cfs_rq_curr(cfs_rq))
462 update_stats_wait_end(cfs_rq, se, now); 464 update_stats_wait_end(cfs_rq, se);
463} 465}
464 466
465/* 467/*
466 * We are picking a new current task - update its stats: 468 * We are picking a new current task - update its stats:
467 */ 469 */
468static inline void 470static inline void
469update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now) 471update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
470{ 472{
471 /* 473 /*
472 * We are starting a new run period: 474 * We are starting a new run period:
473 */ 475 */
474 se->exec_start = now; 476 se->exec_start = rq_of(cfs_rq)->clock;
475} 477}
476 478
477/* 479/*
478 * We are descheduling a task - update its stats: 480 * We are descheduling a task - update its stats:
479 */ 481 */
480static inline void 482static inline void
481update_stats_curr_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now) 483update_stats_curr_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
482{ 484{
483 se->exec_start = 0; 485 se->exec_start = 0;
484} 486}
@@ -487,8 +489,7 @@ update_stats_curr_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
487 * Scheduling class queueing methods: 489 * Scheduling class queueing methods:
488 */ 490 */
489 491
490static void 492static void __enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
491__enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
492{ 493{
493 unsigned long load = cfs_rq->load.weight, delta_fair; 494 unsigned long load = cfs_rq->load.weight, delta_fair;
494 long prev_runtime; 495 long prev_runtime;
@@ -518,12 +519,13 @@ __enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
518 * Track the amount of bonus we've given to sleepers: 519 * Track the amount of bonus we've given to sleepers:
519 */ 520 */
520 cfs_rq->sleeper_bonus += delta_fair; 521 cfs_rq->sleeper_bonus += delta_fair;
522 if (unlikely(cfs_rq->sleeper_bonus > sysctl_sched_runtime_limit))
523 cfs_rq->sleeper_bonus = sysctl_sched_runtime_limit;
521 524
522 schedstat_add(cfs_rq, wait_runtime, se->wait_runtime); 525 schedstat_add(cfs_rq, wait_runtime, se->wait_runtime);
523} 526}
524 527
525static void 528static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
526enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
527{ 529{
528 struct task_struct *tsk = task_of(se); 530 struct task_struct *tsk = task_of(se);
529 unsigned long delta_fair; 531 unsigned long delta_fair;
@@ -538,7 +540,7 @@ enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
538 se->delta_fair_sleep += delta_fair; 540 se->delta_fair_sleep += delta_fair;
539 if (unlikely(abs(se->delta_fair_sleep) >= 541 if (unlikely(abs(se->delta_fair_sleep) >=
540 sysctl_sched_stat_granularity)) { 542 sysctl_sched_stat_granularity)) {
541 __enqueue_sleeper(cfs_rq, se, now); 543 __enqueue_sleeper(cfs_rq, se);
542 se->delta_fair_sleep = 0; 544 se->delta_fair_sleep = 0;
543 } 545 }
544 546
@@ -546,7 +548,7 @@ enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
546 548
547#ifdef CONFIG_SCHEDSTATS 549#ifdef CONFIG_SCHEDSTATS
548 if (se->sleep_start) { 550 if (se->sleep_start) {
549 u64 delta = now - se->sleep_start; 551 u64 delta = rq_of(cfs_rq)->clock - se->sleep_start;
550 552
551 if ((s64)delta < 0) 553 if ((s64)delta < 0)
552 delta = 0; 554 delta = 0;
@@ -558,7 +560,7 @@ enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
558 se->sum_sleep_runtime += delta; 560 se->sum_sleep_runtime += delta;
559 } 561 }
560 if (se->block_start) { 562 if (se->block_start) {
561 u64 delta = now - se->block_start; 563 u64 delta = rq_of(cfs_rq)->clock - se->block_start;
562 564
563 if ((s64)delta < 0) 565 if ((s64)delta < 0)
564 delta = 0; 566 delta = 0;
@@ -573,26 +575,24 @@ enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
573} 575}
574 576
575static void 577static void
576enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, 578enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int wakeup)
577 int wakeup, u64 now)
578{ 579{
579 /* 580 /*
580 * Update the fair clock. 581 * Update the fair clock.
581 */ 582 */
582 update_curr(cfs_rq, now); 583 update_curr(cfs_rq);
583 584
584 if (wakeup) 585 if (wakeup)
585 enqueue_sleeper(cfs_rq, se, now); 586 enqueue_sleeper(cfs_rq, se);
586 587
587 update_stats_enqueue(cfs_rq, se, now); 588 update_stats_enqueue(cfs_rq, se);
588 __enqueue_entity(cfs_rq, se); 589 __enqueue_entity(cfs_rq, se);
589} 590}
590 591
591static void 592static void
592dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, 593dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
593 int sleep, u64 now)
594{ 594{
595 update_stats_dequeue(cfs_rq, se, now); 595 update_stats_dequeue(cfs_rq, se);
596 if (sleep) { 596 if (sleep) {
597 se->sleep_start_fair = cfs_rq->fair_clock; 597 se->sleep_start_fair = cfs_rq->fair_clock;
598#ifdef CONFIG_SCHEDSTATS 598#ifdef CONFIG_SCHEDSTATS
@@ -600,9 +600,9 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
600 struct task_struct *tsk = task_of(se); 600 struct task_struct *tsk = task_of(se);
601 601
602 if (tsk->state & TASK_INTERRUPTIBLE) 602 if (tsk->state & TASK_INTERRUPTIBLE)
603 se->sleep_start = now; 603 se->sleep_start = rq_of(cfs_rq)->clock;
604 if (tsk->state & TASK_UNINTERRUPTIBLE) 604 if (tsk->state & TASK_UNINTERRUPTIBLE)
605 se->block_start = now; 605 se->block_start = rq_of(cfs_rq)->clock;
606 } 606 }
607 cfs_rq->wait_runtime -= se->wait_runtime; 607 cfs_rq->wait_runtime -= se->wait_runtime;
608#endif 608#endif
@@ -629,7 +629,7 @@ __check_preempt_curr_fair(struct cfs_rq *cfs_rq, struct sched_entity *se,
629} 629}
630 630
631static inline void 631static inline void
632set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now) 632set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
633{ 633{
634 /* 634 /*
635 * Any task has to be enqueued before it get to execute on 635 * Any task has to be enqueued before it get to execute on
@@ -638,49 +638,46 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
638 * done a put_prev_task_fair() shortly before this, which 638 * done a put_prev_task_fair() shortly before this, which
639 * updated rq->fair_clock - used by update_stats_wait_end()) 639 * updated rq->fair_clock - used by update_stats_wait_end())
640 */ 640 */
641 update_stats_wait_end(cfs_rq, se, now); 641 update_stats_wait_end(cfs_rq, se);
642 update_stats_curr_start(cfs_rq, se, now); 642 update_stats_curr_start(cfs_rq, se);
643 set_cfs_rq_curr(cfs_rq, se); 643 set_cfs_rq_curr(cfs_rq, se);
644} 644}
645 645
646static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq, u64 now) 646static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq)
647{ 647{
648 struct sched_entity *se = __pick_next_entity(cfs_rq); 648 struct sched_entity *se = __pick_next_entity(cfs_rq);
649 649
650 set_next_entity(cfs_rq, se, now); 650 set_next_entity(cfs_rq, se);
651 651
652 return se; 652 return se;
653} 653}
654 654
655static void 655static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
656put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev, u64 now)
657{ 656{
658 /* 657 /*
659 * If still on the runqueue then deactivate_task() 658 * If still on the runqueue then deactivate_task()
660 * was not called and update_curr() has to be done: 659 * was not called and update_curr() has to be done:
661 */ 660 */
662 if (prev->on_rq) 661 if (prev->on_rq)
663 update_curr(cfs_rq, now); 662 update_curr(cfs_rq);
664 663
665 update_stats_curr_end(cfs_rq, prev, now); 664 update_stats_curr_end(cfs_rq, prev);
666 665
667 if (prev->on_rq) 666 if (prev->on_rq)
668 update_stats_wait_start(cfs_rq, prev, now); 667 update_stats_wait_start(cfs_rq, prev);
669 set_cfs_rq_curr(cfs_rq, NULL); 668 set_cfs_rq_curr(cfs_rq, NULL);
670} 669}
671 670
672static void entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) 671static void entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
673{ 672{
674 struct rq *rq = rq_of(cfs_rq);
675 struct sched_entity *next; 673 struct sched_entity *next;
676 u64 now = __rq_clock(rq);
677 674
678 /* 675 /*
679 * Dequeue and enqueue the task to update its 676 * Dequeue and enqueue the task to update its
680 * position within the tree: 677 * position within the tree:
681 */ 678 */
682 dequeue_entity(cfs_rq, curr, 0, now); 679 dequeue_entity(cfs_rq, curr, 0);
683 enqueue_entity(cfs_rq, curr, 0, now); 680 enqueue_entity(cfs_rq, curr, 0);
684 681
685 /* 682 /*
686 * Reschedule if another task tops the current one. 683 * Reschedule if another task tops the current one.
@@ -785,8 +782,7 @@ static inline int is_same_group(struct task_struct *curr, struct task_struct *p)
785 * increased. Here we update the fair scheduling stats and 782 * increased. Here we update the fair scheduling stats and
786 * then put the task into the rbtree: 783 * then put the task into the rbtree:
787 */ 784 */
788static void 785static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup)
789enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup, u64 now)
790{ 786{
791 struct cfs_rq *cfs_rq; 787 struct cfs_rq *cfs_rq;
792 struct sched_entity *se = &p->se; 788 struct sched_entity *se = &p->se;
@@ -795,7 +791,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup, u64 now)
795 if (se->on_rq) 791 if (se->on_rq)
796 break; 792 break;
797 cfs_rq = cfs_rq_of(se); 793 cfs_rq = cfs_rq_of(se);
798 enqueue_entity(cfs_rq, se, wakeup, now); 794 enqueue_entity(cfs_rq, se, wakeup);
799 } 795 }
800} 796}
801 797
@@ -804,15 +800,14 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup, u64 now)
804 * decreased. We remove the task from the rbtree and 800 * decreased. We remove the task from the rbtree and
805 * update the fair scheduling stats: 801 * update the fair scheduling stats:
806 */ 802 */
807static void 803static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int sleep)
808dequeue_task_fair(struct rq *rq, struct task_struct *p, int sleep, u64 now)
809{ 804{
810 struct cfs_rq *cfs_rq; 805 struct cfs_rq *cfs_rq;
811 struct sched_entity *se = &p->se; 806 struct sched_entity *se = &p->se;
812 807
813 for_each_sched_entity(se) { 808 for_each_sched_entity(se) {
814 cfs_rq = cfs_rq_of(se); 809 cfs_rq = cfs_rq_of(se);
815 dequeue_entity(cfs_rq, se, sleep, now); 810 dequeue_entity(cfs_rq, se, sleep);
816 /* Don't dequeue parent if it has other entities besides us */ 811 /* Don't dequeue parent if it has other entities besides us */
817 if (cfs_rq->load.weight) 812 if (cfs_rq->load.weight)
818 break; 813 break;
@@ -825,14 +820,14 @@ dequeue_task_fair(struct rq *rq, struct task_struct *p, int sleep, u64 now)
825static void yield_task_fair(struct rq *rq, struct task_struct *p) 820static void yield_task_fair(struct rq *rq, struct task_struct *p)
826{ 821{
827 struct cfs_rq *cfs_rq = task_cfs_rq(p); 822 struct cfs_rq *cfs_rq = task_cfs_rq(p);
828 u64 now = __rq_clock(rq);
829 823
824 __update_rq_clock(rq);
830 /* 825 /*
831 * Dequeue and enqueue the task to update its 826 * Dequeue and enqueue the task to update its
832 * position within the tree: 827 * position within the tree:
833 */ 828 */
834 dequeue_entity(cfs_rq, &p->se, 0, now); 829 dequeue_entity(cfs_rq, &p->se, 0);
835 enqueue_entity(cfs_rq, &p->se, 0, now); 830 enqueue_entity(cfs_rq, &p->se, 0);
836} 831}
837 832
838/* 833/*
@@ -845,7 +840,8 @@ static void check_preempt_curr_fair(struct rq *rq, struct task_struct *p)
845 unsigned long gran; 840 unsigned long gran;
846 841
847 if (unlikely(rt_prio(p->prio))) { 842 if (unlikely(rt_prio(p->prio))) {
848 update_curr(cfs_rq, rq_clock(rq)); 843 update_rq_clock(rq);
844 update_curr(cfs_rq);
849 resched_task(curr); 845 resched_task(curr);
850 return; 846 return;
851 } 847 }
@@ -861,7 +857,7 @@ static void check_preempt_curr_fair(struct rq *rq, struct task_struct *p)
861 __check_preempt_curr_fair(cfs_rq, &p->se, &curr->se, gran); 857 __check_preempt_curr_fair(cfs_rq, &p->se, &curr->se, gran);
862} 858}
863 859
864static struct task_struct *pick_next_task_fair(struct rq *rq, u64 now) 860static struct task_struct *pick_next_task_fair(struct rq *rq)
865{ 861{
866 struct cfs_rq *cfs_rq = &rq->cfs; 862 struct cfs_rq *cfs_rq = &rq->cfs;
867 struct sched_entity *se; 863 struct sched_entity *se;
@@ -870,7 +866,7 @@ static struct task_struct *pick_next_task_fair(struct rq *rq, u64 now)
870 return NULL; 866 return NULL;
871 867
872 do { 868 do {
873 se = pick_next_entity(cfs_rq, now); 869 se = pick_next_entity(cfs_rq);
874 cfs_rq = group_cfs_rq(se); 870 cfs_rq = group_cfs_rq(se);
875 } while (cfs_rq); 871 } while (cfs_rq);
876 872
@@ -880,14 +876,14 @@ static struct task_struct *pick_next_task_fair(struct rq *rq, u64 now)
880/* 876/*
881 * Account for a descheduled task: 877 * Account for a descheduled task:
882 */ 878 */
883static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, u64 now) 879static void put_prev_task_fair(struct rq *rq, struct task_struct *prev)
884{ 880{
885 struct sched_entity *se = &prev->se; 881 struct sched_entity *se = &prev->se;
886 struct cfs_rq *cfs_rq; 882 struct cfs_rq *cfs_rq;
887 883
888 for_each_sched_entity(se) { 884 for_each_sched_entity(se) {
889 cfs_rq = cfs_rq_of(se); 885 cfs_rq = cfs_rq_of(se);
890 put_prev_entity(cfs_rq, se, now); 886 put_prev_entity(cfs_rq, se);
891 } 887 }
892} 888}
893 889
@@ -930,6 +926,7 @@ static struct task_struct *load_balance_next_fair(void *arg)
930 return __load_balance_iterator(cfs_rq, cfs_rq->rb_load_balance_curr); 926 return __load_balance_iterator(cfs_rq, cfs_rq->rb_load_balance_curr);
931} 927}
932 928
929#ifdef CONFIG_FAIR_GROUP_SCHED
933static int cfs_rq_best_prio(struct cfs_rq *cfs_rq) 930static int cfs_rq_best_prio(struct cfs_rq *cfs_rq)
934{ 931{
935 struct sched_entity *curr; 932 struct sched_entity *curr;
@@ -943,12 +940,13 @@ static int cfs_rq_best_prio(struct cfs_rq *cfs_rq)
943 940
944 return p->prio; 941 return p->prio;
945} 942}
943#endif
946 944
947static int 945static unsigned long
948load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, 946load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
949 unsigned long max_nr_move, unsigned long max_load_move, 947 unsigned long max_nr_move, unsigned long max_load_move,
950 struct sched_domain *sd, enum cpu_idle_type idle, 948 struct sched_domain *sd, enum cpu_idle_type idle,
951 int *all_pinned, unsigned long *total_load_moved) 949 int *all_pinned, int *this_best_prio)
952{ 950{
953 struct cfs_rq *busy_cfs_rq; 951 struct cfs_rq *busy_cfs_rq;
954 unsigned long load_moved, total_nr_moved = 0, nr_moved; 952 unsigned long load_moved, total_nr_moved = 0, nr_moved;
@@ -959,15 +957,14 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
959 cfs_rq_iterator.next = load_balance_next_fair; 957 cfs_rq_iterator.next = load_balance_next_fair;
960 958
961 for_each_leaf_cfs_rq(busiest, busy_cfs_rq) { 959 for_each_leaf_cfs_rq(busiest, busy_cfs_rq) {
960#ifdef CONFIG_FAIR_GROUP_SCHED
962 struct cfs_rq *this_cfs_rq; 961 struct cfs_rq *this_cfs_rq;
963 long imbalance; 962 long imbalance;
964 unsigned long maxload; 963 unsigned long maxload;
965 int this_best_prio, best_prio, best_prio_seen = 0;
966 964
967 this_cfs_rq = cpu_cfs_rq(busy_cfs_rq, this_cpu); 965 this_cfs_rq = cpu_cfs_rq(busy_cfs_rq, this_cpu);
968 966
969 imbalance = busy_cfs_rq->load.weight - 967 imbalance = busy_cfs_rq->load.weight - this_cfs_rq->load.weight;
970 this_cfs_rq->load.weight;
971 /* Don't pull if this_cfs_rq has more load than busy_cfs_rq */ 968 /* Don't pull if this_cfs_rq has more load than busy_cfs_rq */
972 if (imbalance <= 0) 969 if (imbalance <= 0)
973 continue; 970 continue;
@@ -976,27 +973,17 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
976 imbalance /= 2; 973 imbalance /= 2;
977 maxload = min(rem_load_move, imbalance); 974 maxload = min(rem_load_move, imbalance);
978 975
979 this_best_prio = cfs_rq_best_prio(this_cfs_rq); 976 *this_best_prio = cfs_rq_best_prio(this_cfs_rq);
980 best_prio = cfs_rq_best_prio(busy_cfs_rq); 977#else
981 978# define maxload rem_load_move
982 /* 979#endif
983 * Enable handling of the case where there is more than one task
984 * with the best priority. If the current running task is one
985 * of those with prio==best_prio we know it won't be moved
986 * and therefore it's safe to override the skip (based on load)
987 * of any task we find with that prio.
988 */
989 if (cfs_rq_curr(busy_cfs_rq) == &busiest->curr->se)
990 best_prio_seen = 1;
991
992 /* pass busy_cfs_rq argument into 980 /* pass busy_cfs_rq argument into
993 * load_balance_[start|next]_fair iterators 981 * load_balance_[start|next]_fair iterators
994 */ 982 */
995 cfs_rq_iterator.arg = busy_cfs_rq; 983 cfs_rq_iterator.arg = busy_cfs_rq;
996 nr_moved = balance_tasks(this_rq, this_cpu, busiest, 984 nr_moved = balance_tasks(this_rq, this_cpu, busiest,
997 max_nr_move, maxload, sd, idle, all_pinned, 985 max_nr_move, maxload, sd, idle, all_pinned,
998 &load_moved, this_best_prio, best_prio, 986 &load_moved, this_best_prio, &cfs_rq_iterator);
999 best_prio_seen, &cfs_rq_iterator);
1000 987
1001 total_nr_moved += nr_moved; 988 total_nr_moved += nr_moved;
1002 max_nr_move -= nr_moved; 989 max_nr_move -= nr_moved;
@@ -1006,9 +993,7 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
1006 break; 993 break;
1007 } 994 }
1008 995
1009 *total_load_moved = max_load_move - rem_load_move; 996 return max_load_move - rem_load_move;
1010
1011 return total_nr_moved;
1012} 997}
1013 998
1014/* 999/*
@@ -1032,14 +1017,14 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr)
1032 * monopolize the CPU. Note: the parent runqueue is locked, 1017 * monopolize the CPU. Note: the parent runqueue is locked,
1033 * the child is not running yet. 1018 * the child is not running yet.
1034 */ 1019 */
1035static void task_new_fair(struct rq *rq, struct task_struct *p, u64 now) 1020static void task_new_fair(struct rq *rq, struct task_struct *p)
1036{ 1021{
1037 struct cfs_rq *cfs_rq = task_cfs_rq(p); 1022 struct cfs_rq *cfs_rq = task_cfs_rq(p);
1038 struct sched_entity *se = &p->se; 1023 struct sched_entity *se = &p->se;
1039 1024
1040 sched_info_queued(p); 1025 sched_info_queued(p);
1041 1026
1042 update_stats_enqueue(cfs_rq, se, now); 1027 update_stats_enqueue(cfs_rq, se);
1043 /* 1028 /*
1044 * Child runs first: we let it run before the parent 1029 * Child runs first: we let it run before the parent
1045 * until it reschedules once. We set up the key so that 1030 * until it reschedules once. We set up the key so that
@@ -1072,15 +1057,10 @@ static void task_new_fair(struct rq *rq, struct task_struct *p, u64 now)
1072 */ 1057 */
1073static void set_curr_task_fair(struct rq *rq) 1058static void set_curr_task_fair(struct rq *rq)
1074{ 1059{
1075 struct task_struct *curr = rq->curr; 1060 struct sched_entity *se = &rq->curr.se;
1076 struct sched_entity *se = &curr->se;
1077 u64 now = rq_clock(rq);
1078 struct cfs_rq *cfs_rq;
1079 1061
1080 for_each_sched_entity(se) { 1062 for_each_sched_entity(se)
1081 cfs_rq = cfs_rq_of(se); 1063 set_next_entity(cfs_rq_of(se), se);
1082 set_next_entity(cfs_rq, se, now);
1083 }
1084} 1064}
1085#else 1065#else
1086static void set_curr_task_fair(struct rq *rq) 1066static void set_curr_task_fair(struct rq *rq)
@@ -1109,12 +1089,11 @@ struct sched_class fair_sched_class __read_mostly = {
1109}; 1089};
1110 1090
1111#ifdef CONFIG_SCHED_DEBUG 1091#ifdef CONFIG_SCHED_DEBUG
1112void print_cfs_stats(struct seq_file *m, int cpu, u64 now) 1092static void print_cfs_stats(struct seq_file *m, int cpu)
1113{ 1093{
1114 struct rq *rq = cpu_rq(cpu);
1115 struct cfs_rq *cfs_rq; 1094 struct cfs_rq *cfs_rq;
1116 1095
1117 for_each_leaf_cfs_rq(rq, cfs_rq) 1096 for_each_leaf_cfs_rq(cpu_rq(cpu), cfs_rq)
1118 print_cfs_rq(m, cpu, cfs_rq, now); 1097 print_cfs_rq(m, cpu, cfs_rq);
1119} 1098}
1120#endif 1099#endif
diff --git a/kernel/sched_idletask.c b/kernel/sched_idletask.c
index 41841e741c..3503fb2d9f 100644
--- a/kernel/sched_idletask.c
+++ b/kernel/sched_idletask.c
@@ -13,7 +13,7 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p)
13 resched_task(rq->idle); 13 resched_task(rq->idle);
14} 14}
15 15
16static struct task_struct *pick_next_task_idle(struct rq *rq, u64 now) 16static struct task_struct *pick_next_task_idle(struct rq *rq)
17{ 17{
18 schedstat_inc(rq, sched_goidle); 18 schedstat_inc(rq, sched_goidle);
19 19
@@ -25,7 +25,7 @@ static struct task_struct *pick_next_task_idle(struct rq *rq, u64 now)
25 * message if some code attempts to do it: 25 * message if some code attempts to do it:
26 */ 26 */
27static void 27static void
28dequeue_task_idle(struct rq *rq, struct task_struct *p, int sleep, u64 now) 28dequeue_task_idle(struct rq *rq, struct task_struct *p, int sleep)
29{ 29{
30 spin_unlock_irq(&rq->lock); 30 spin_unlock_irq(&rq->lock);
31 printk(KERN_ERR "bad: scheduling from the idle thread!\n"); 31 printk(KERN_ERR "bad: scheduling from the idle thread!\n");
@@ -33,15 +33,15 @@ dequeue_task_idle(struct rq *rq, struct task_struct *p, int sleep, u64 now)
33 spin_lock_irq(&rq->lock); 33 spin_lock_irq(&rq->lock);
34} 34}
35 35
36static void put_prev_task_idle(struct rq *rq, struct task_struct *prev, u64 now) 36static void put_prev_task_idle(struct rq *rq, struct task_struct *prev)
37{ 37{
38} 38}
39 39
40static int 40static unsigned long
41load_balance_idle(struct rq *this_rq, int this_cpu, struct rq *busiest, 41load_balance_idle(struct rq *this_rq, int this_cpu, struct rq *busiest,
42 unsigned long max_nr_move, unsigned long max_load_move, 42 unsigned long max_nr_move, unsigned long max_load_move,
43 struct sched_domain *sd, enum cpu_idle_type idle, 43 struct sched_domain *sd, enum cpu_idle_type idle,
44 int *all_pinned, unsigned long *total_load_moved) 44 int *all_pinned, int *this_best_prio)
45{ 45{
46 return 0; 46 return 0;
47} 47}
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index 002fcf8d3f..dcdcad632f 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -7,7 +7,7 @@
7 * Update the current task's runtime statistics. Skip current tasks that 7 * Update the current task's runtime statistics. Skip current tasks that
8 * are not in our scheduling class. 8 * are not in our scheduling class.
9 */ 9 */
10static inline void update_curr_rt(struct rq *rq, u64 now) 10static inline void update_curr_rt(struct rq *rq)
11{ 11{
12 struct task_struct *curr = rq->curr; 12 struct task_struct *curr = rq->curr;
13 u64 delta_exec; 13 u64 delta_exec;
@@ -15,18 +15,17 @@ static inline void update_curr_rt(struct rq *rq, u64 now)
15 if (!task_has_rt_policy(curr)) 15 if (!task_has_rt_policy(curr))
16 return; 16 return;
17 17
18 delta_exec = now - curr->se.exec_start; 18 delta_exec = rq->clock - curr->se.exec_start;
19 if (unlikely((s64)delta_exec < 0)) 19 if (unlikely((s64)delta_exec < 0))
20 delta_exec = 0; 20 delta_exec = 0;
21 21
22 schedstat_set(curr->se.exec_max, max(curr->se.exec_max, delta_exec)); 22 schedstat_set(curr->se.exec_max, max(curr->se.exec_max, delta_exec));
23 23
24 curr->se.sum_exec_runtime += delta_exec; 24 curr->se.sum_exec_runtime += delta_exec;
25 curr->se.exec_start = now; 25 curr->se.exec_start = rq->clock;
26} 26}
27 27
28static void 28static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup)
29enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup, u64 now)
30{ 29{
31 struct rt_prio_array *array = &rq->rt.active; 30 struct rt_prio_array *array = &rq->rt.active;
32 31
@@ -37,12 +36,11 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup, u64 now)
37/* 36/*
38 * Adding/removing a task to/from a priority array: 37 * Adding/removing a task to/from a priority array:
39 */ 38 */
40static void 39static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep)
41dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep, u64 now)
42{ 40{
43 struct rt_prio_array *array = &rq->rt.active; 41 struct rt_prio_array *array = &rq->rt.active;
44 42
45 update_curr_rt(rq, now); 43 update_curr_rt(rq);
46 44
47 list_del(&p->run_list); 45 list_del(&p->run_list);
48 if (list_empty(array->queue + p->prio)) 46 if (list_empty(array->queue + p->prio))
@@ -75,7 +73,7 @@ static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p)
75 resched_task(rq->curr); 73 resched_task(rq->curr);
76} 74}
77 75
78static struct task_struct *pick_next_task_rt(struct rq *rq, u64 now) 76static struct task_struct *pick_next_task_rt(struct rq *rq)
79{ 77{
80 struct rt_prio_array *array = &rq->rt.active; 78 struct rt_prio_array *array = &rq->rt.active;
81 struct task_struct *next; 79 struct task_struct *next;
@@ -89,14 +87,14 @@ static struct task_struct *pick_next_task_rt(struct rq *rq, u64 now)
89 queue = array->queue + idx; 87 queue = array->queue + idx;
90 next = list_entry(queue->next, struct task_struct, run_list); 88 next = list_entry(queue->next, struct task_struct, run_list);
91 89
92 next->se.exec_start = now; 90 next->se.exec_start = rq->clock;
93 91
94 return next; 92 return next;
95} 93}
96 94
97static void put_prev_task_rt(struct rq *rq, struct task_struct *p, u64 now) 95static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
98{ 96{
99 update_curr_rt(rq, now); 97 update_curr_rt(rq);
100 p->se.exec_start = 0; 98 p->se.exec_start = 0;
101} 99}
102 100
@@ -172,28 +170,15 @@ static struct task_struct *load_balance_next_rt(void *arg)
172 return p; 170 return p;
173} 171}
174 172
175static int 173static unsigned long
176load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, 174load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest,
177 unsigned long max_nr_move, unsigned long max_load_move, 175 unsigned long max_nr_move, unsigned long max_load_move,
178 struct sched_domain *sd, enum cpu_idle_type idle, 176 struct sched_domain *sd, enum cpu_idle_type idle,
179 int *all_pinned, unsigned long *load_moved) 177 int *all_pinned, int *this_best_prio)
180{ 178{
181 int this_best_prio, best_prio, best_prio_seen = 0;
182 int nr_moved; 179 int nr_moved;
183 struct rq_iterator rt_rq_iterator; 180 struct rq_iterator rt_rq_iterator;
184 181 unsigned long load_moved;
185 best_prio = sched_find_first_bit(busiest->rt.active.bitmap);
186 this_best_prio = sched_find_first_bit(this_rq->rt.active.bitmap);
187
188 /*
189 * Enable handling of the case where there is more than one task
190 * with the best priority. If the current running task is one
191 * of those with prio==best_prio we know it won't be moved
192 * and therefore it's safe to override the skip (based on load)
193 * of any task we find with that prio.
194 */
195 if (busiest->curr->prio == best_prio)
196 best_prio_seen = 1;
197 182
198 rt_rq_iterator.start = load_balance_start_rt; 183 rt_rq_iterator.start = load_balance_start_rt;
199 rt_rq_iterator.next = load_balance_next_rt; 184 rt_rq_iterator.next = load_balance_next_rt;
@@ -203,11 +188,10 @@ load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest,
203 rt_rq_iterator.arg = busiest; 188 rt_rq_iterator.arg = busiest;
204 189
205 nr_moved = balance_tasks(this_rq, this_cpu, busiest, max_nr_move, 190 nr_moved = balance_tasks(this_rq, this_cpu, busiest, max_nr_move,
206 max_load_move, sd, idle, all_pinned, load_moved, 191 max_load_move, sd, idle, all_pinned, &load_moved,
207 this_best_prio, best_prio, best_prio_seen, 192 this_best_prio, &rt_rq_iterator);
208 &rt_rq_iterator);
209 193
210 return nr_moved; 194 return load_moved;
211} 195}
212 196
213static void task_tick_rt(struct rq *rq, struct task_struct *p) 197static void task_tick_rt(struct rq *rq, struct task_struct *p)
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 79c891e626..8bdb8c07e0 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -1023,6 +1023,7 @@ static ctl_table vm_table[] = {
1023 .mode = 0644, 1023 .mode = 0644,
1024 .proc_handler = &proc_doulongvec_minmax, 1024 .proc_handler = &proc_doulongvec_minmax,
1025 }, 1025 },
1026#endif
1026#ifdef CONFIG_NUMA 1027#ifdef CONFIG_NUMA
1027 { 1028 {
1028 .ctl_name = CTL_UNNUMBERED, 1029 .ctl_name = CTL_UNNUMBERED,
@@ -1034,7 +1035,6 @@ static ctl_table vm_table[] = {
1034 .strategy = &sysctl_string, 1035 .strategy = &sysctl_string,
1035 }, 1036 },
1036#endif 1037#endif
1037#endif
1038#if defined(CONFIG_X86_32) || \ 1038#if defined(CONFIG_X86_32) || \
1039 (defined(CONFIG_SUPERH) && defined(CONFIG_VSYSCALL)) 1039 (defined(CONFIG_SUPERH) && defined(CONFIG_VSYSCALL))
1040 { 1040 {
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index 2ad1c37b8d..41dd3105ce 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -113,16 +113,6 @@ int clockevents_register_notifier(struct notifier_block *nb)
113 return ret; 113 return ret;
114} 114}
115 115
116/**
117 * clockevents_unregister_notifier - unregister a clock events change listener
118 */
119void clockevents_unregister_notifier(struct notifier_block *nb)
120{
121 spin_lock(&clockevents_lock);
122 raw_notifier_chain_unregister(&clockevents_chain, nb);
123 spin_unlock(&clockevents_lock);
124}
125
126/* 116/*
127 * Notify about a clock event change. Called with clockevents_lock 117 * Notify about a clock event change. Called with clockevents_lock
128 * held. 118 * held.
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-05 11:02:00 -0400