diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2011-07-24 12:07:03 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2011-07-24 12:07:03 -0400 |
commit | 5fabc487c96819dd12ddb9414835d170fd9cd6d5 (patch) | |
tree | 01532d492e5074b0d3add29bf92ebf9a9d161e9e /kernel/sched.c | |
parent | c61264f98c1a974ee6f545f61a4ab33b141d6bda (diff) | |
parent | 3f68b0318bbbd61bf08478ab99a149f0d9e5156e (diff) |
Merge branch 'kvm-updates/3.1' of git://git.kernel.org/pub/scm/virt/kvm/kvm
* 'kvm-updates/3.1' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (143 commits)
KVM: IOMMU: Disable device assignment without interrupt remapping
KVM: MMU: trace mmio page fault
KVM: MMU: mmio page fault support
KVM: MMU: reorganize struct kvm_shadow_walk_iterator
KVM: MMU: lockless walking shadow page table
KVM: MMU: do not need atomicly to set/clear spte
KVM: MMU: introduce the rules to modify shadow page table
KVM: MMU: abstract some functions to handle fault pfn
KVM: MMU: filter out the mmio pfn from the fault pfn
KVM: MMU: remove bypass_guest_pf
KVM: MMU: split kvm_mmu_free_page
KVM: MMU: count used shadow pages on prepareing path
KVM: MMU: rename 'pt_write' to 'emulate'
KVM: MMU: cleanup for FNAME(fetch)
KVM: MMU: optimize to handle dirty bit
KVM: MMU: cache mmio info on page fault path
KVM: x86: introduce vcpu_mmio_gva_to_gpa to cleanup the code
KVM: MMU: do not update slot bitmap if spte is nonpresent
KVM: MMU: fix walking shadow page table
KVM guest: KVM Steal time registration
...
Diffstat (limited to 'kernel/sched.c')
-rw-r--r-- | kernel/sched.c | 90 |
1 files changed, 80 insertions, 10 deletions
diff --git a/kernel/sched.c b/kernel/sched.c index 9aaf567c5da5..751a7cc6a5cd 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
@@ -75,6 +75,9 @@ | |||
75 | #include <asm/tlb.h> | 75 | #include <asm/tlb.h> |
76 | #include <asm/irq_regs.h> | 76 | #include <asm/irq_regs.h> |
77 | #include <asm/mutex.h> | 77 | #include <asm/mutex.h> |
78 | #ifdef CONFIG_PARAVIRT | ||
79 | #include <asm/paravirt.h> | ||
80 | #endif | ||
78 | 81 | ||
79 | #include "sched_cpupri.h" | 82 | #include "sched_cpupri.h" |
80 | #include "workqueue_sched.h" | 83 | #include "workqueue_sched.h" |
@@ -528,6 +531,12 @@ struct rq { | |||
528 | #ifdef CONFIG_IRQ_TIME_ACCOUNTING | 531 | #ifdef CONFIG_IRQ_TIME_ACCOUNTING |
529 | u64 prev_irq_time; | 532 | u64 prev_irq_time; |
530 | #endif | 533 | #endif |
534 | #ifdef CONFIG_PARAVIRT | ||
535 | u64 prev_steal_time; | ||
536 | #endif | ||
537 | #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING | ||
538 | u64 prev_steal_time_rq; | ||
539 | #endif | ||
531 | 540 | ||
532 | /* calc_load related fields */ | 541 | /* calc_load related fields */ |
533 | unsigned long calc_load_update; | 542 | unsigned long calc_load_update; |
@@ -1921,10 +1930,28 @@ void account_system_vtime(struct task_struct *curr) | |||
1921 | } | 1930 | } |
1922 | EXPORT_SYMBOL_GPL(account_system_vtime); | 1931 | EXPORT_SYMBOL_GPL(account_system_vtime); |
1923 | 1932 | ||
1924 | static void update_rq_clock_task(struct rq *rq, s64 delta) | 1933 | #endif /* CONFIG_IRQ_TIME_ACCOUNTING */ |
1934 | |||
1935 | #ifdef CONFIG_PARAVIRT | ||
1936 | static inline u64 steal_ticks(u64 steal) | ||
1925 | { | 1937 | { |
1926 | s64 irq_delta; | 1938 | if (unlikely(steal > NSEC_PER_SEC)) |
1939 | return div_u64(steal, TICK_NSEC); | ||
1927 | 1940 | ||
1941 | return __iter_div_u64_rem(steal, TICK_NSEC, &steal); | ||
1942 | } | ||
1943 | #endif | ||
1944 | |||
1945 | static void update_rq_clock_task(struct rq *rq, s64 delta) | ||
1946 | { | ||
1947 | /* | ||
1948 | * In theory, the compile should just see 0 here, and optimize out the call | ||
1949 | * to sched_rt_avg_update. But I don't trust it... | ||
1950 | */ | ||
1951 | #if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING) | ||
1952 | s64 steal = 0, irq_delta = 0; | ||
1953 | #endif | ||
1954 | #ifdef CONFIG_IRQ_TIME_ACCOUNTING | ||
1928 | irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; | 1955 | irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; |
1929 | 1956 | ||
1930 | /* | 1957 | /* |
@@ -1947,12 +1974,35 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) | |||
1947 | 1974 | ||
1948 | rq->prev_irq_time += irq_delta; | 1975 | rq->prev_irq_time += irq_delta; |
1949 | delta -= irq_delta; | 1976 | delta -= irq_delta; |
1977 | #endif | ||
1978 | #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING | ||
1979 | if (static_branch((¶virt_steal_rq_enabled))) { | ||
1980 | u64 st; | ||
1981 | |||
1982 | steal = paravirt_steal_clock(cpu_of(rq)); | ||
1983 | steal -= rq->prev_steal_time_rq; | ||
1984 | |||
1985 | if (unlikely(steal > delta)) | ||
1986 | steal = delta; | ||
1987 | |||
1988 | st = steal_ticks(steal); | ||
1989 | steal = st * TICK_NSEC; | ||
1990 | |||
1991 | rq->prev_steal_time_rq += steal; | ||
1992 | |||
1993 | delta -= steal; | ||
1994 | } | ||
1995 | #endif | ||
1996 | |||
1950 | rq->clock_task += delta; | 1997 | rq->clock_task += delta; |
1951 | 1998 | ||
1952 | if (irq_delta && sched_feat(NONIRQ_POWER)) | 1999 | #if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING) |
1953 | sched_rt_avg_update(rq, irq_delta); | 2000 | if ((irq_delta + steal) && sched_feat(NONTASK_POWER)) |
2001 | sched_rt_avg_update(rq, irq_delta + steal); | ||
2002 | #endif | ||
1954 | } | 2003 | } |
1955 | 2004 | ||
2005 | #ifdef CONFIG_IRQ_TIME_ACCOUNTING | ||
1956 | static int irqtime_account_hi_update(void) | 2006 | static int irqtime_account_hi_update(void) |
1957 | { | 2007 | { |
1958 | struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; | 2008 | struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; |
@@ -1987,12 +2037,7 @@ static int irqtime_account_si_update(void) | |||
1987 | 2037 | ||
1988 | #define sched_clock_irqtime (0) | 2038 | #define sched_clock_irqtime (0) |
1989 | 2039 | ||
1990 | static void update_rq_clock_task(struct rq *rq, s64 delta) | 2040 | #endif |
1991 | { | ||
1992 | rq->clock_task += delta; | ||
1993 | } | ||
1994 | |||
1995 | #endif /* CONFIG_IRQ_TIME_ACCOUNTING */ | ||
1996 | 2041 | ||
1997 | #include "sched_idletask.c" | 2042 | #include "sched_idletask.c" |
1998 | #include "sched_fair.c" | 2043 | #include "sched_fair.c" |
@@ -3845,6 +3890,25 @@ void account_idle_time(cputime_t cputime) | |||
3845 | cpustat->idle = cputime64_add(cpustat->idle, cputime64); | 3890 | cpustat->idle = cputime64_add(cpustat->idle, cputime64); |
3846 | } | 3891 | } |
3847 | 3892 | ||
3893 | static __always_inline bool steal_account_process_tick(void) | ||
3894 | { | ||
3895 | #ifdef CONFIG_PARAVIRT | ||
3896 | if (static_branch(¶virt_steal_enabled)) { | ||
3897 | u64 steal, st = 0; | ||
3898 | |||
3899 | steal = paravirt_steal_clock(smp_processor_id()); | ||
3900 | steal -= this_rq()->prev_steal_time; | ||
3901 | |||
3902 | st = steal_ticks(steal); | ||
3903 | this_rq()->prev_steal_time += st * TICK_NSEC; | ||
3904 | |||
3905 | account_steal_time(st); | ||
3906 | return st; | ||
3907 | } | ||
3908 | #endif | ||
3909 | return false; | ||
3910 | } | ||
3911 | |||
3848 | #ifndef CONFIG_VIRT_CPU_ACCOUNTING | 3912 | #ifndef CONFIG_VIRT_CPU_ACCOUNTING |
3849 | 3913 | ||
3850 | #ifdef CONFIG_IRQ_TIME_ACCOUNTING | 3914 | #ifdef CONFIG_IRQ_TIME_ACCOUNTING |
@@ -3876,6 +3940,9 @@ static void irqtime_account_process_tick(struct task_struct *p, int user_tick, | |||
3876 | cputime64_t tmp = cputime_to_cputime64(cputime_one_jiffy); | 3940 | cputime64_t tmp = cputime_to_cputime64(cputime_one_jiffy); |
3877 | struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; | 3941 | struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; |
3878 | 3942 | ||
3943 | if (steal_account_process_tick()) | ||
3944 | return; | ||
3945 | |||
3879 | if (irqtime_account_hi_update()) { | 3946 | if (irqtime_account_hi_update()) { |
3880 | cpustat->irq = cputime64_add(cpustat->irq, tmp); | 3947 | cpustat->irq = cputime64_add(cpustat->irq, tmp); |
3881 | } else if (irqtime_account_si_update()) { | 3948 | } else if (irqtime_account_si_update()) { |
@@ -3929,6 +3996,9 @@ void account_process_tick(struct task_struct *p, int user_tick) | |||
3929 | return; | 3996 | return; |
3930 | } | 3997 | } |
3931 | 3998 | ||
3999 | if (steal_account_process_tick()) | ||
4000 | return; | ||
4001 | |||
3932 | if (user_tick) | 4002 | if (user_tick) |
3933 | account_user_time(p, cputime_one_jiffy, one_jiffy_scaled); | 4003 | account_user_time(p, cputime_one_jiffy, one_jiffy_scaled); |
3934 | else if ((p != rq->idle) || (irq_count() != HARDIRQ_OFFSET)) | 4004 | else if ((p != rq->idle) || (irq_count() != HARDIRQ_OFFSET)) |