diff options
| author | Arjan van de Ven <arjan@linux.intel.com> | 2008-10-17 12:20:26 -0400 |
|---|---|---|
| committer | Arjan van de Ven <arjan@linux.intel.com> | 2008-10-17 12:20:26 -0400 |
| commit | 651dab4264e4ba0e563f5ff56f748127246e9065 (patch) | |
| tree | 016630974bdcb00fe529b673f96d389e0fd6dc94 /kernel | |
| parent | 40b8606253552109815786e5d4b0de98782d31f5 (diff) | |
| parent | 2e532d68a2b3e2aa6b19731501222069735c741c (diff) | |
Merge commit 'linus/master' into merge-linus
Conflicts:
arch/x86/kvm/i8254.c
Diffstat (limited to 'kernel')
56 files changed, 1676 insertions, 900 deletions
diff --git a/kernel/acct.c b/kernel/acct.c index dd68b9059418..f6006a60df5d 100644 --- a/kernel/acct.c +++ b/kernel/acct.c | |||
| @@ -548,7 +548,7 @@ static void do_acct_process(struct bsd_acct_struct *acct, | |||
| 548 | #endif | 548 | #endif |
| 549 | 549 | ||
| 550 | spin_lock_irq(¤t->sighand->siglock); | 550 | spin_lock_irq(¤t->sighand->siglock); |
| 551 | tty = current->signal->tty; | 551 | tty = current->signal->tty; /* Safe as we hold the siglock */ |
| 552 | ac.ac_tty = tty ? old_encode_dev(tty_devnum(tty)) : 0; | 552 | ac.ac_tty = tty ? old_encode_dev(tty_devnum(tty)) : 0; |
| 553 | ac.ac_utime = encode_comp_t(jiffies_to_AHZ(cputime_to_jiffies(pacct->ac_utime))); | 553 | ac.ac_utime = encode_comp_t(jiffies_to_AHZ(cputime_to_jiffies(pacct->ac_utime))); |
| 554 | ac.ac_stime = encode_comp_t(jiffies_to_AHZ(cputime_to_jiffies(pacct->ac_stime))); | 554 | ac.ac_stime = encode_comp_t(jiffies_to_AHZ(cputime_to_jiffies(pacct->ac_stime))); |
diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 59cedfb040e7..cf5bc2f5f9c3 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c | |||
| @@ -246,8 +246,8 @@ static int audit_match_perm(struct audit_context *ctx, int mask) | |||
| 246 | unsigned n; | 246 | unsigned n; |
| 247 | if (unlikely(!ctx)) | 247 | if (unlikely(!ctx)) |
| 248 | return 0; | 248 | return 0; |
| 249 | |||
| 250 | n = ctx->major; | 249 | n = ctx->major; |
| 250 | |||
| 251 | switch (audit_classify_syscall(ctx->arch, n)) { | 251 | switch (audit_classify_syscall(ctx->arch, n)) { |
| 252 | case 0: /* native */ | 252 | case 0: /* native */ |
| 253 | if ((mask & AUDIT_PERM_WRITE) && | 253 | if ((mask & AUDIT_PERM_WRITE) && |
| @@ -1204,13 +1204,13 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts | |||
| 1204 | (context->return_valid==AUDITSC_SUCCESS)?"yes":"no", | 1204 | (context->return_valid==AUDITSC_SUCCESS)?"yes":"no", |
| 1205 | context->return_code); | 1205 | context->return_code); |
| 1206 | 1206 | ||
| 1207 | mutex_lock(&tty_mutex); | 1207 | spin_lock_irq(&tsk->sighand->siglock); |
| 1208 | read_lock(&tasklist_lock); | ||
| 1209 | if (tsk->signal && tsk->signal->tty && tsk->signal->tty->name) | 1208 | if (tsk->signal && tsk->signal->tty && tsk->signal->tty->name) |
| 1210 | tty = tsk->signal->tty->name; | 1209 | tty = tsk->signal->tty->name; |
| 1211 | else | 1210 | else |
| 1212 | tty = "(none)"; | 1211 | tty = "(none)"; |
| 1213 | read_unlock(&tasklist_lock); | 1212 | spin_unlock_irq(&tsk->sighand->siglock); |
| 1213 | |||
| 1214 | audit_log_format(ab, | 1214 | audit_log_format(ab, |
| 1215 | " a0=%lx a1=%lx a2=%lx a3=%lx items=%d" | 1215 | " a0=%lx a1=%lx a2=%lx a3=%lx items=%d" |
| 1216 | " ppid=%d pid=%d auid=%u uid=%u gid=%u" | 1216 | " ppid=%d pid=%d auid=%u uid=%u gid=%u" |
| @@ -1230,7 +1230,6 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts | |||
| 1230 | context->egid, context->sgid, context->fsgid, tty, | 1230 | context->egid, context->sgid, context->fsgid, tty, |
| 1231 | tsk->sessionid); | 1231 | tsk->sessionid); |
| 1232 | 1232 | ||
| 1233 | mutex_unlock(&tty_mutex); | ||
| 1234 | 1233 | ||
| 1235 | audit_log_task_info(ab, tsk); | 1234 | audit_log_task_info(ab, tsk); |
| 1236 | if (context->filterkey) { | 1235 | if (context->filterkey) { |
diff --git a/kernel/cgroup.c b/kernel/cgroup.c index 13932abde159..8c6e1c17e6d3 100644 --- a/kernel/cgroup.c +++ b/kernel/cgroup.c | |||
| @@ -2735,21 +2735,24 @@ void cgroup_fork_callbacks(struct task_struct *child) | |||
| 2735 | * Called on every change to mm->owner. mm_init_owner() does not | 2735 | * Called on every change to mm->owner. mm_init_owner() does not |
| 2736 | * invoke this routine, since it assigns the mm->owner the first time | 2736 | * invoke this routine, since it assigns the mm->owner the first time |
| 2737 | * and does not change it. | 2737 | * and does not change it. |
| 2738 | * | ||
| 2739 | * The callbacks are invoked with mmap_sem held in read mode. | ||
| 2738 | */ | 2740 | */ |
| 2739 | void cgroup_mm_owner_callbacks(struct task_struct *old, struct task_struct *new) | 2741 | void cgroup_mm_owner_callbacks(struct task_struct *old, struct task_struct *new) |
| 2740 | { | 2742 | { |
| 2741 | struct cgroup *oldcgrp, *newcgrp; | 2743 | struct cgroup *oldcgrp, *newcgrp = NULL; |
| 2742 | 2744 | ||
| 2743 | if (need_mm_owner_callback) { | 2745 | if (need_mm_owner_callback) { |
| 2744 | int i; | 2746 | int i; |
| 2745 | for (i = 0; i < CGROUP_SUBSYS_COUNT; i++) { | 2747 | for (i = 0; i < CGROUP_SUBSYS_COUNT; i++) { |
| 2746 | struct cgroup_subsys *ss = subsys[i]; | 2748 | struct cgroup_subsys *ss = subsys[i]; |
| 2747 | oldcgrp = task_cgroup(old, ss->subsys_id); | 2749 | oldcgrp = task_cgroup(old, ss->subsys_id); |
| 2748 | newcgrp = task_cgroup(new, ss->subsys_id); | 2750 | if (new) |
| 2751 | newcgrp = task_cgroup(new, ss->subsys_id); | ||
| 2749 | if (oldcgrp == newcgrp) | 2752 | if (oldcgrp == newcgrp) |
| 2750 | continue; | 2753 | continue; |
| 2751 | if (ss->mm_owner_changed) | 2754 | if (ss->mm_owner_changed) |
| 2752 | ss->mm_owner_changed(ss, oldcgrp, newcgrp); | 2755 | ss->mm_owner_changed(ss, oldcgrp, newcgrp, new); |
| 2753 | } | 2756 | } |
| 2754 | } | 2757 | } |
| 2755 | } | 2758 | } |
diff --git a/kernel/compat.c b/kernel/compat.c index 32c254a8ab9a..143990e48cb9 100644 --- a/kernel/compat.c +++ b/kernel/compat.c | |||
| @@ -26,6 +26,64 @@ | |||
| 26 | 26 | ||
| 27 | #include <asm/uaccess.h> | 27 | #include <asm/uaccess.h> |
| 28 | 28 | ||
| 29 | /* | ||
| 30 | * Note that the native side is already converted to a timespec, because | ||
| 31 | * that's what we want anyway. | ||
| 32 | */ | ||
| 33 | static int compat_get_timeval(struct timespec *o, | ||
| 34 | struct compat_timeval __user *i) | ||
| 35 | { | ||
| 36 | long usec; | ||
| 37 | |||
| 38 | if (get_user(o->tv_sec, &i->tv_sec) || | ||
| 39 | get_user(usec, &i->tv_usec)) | ||
| 40 | return -EFAULT; | ||
| 41 | o->tv_nsec = usec * 1000; | ||
| 42 | return 0; | ||
| 43 | } | ||
| 44 | |||
| 45 | static int compat_put_timeval(struct compat_timeval __user *o, | ||
| 46 | struct timeval *i) | ||
| 47 | { | ||
| 48 | return (put_user(i->tv_sec, &o->tv_sec) || | ||
| 49 | put_user(i->tv_usec, &o->tv_usec)) ? -EFAULT : 0; | ||
| 50 | } | ||
| 51 | |||
| 52 | asmlinkage long compat_sys_gettimeofday(struct compat_timeval __user *tv, | ||
| 53 | struct timezone __user *tz) | ||
| 54 | { | ||
| 55 | if (tv) { | ||
| 56 | struct timeval ktv; | ||
| 57 | do_gettimeofday(&ktv); | ||
| 58 | if (compat_put_timeval(tv, &ktv)) | ||
| 59 | return -EFAULT; | ||
| 60 | } | ||
| 61 | if (tz) { | ||
| 62 | if (copy_to_user(tz, &sys_tz, sizeof(sys_tz))) | ||
| 63 | return -EFAULT; | ||
| 64 | } | ||
| 65 | |||
| 66 | return 0; | ||
| 67 | } | ||
| 68 | |||
| 69 | asmlinkage long compat_sys_settimeofday(struct compat_timeval __user *tv, | ||
| 70 | struct timezone __user *tz) | ||
| 71 | { | ||
| 72 | struct timespec kts; | ||
| 73 | struct timezone ktz; | ||
| 74 | |||
| 75 | if (tv) { | ||
| 76 | if (compat_get_timeval(&kts, tv)) | ||
| 77 | return -EFAULT; | ||
| 78 | } | ||
| 79 | if (tz) { | ||
| 80 | if (copy_from_user(&ktz, tz, sizeof(ktz))) | ||
| 81 | return -EFAULT; | ||
| 82 | } | ||
| 83 | |||
| 84 | return do_sys_settimeofday(tv ? &kts : NULL, tz ? &ktz : NULL); | ||
| 85 | } | ||
| 86 | |||
| 29 | int get_compat_timespec(struct timespec *ts, const struct compat_timespec __user *cts) | 87 | int get_compat_timespec(struct timespec *ts, const struct compat_timespec __user *cts) |
| 30 | { | 88 | { |
| 31 | return (!access_ok(VERIFY_READ, cts, sizeof(*cts)) || | 89 | return (!access_ok(VERIFY_READ, cts, sizeof(*cts)) || |
diff --git a/kernel/cpu.c b/kernel/cpu.c index f17e9854c246..86d49045daed 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c | |||
| @@ -199,13 +199,14 @@ static int __ref take_cpu_down(void *_param) | |||
| 199 | struct take_cpu_down_param *param = _param; | 199 | struct take_cpu_down_param *param = _param; |
| 200 | int err; | 200 | int err; |
| 201 | 201 | ||
| 202 | raw_notifier_call_chain(&cpu_chain, CPU_DYING | param->mod, | ||
| 203 | param->hcpu); | ||
| 204 | /* Ensure this CPU doesn't handle any more interrupts. */ | 202 | /* Ensure this CPU doesn't handle any more interrupts. */ |
| 205 | err = __cpu_disable(); | 203 | err = __cpu_disable(); |
| 206 | if (err < 0) | 204 | if (err < 0) |
| 207 | return err; | 205 | return err; |
| 208 | 206 | ||
| 207 | raw_notifier_call_chain(&cpu_chain, CPU_DYING | param->mod, | ||
| 208 | param->hcpu); | ||
| 209 | |||
| 209 | /* Force idle task to run as soon as we yield: it should | 210 | /* Force idle task to run as soon as we yield: it should |
| 210 | immediately notice cpu is offline and die quickly. */ | 211 | immediately notice cpu is offline and die quickly. */ |
| 211 | sched_idle_next(); | 212 | sched_idle_next(); |
| @@ -453,6 +454,25 @@ out: | |||
| 453 | } | 454 | } |
| 454 | #endif /* CONFIG_PM_SLEEP_SMP */ | 455 | #endif /* CONFIG_PM_SLEEP_SMP */ |
| 455 | 456 | ||
| 457 | /** | ||
| 458 | * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers | ||
| 459 | * @cpu: cpu that just started | ||
| 460 | * | ||
| 461 | * This function calls the cpu_chain notifiers with CPU_STARTING. | ||
| 462 | * It must be called by the arch code on the new cpu, before the new cpu | ||
| 463 | * enables interrupts and before the "boot" cpu returns from __cpu_up(). | ||
| 464 | */ | ||
| 465 | void notify_cpu_starting(unsigned int cpu) | ||
| 466 | { | ||
| 467 | unsigned long val = CPU_STARTING; | ||
| 468 | |||
| 469 | #ifdef CONFIG_PM_SLEEP_SMP | ||
| 470 | if (cpu_isset(cpu, frozen_cpus)) | ||
| 471 | val = CPU_STARTING_FROZEN; | ||
| 472 | #endif /* CONFIG_PM_SLEEP_SMP */ | ||
| 473 | raw_notifier_call_chain(&cpu_chain, val, (void *)(long)cpu); | ||
| 474 | } | ||
| 475 | |||
| 456 | #endif /* CONFIG_SMP */ | 476 | #endif /* CONFIG_SMP */ |
| 457 | 477 | ||
| 458 | /* | 478 | /* |
diff --git a/kernel/cpuset.c b/kernel/cpuset.c index d5ab79cf516d..eab7bd6628e0 100644 --- a/kernel/cpuset.c +++ b/kernel/cpuset.c | |||
| @@ -14,6 +14,8 @@ | |||
| 14 | * 2003-10-22 Updates by Stephen Hemminger. | 14 | * 2003-10-22 Updates by Stephen Hemminger. |
| 15 | * 2004 May-July Rework by Paul Jackson. | 15 | * 2004 May-July Rework by Paul Jackson. |
| 16 | * 2006 Rework by Paul Menage to use generic cgroups | 16 | * 2006 Rework by Paul Menage to use generic cgroups |
| 17 | * 2008 Rework of the scheduler domains and CPU hotplug handling | ||
| 18 | * by Max Krasnyansky | ||
| 17 | * | 19 | * |
| 18 | * This file is subject to the terms and conditions of the GNU General Public | 20 | * This file is subject to the terms and conditions of the GNU General Public |
| 19 | * License. See the file COPYING in the main directory of the Linux | 21 | * License. See the file COPYING in the main directory of the Linux |
| @@ -236,9 +238,11 @@ static struct cpuset top_cpuset = { | |||
| 236 | 238 | ||
| 237 | static DEFINE_MUTEX(callback_mutex); | 239 | static DEFINE_MUTEX(callback_mutex); |
| 238 | 240 | ||
| 239 | /* This is ugly, but preserves the userspace API for existing cpuset | 241 | /* |
| 242 | * This is ugly, but preserves the userspace API for existing cpuset | ||
| 240 | * users. If someone tries to mount the "cpuset" filesystem, we | 243 | * users. If someone tries to mount the "cpuset" filesystem, we |
| 241 | * silently switch it to mount "cgroup" instead */ | 244 | * silently switch it to mount "cgroup" instead |
| 245 | */ | ||
| 242 | static int cpuset_get_sb(struct file_system_type *fs_type, | 246 | static int cpuset_get_sb(struct file_system_type *fs_type, |
| 243 | int flags, const char *unused_dev_name, | 247 | int flags, const char *unused_dev_name, |
| 244 | void *data, struct vfsmount *mnt) | 248 | void *data, struct vfsmount *mnt) |
| @@ -473,10 +477,9 @@ static int validate_change(const struct cpuset *cur, const struct cpuset *trial) | |||
| 473 | } | 477 | } |
| 474 | 478 | ||
| 475 | /* | 479 | /* |
| 476 | * Helper routine for rebuild_sched_domains(). | 480 | * Helper routine for generate_sched_domains(). |
| 477 | * Do cpusets a, b have overlapping cpus_allowed masks? | 481 | * Do cpusets a, b have overlapping cpus_allowed masks? |
| 478 | */ | 482 | */ |
| 479 | |||
| 480 | static int cpusets_overlap(struct cpuset *a, struct cpuset *b) | 483 | static int cpusets_overlap(struct cpuset *a, struct cpuset *b) |
| 481 | { | 484 | { |
| 482 | return cpus_intersects(a->cpus_allowed, b->cpus_allowed); | 485 | return cpus_intersects(a->cpus_allowed, b->cpus_allowed); |
| @@ -518,26 +521,15 @@ update_domain_attr_tree(struct sched_domain_attr *dattr, struct cpuset *c) | |||
| 518 | } | 521 | } |
| 519 | 522 | ||
| 520 | /* | 523 | /* |
| 521 | * rebuild_sched_domains() | 524 | * generate_sched_domains() |
| 522 | * | 525 | * |
| 523 | * This routine will be called to rebuild the scheduler's dynamic | 526 | * This function builds a partial partition of the systems CPUs |
| 524 | * sched domains: | 527 | * A 'partial partition' is a set of non-overlapping subsets whose |
| 525 | * - if the flag 'sched_load_balance' of any cpuset with non-empty | 528 | * union is a subset of that set. |
| 526 | * 'cpus' changes, | 529 | * The output of this function needs to be passed to kernel/sched.c |
| 527 | * - or if the 'cpus' allowed changes in any cpuset which has that | 530 | * partition_sched_domains() routine, which will rebuild the scheduler's |
| 528 | * flag enabled, | 531 | * load balancing domains (sched domains) as specified by that partial |
| 529 | * - or if the 'sched_relax_domain_level' of any cpuset which has | 532 | * partition. |
| 530 | * that flag enabled and with non-empty 'cpus' changes, | ||
| 531 | * - or if any cpuset with non-empty 'cpus' is removed, | ||
| 532 | * - or if a cpu gets offlined. | ||
| 533 | * | ||
| 534 | * This routine builds a partial partition of the systems CPUs | ||
| 535 | * (the set of non-overlappping cpumask_t's in the array 'part' | ||
| 536 | * below), and passes that partial partition to the kernel/sched.c | ||
| 537 | * partition_sched_domains() routine, which will rebuild the | ||
| 538 | * schedulers load balancing domains (sched domains) as specified | ||
| 539 | * by that partial partition. A 'partial partition' is a set of | ||
| 540 | * non-overlapping subsets whose union is a subset of that set. | ||
| 541 | * | 533 | * |
| 542 | * See "What is sched_load_balance" in Documentation/cpusets.txt | 534 | * See "What is sched_load_balance" in Documentation/cpusets.txt |
| 543 | * for a background explanation of this. | 535 | * for a background explanation of this. |
| @@ -547,13 +539,7 @@ update_domain_attr_tree(struct sched_domain_attr *dattr, struct cpuset *c) | |||
| 547 | * domains when operating in the severe memory shortage situations | 539 | * domains when operating in the severe memory shortage situations |
| 548 | * that could cause allocation failures below. | 540 | * that could cause allocation failures below. |
| 549 | * | 541 | * |
| 550 | * Call with cgroup_mutex held. May take callback_mutex during | 542 | * Must be called with cgroup_lock held. |
| 551 | * call due to the kfifo_alloc() and kmalloc() calls. May nest | ||
| 552 | * a call to the get_online_cpus()/put_online_cpus() pair. | ||
| 553 | * Must not be called holding callback_mutex, because we must not | ||
| 554 | * call get_online_cpus() while holding callback_mutex. Elsewhere | ||
| 555 | * the kernel nests callback_mutex inside get_online_cpus() calls. | ||
| 556 | * So the reverse nesting would risk an ABBA deadlock. | ||
| 557 | * | 543 | * |
| 558 | * The three key local variables below are: | 544 | * The three key local variables below are: |
| 559 | * q - a linked-list queue of cpuset pointers, used to implement a | 545 | * q - a linked-list queue of cpuset pointers, used to implement a |
| @@ -588,10 +574,10 @@ update_domain_attr_tree(struct sched_domain_attr *dattr, struct cpuset *c) | |||
| 588 | * element of the partition (one sched domain) to be passed to | 574 | * element of the partition (one sched domain) to be passed to |
| 589 | * partition_sched_domains(). | 575 | * partition_sched_domains(). |
| 590 | */ | 576 | */ |
| 591 | 577 | static int generate_sched_domains(cpumask_t **domains, | |
| 592 | void rebuild_sched_domains(void) | 578 | struct sched_domain_attr **attributes) |
| 593 | { | 579 | { |
| 594 | LIST_HEAD(q); /* queue of cpusets to be scanned*/ | 580 | LIST_HEAD(q); /* queue of cpusets to be scanned */ |
| 595 | struct cpuset *cp; /* scans q */ | 581 | struct cpuset *cp; /* scans q */ |
| 596 | struct cpuset **csa; /* array of all cpuset ptrs */ | 582 | struct cpuset **csa; /* array of all cpuset ptrs */ |
| 597 | int csn; /* how many cpuset ptrs in csa so far */ | 583 | int csn; /* how many cpuset ptrs in csa so far */ |
| @@ -601,23 +587,26 @@ void rebuild_sched_domains(void) | |||
| 601 | int ndoms; /* number of sched domains in result */ | 587 | int ndoms; /* number of sched domains in result */ |
| 602 | int nslot; /* next empty doms[] cpumask_t slot */ | 588 | int nslot; /* next empty doms[] cpumask_t slot */ |
| 603 | 589 | ||
| 604 | csa = NULL; | 590 | ndoms = 0; |
| 605 | doms = NULL; | 591 | doms = NULL; |
| 606 | dattr = NULL; | 592 | dattr = NULL; |
| 593 | csa = NULL; | ||
| 607 | 594 | ||
| 608 | /* Special case for the 99% of systems with one, full, sched domain */ | 595 | /* Special case for the 99% of systems with one, full, sched domain */ |
| 609 | if (is_sched_load_balance(&top_cpuset)) { | 596 | if (is_sched_load_balance(&top_cpuset)) { |
| 610 | ndoms = 1; | ||
| 611 | doms = kmalloc(sizeof(cpumask_t), GFP_KERNEL); | 597 | doms = kmalloc(sizeof(cpumask_t), GFP_KERNEL); |
| 612 | if (!doms) | 598 | if (!doms) |
| 613 | goto rebuild; | 599 | goto done; |
| 600 | |||
| 614 | dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL); | 601 | dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL); |
| 615 | if (dattr) { | 602 | if (dattr) { |
| 616 | *dattr = SD_ATTR_INIT; | 603 | *dattr = SD_ATTR_INIT; |
| 617 | update_domain_attr_tree(dattr, &top_cpuset); | 604 | update_domain_attr_tree(dattr, &top_cpuset); |
| 618 | } | 605 | } |
| 619 | *doms = top_cpuset.cpus_allowed; | 606 | *doms = top_cpuset.cpus_allowed; |
| 620 | goto rebuild; | 607 | |
| 608 | ndoms = 1; | ||
| 609 | goto done; | ||
| 621 | } | 610 | } |
| 622 | 611 | ||
| 623 | csa = kmalloc(number_of_cpusets * sizeof(cp), GFP_KERNEL); | 612 | csa = kmalloc(number_of_cpusets * sizeof(cp), GFP_KERNEL); |
| @@ -680,61 +669,141 @@ restart: | |||
| 680 | } | 669 | } |
| 681 | } | 670 | } |
| 682 | 671 | ||
| 683 | /* Convert <csn, csa> to <ndoms, doms> */ | 672 | /* |
| 673 | * Now we know how many domains to create. | ||
| 674 | * Convert <csn, csa> to <ndoms, doms> and populate cpu masks. | ||
| 675 | */ | ||
| 684 | doms = kmalloc(ndoms * sizeof(cpumask_t), GFP_KERNEL); | 676 | doms = kmalloc(ndoms * sizeof(cpumask_t), GFP_KERNEL); |
| 685 | if (!doms) | 677 | if (!doms) { |
| 686 | goto rebuild; | 678 | ndoms = 0; |
| 679 | goto done; | ||
| 680 | } | ||
| 681 | |||
| 682 | /* | ||
| 683 | * The rest of the code, including the scheduler, can deal with | ||
| 684 | * dattr==NULL case. No need to abort if alloc fails. | ||
| 685 | */ | ||
| 687 | dattr = kmalloc(ndoms * sizeof(struct sched_domain_attr), GFP_KERNEL); | 686 | dattr = kmalloc(ndoms * sizeof(struct sched_domain_attr), GFP_KERNEL); |
| 688 | 687 | ||
| 689 | for (nslot = 0, i = 0; i < csn; i++) { | 688 | for (nslot = 0, i = 0; i < csn; i++) { |
| 690 | struct cpuset *a = csa[i]; | 689 | struct cpuset *a = csa[i]; |
| 690 | cpumask_t *dp; | ||
| 691 | int apn = a->pn; | 691 | int apn = a->pn; |
| 692 | 692 | ||
| 693 | if (apn >= 0) { | 693 | if (apn < 0) { |
| 694 | cpumask_t *dp = doms + nslot; | 694 | /* Skip completed partitions */ |
| 695 | 695 | continue; | |
| 696 | if (nslot == ndoms) { | 696 | } |
| 697 | static int warnings = 10; | 697 | |
| 698 | if (warnings) { | 698 | dp = doms + nslot; |
| 699 | printk(KERN_WARNING | 699 | |
| 700 | "rebuild_sched_domains confused:" | 700 | if (nslot == ndoms) { |
| 701 | " nslot %d, ndoms %d, csn %d, i %d," | 701 | static int warnings = 10; |
| 702 | " apn %d\n", | 702 | if (warnings) { |
| 703 | nslot, ndoms, csn, i, apn); | 703 | printk(KERN_WARNING |
| 704 | warnings--; | 704 | "rebuild_sched_domains confused:" |
| 705 | } | 705 | " nslot %d, ndoms %d, csn %d, i %d," |
| 706 | continue; | 706 | " apn %d\n", |
| 707 | nslot, ndoms, csn, i, apn); | ||
| 708 | warnings--; | ||
| 707 | } | 709 | } |
| 710 | continue; | ||
| 711 | } | ||
| 708 | 712 | ||
| 709 | cpus_clear(*dp); | 713 | cpus_clear(*dp); |
| 710 | if (dattr) | 714 | if (dattr) |
| 711 | *(dattr + nslot) = SD_ATTR_INIT; | 715 | *(dattr + nslot) = SD_ATTR_INIT; |
| 712 | for (j = i; j < csn; j++) { | 716 | for (j = i; j < csn; j++) { |
| 713 | struct cpuset *b = csa[j]; | 717 | struct cpuset *b = csa[j]; |
| 714 | 718 | ||
| 715 | if (apn == b->pn) { | 719 | if (apn == b->pn) { |
| 716 | cpus_or(*dp, *dp, b->cpus_allowed); | 720 | cpus_or(*dp, *dp, b->cpus_allowed); |
| 717 | b->pn = -1; | 721 | if (dattr) |
| 718 | if (dattr) | 722 | update_domain_attr_tree(dattr + nslot, b); |
| 719 | update_domain_attr_tree(dattr | 723 | |
| 720 | + nslot, b); | 724 | /* Done with this partition */ |
| 721 | } | 725 | b->pn = -1; |
| 722 | } | 726 | } |
| 723 | nslot++; | ||
| 724 | } | 727 | } |
| 728 | nslot++; | ||
| 725 | } | 729 | } |
| 726 | BUG_ON(nslot != ndoms); | 730 | BUG_ON(nslot != ndoms); |
| 727 | 731 | ||
| 728 | rebuild: | 732 | done: |
| 729 | /* Have scheduler rebuild sched domains */ | 733 | kfree(csa); |
| 734 | |||
| 735 | *domains = doms; | ||
| 736 | *attributes = dattr; | ||
| 737 | return ndoms; | ||
| 738 | } | ||
| 739 | |||
| 740 | /* | ||
| 741 | * Rebuild scheduler domains. | ||
| 742 | * | ||
| 743 | * Call with neither cgroup_mutex held nor within get_online_cpus(). | ||
| 744 | * Takes both cgroup_mutex and get_online_cpus(). | ||
| 745 | * | ||
| 746 | * Cannot be directly called from cpuset code handling changes | ||
| 747 | * to the cpuset pseudo-filesystem, because it cannot be called | ||
| 748 | * from code that already holds cgroup_mutex. | ||
| 749 | */ | ||
| 750 | static void do_rebuild_sched_domains(struct work_struct *unused) | ||
| 751 | { | ||
| 752 | struct sched_domain_attr *attr; | ||
| 753 | cpumask_t *doms; | ||
| 754 | int ndoms; | ||
| 755 | |||
| 730 | get_online_cpus(); | 756 | get_online_cpus(); |
| 731 | partition_sched_domains(ndoms, doms, dattr); | 757 | |
| 758 | /* Generate domain masks and attrs */ | ||
| 759 | cgroup_lock(); | ||
| 760 | ndoms = generate_sched_domains(&doms, &attr); | ||
| 761 | cgroup_unlock(); | ||
| 762 | |||
| 763 | /* Have scheduler rebuild the domains */ | ||
| 764 | partition_sched_domains(ndoms, doms, attr); | ||
| 765 | |||
| 732 | put_online_cpus(); | 766 | put_online_cpus(); |
| 767 | } | ||
| 733 | 768 | ||
| 734 | done: | 769 | static DECLARE_WORK(rebuild_sched_domains_work, do_rebuild_sched_domains); |
| 735 | kfree(csa); | 770 | |
| 736 | /* Don't kfree(doms) -- partition_sched_domains() does that. */ | 771 | /* |
| 737 | /* Don't kfree(dattr) -- partition_sched_domains() does that. */ | 772 | * Rebuild scheduler domains, asynchronously via workqueue. |
| 773 | * | ||
| 774 | * If the flag 'sched_load_balance' of any cpuset with non-empty | ||
| 775 | * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset | ||
| 776 | * which has that flag enabled, or if any cpuset with a non-empty | ||
| 777 | * 'cpus' is removed, then call this routine to rebuild the | ||
| 778 | * scheduler's dynamic sched domains. | ||
| 779 | * | ||
| 780 | * The rebuild_sched_domains() and partition_sched_domains() | ||
| 781 | * routines must nest cgroup_lock() inside get_online_cpus(), | ||
| 782 | * but such cpuset changes as these must nest that locking the | ||
| 783 | * other way, holding cgroup_lock() for much of the code. | ||
| 784 | * | ||
| 785 | * So in order to avoid an ABBA deadlock, the cpuset code handling | ||
| 786 | * these user changes delegates the actual sched domain rebuilding | ||
| 787 | * to a separate workqueue thread, which ends up processing the | ||
| 788 | * above do_rebuild_sched_domains() function. | ||
| 789 | */ | ||
| 790 | static void async_rebuild_sched_domains(void) | ||
| 791 | { | ||
| 792 | schedule_work(&rebuild_sched_domains_work); | ||
| 793 | } | ||
| 794 | |||
| 795 | /* | ||
| 796 | * Accomplishes the same scheduler domain rebuild as the above | ||
| 797 | * async_rebuild_sched_domains(), however it directly calls the | ||
| 798 | * rebuild routine synchronously rather than calling it via an | ||
| 799 | * asynchronous work thread. | ||
| 800 | * | ||
| 801 | * This can only be called from code that is not holding | ||
| 802 | * cgroup_mutex (not nested in a cgroup_lock() call.) | ||
| 803 | */ | ||
| 804 | void rebuild_sched_domains(void) | ||
| 805 | { | ||
| 806 | do_rebuild_sched_domains(NULL); | ||
| 738 | } | 807 | } |
| 739 | 808 | ||
| 740 | /** | 809 | /** |
| @@ -774,37 +843,25 @@ static void cpuset_change_cpumask(struct task_struct *tsk, | |||
| 774 | /** | 843 | /** |
| 775 | * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset. | 844 | * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset. |
| 776 | * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed | 845 | * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed |
| 846 | * @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks() | ||
| 777 | * | 847 | * |
| 778 | * Called with cgroup_mutex held | 848 | * Called with cgroup_mutex held |
| 779 | * | 849 | * |
| 780 | * The cgroup_scan_tasks() function will scan all the tasks in a cgroup, | 850 | * The cgroup_scan_tasks() function will scan all the tasks in a cgroup, |
| 781 | * calling callback functions for each. | 851 | * calling callback functions for each. |
| 782 | * | 852 | * |
| 783 | * Return 0 if successful, -errno if not. | 853 | * No return value. It's guaranteed that cgroup_scan_tasks() always returns 0 |
| 854 | * if @heap != NULL. | ||
| 784 | */ | 855 | */ |
| 785 | static int update_tasks_cpumask(struct cpuset *cs) | 856 | static void update_tasks_cpumask(struct cpuset *cs, struct ptr_heap *heap) |
| 786 | { | 857 | { |
| 787 | struct cgroup_scanner scan; | 858 | struct cgroup_scanner scan; |
| 788 | struct ptr_heap heap; | ||
| 789 | int retval; | ||
| 790 | |||
| 791 | /* | ||
| 792 | * cgroup_scan_tasks() will initialize heap->gt for us. | ||
| 793 | * heap_init() is still needed here for we should not change | ||
| 794 | * cs->cpus_allowed when heap_init() fails. | ||
| 795 | */ | ||
| 796 | retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL); | ||
| 797 | if (retval) | ||
| 798 | return retval; | ||
| 799 | 859 | ||
| 800 | scan.cg = cs->css.cgroup; | 860 | scan.cg = cs->css.cgroup; |
| 801 | scan.test_task = cpuset_test_cpumask; | 861 | scan.test_task = cpuset_test_cpumask; |
| 802 | scan.process_task = cpuset_change_cpumask; | 862 | scan.process_task = cpuset_change_cpumask; |
| 803 | scan.heap = &heap; | 863 | scan.heap = heap; |
| 804 | retval = cgroup_scan_tasks(&scan); | 864 | cgroup_scan_tasks(&scan); |
| 805 | |||
| 806 | heap_free(&heap); | ||
| 807 | return retval; | ||
| 808 | } | 865 | } |
| 809 | 866 | ||
| 810 | /** | 867 | /** |
| @@ -814,6 +871,7 @@ static int update_tasks_cpumask(struct cpuset *cs) | |||
| 814 | */ | 871 | */ |
| 815 | static int update_cpumask(struct cpuset *cs, const char *buf) | 872 | static int update_cpumask(struct cpuset *cs, const char *buf) |
| 816 | { | 873 | { |
| 874 | struct ptr_heap heap; | ||
| 817 | struct cpuset trialcs; | 875 | struct cpuset trialcs; |
| 818 | int retval; | 876 | int retval; |
| 819 | int is_load_balanced; | 877 | int is_load_balanced; |
| @@ -848,6 +906,10 @@ static int update_cpumask(struct cpuset *cs, const char *buf) | |||
| 848 | if (cpus_equal(cs->cpus_allowed, trialcs.cpus_allowed)) | 906 | if (cpus_equal(cs->cpus_allowed, trialcs.cpus_allowed)) |
| 849 | return 0; | 907 | return 0; |
| 850 | 908 | ||
| 909 | retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL); | ||
| 910 | if (retval) | ||
| 911 | return retval; | ||
| 912 | |||
| 851 | is_load_balanced = is_sched_load_balance(&trialcs); | 913 | is_load_balanced = is_sched_load_balance(&trialcs); |
| 852 | 914 | ||
| 853 | mutex_lock(&callback_mutex); | 915 | mutex_lock(&callback_mutex); |
| @@ -858,12 +920,12 @@ static int update_cpumask(struct cpuset *cs, const char *buf) | |||
| 858 | * Scan tasks in the cpuset, and update the cpumasks of any | 920 | * Scan tasks in the cpuset, and update the cpumasks of any |
| 859 | * that need an update. | 921 | * that need an update. |
| 860 | */ | 922 | */ |
| 861 | retval = update_tasks_cpumask(cs); | 923 | update_tasks_cpumask(cs, &heap); |
| 862 | if (retval < 0) | 924 | |
| 863 | return retval; | 925 | heap_free(&heap); |
| 864 | 926 | ||
| 865 | if (is_load_balanced) | 927 | if (is_load_balanced) |
| 866 | rebuild_sched_domains(); | 928 | async_rebuild_sched_domains(); |
| 867 | return 0; | 929 | return 0; |
| 868 | } | 930 | } |
| 869 | 931 | ||
| @@ -1090,7 +1152,7 @@ static int update_relax_domain_level(struct cpuset *cs, s64 val) | |||
| 1090 | if (val != cs->relax_domain_level) { | 1152 | if (val != cs->relax_domain_level) { |
| 1091 | cs->relax_domain_level = val; | 1153 | cs->relax_domain_level = val; |
| 1092 | if (!cpus_empty(cs->cpus_allowed) && is_sched_load_balance(cs)) | 1154 | if (!cpus_empty(cs->cpus_allowed) && is_sched_load_balance(cs)) |
| 1093 | rebuild_sched_domains(); | 1155 | async_rebuild_sched_domains(); |
| 1094 | } | 1156 | } |
| 1095 | 1157 | ||
| 1096 | return 0; | 1158 | return 0; |
| @@ -1131,7 +1193,7 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, | |||
| 1131 | mutex_unlock(&callback_mutex); | 1193 | mutex_unlock(&callback_mutex); |
| 1132 | 1194 | ||
| 1133 | if (cpus_nonempty && balance_flag_changed) | 1195 | if (cpus_nonempty && balance_flag_changed) |
| 1134 | rebuild_sched_domains(); | 1196 | async_rebuild_sched_domains(); |
| 1135 | 1197 | ||
| 1136 | return 0; | 1198 | return 0; |
| 1137 | } | 1199 | } |
| @@ -1492,6 +1554,9 @@ static u64 cpuset_read_u64(struct cgroup *cont, struct cftype *cft) | |||
| 1492 | default: | 1554 | default: |
| 1493 | BUG(); | 1555 | BUG(); |
| 1494 | } | 1556 | } |
| 1557 | |||
| 1558 | /* Unreachable but makes gcc happy */ | ||
| 1559 | return 0; | ||
| 1495 | } | 1560 | } |
| 1496 | 1561 | ||
| 1497 | static s64 cpuset_read_s64(struct cgroup *cont, struct cftype *cft) | 1562 | static s64 cpuset_read_s64(struct cgroup *cont, struct cftype *cft) |
| @@ -1504,6 +1569,9 @@ static s64 cpuset_read_s64(struct cgroup *cont, struct cftype *cft) | |||
| 1504 | default: | 1569 | default: |
| 1505 | BUG(); | 1570 | BUG(); |
| 1506 | } | 1571 | } |
| 1572 | |||
| 1573 | /* Unrechable but makes gcc happy */ | ||
| 1574 | return 0; | ||
| 1507 | } | 1575 | } |
| 1508 | 1576 | ||
| 1509 | 1577 | ||
| @@ -1692,15 +1760,9 @@ static struct cgroup_subsys_state *cpuset_create( | |||
| 1692 | } | 1760 | } |
| 1693 | 1761 | ||
| 1694 | /* | 1762 | /* |
| 1695 | * Locking note on the strange update_flag() call below: | ||
| 1696 | * | ||
| 1697 | * If the cpuset being removed has its flag 'sched_load_balance' | 1763 | * If the cpuset being removed has its flag 'sched_load_balance' |
| 1698 | * enabled, then simulate turning sched_load_balance off, which | 1764 | * enabled, then simulate turning sched_load_balance off, which |
| 1699 | * will call rebuild_sched_domains(). The get_online_cpus() | 1765 | * will call async_rebuild_sched_domains(). |
| 1700 | * call in rebuild_sched_domains() must not be made while holding | ||
| 1701 | * callback_mutex. Elsewhere the kernel nests callback_mutex inside | ||
| 1702 | * get_online_cpus() calls. So the reverse nesting would risk an | ||
| 1703 | * ABBA deadlock. | ||
| 1704 | */ | 1766 | */ |
| 1705 | 1767 | ||
| 1706 | static void cpuset_destroy(struct cgroup_subsys *ss, struct cgroup *cont) | 1768 | static void cpuset_destroy(struct cgroup_subsys *ss, struct cgroup *cont) |
| @@ -1719,7 +1781,7 @@ static void cpuset_destroy(struct cgroup_subsys *ss, struct cgroup *cont) | |||
| 1719 | struct cgroup_subsys cpuset_subsys = { | 1781 | struct cgroup_subsys cpuset_subsys = { |
| 1720 | .name = "cpuset", | 1782 | .name = "cpuset", |
| 1721 | .create = cpuset_create, | 1783 | .create = cpuset_create, |
| 1722 | .destroy = cpuset_destroy, | 1784 | .destroy = cpuset_destroy, |
| 1723 | .can_attach = cpuset_can_attach, | 1785 | .can_attach = cpuset_can_attach, |
| 1724 | .attach = cpuset_attach, | 1786 | .attach = cpuset_attach, |
| 1725 | .populate = cpuset_populate, | 1787 | .populate = cpuset_populate, |
| @@ -1811,7 +1873,7 @@ static void move_member_tasks_to_cpuset(struct cpuset *from, struct cpuset *to) | |||
| 1811 | } | 1873 | } |
| 1812 | 1874 | ||
| 1813 | /* | 1875 | /* |
| 1814 | * If common_cpu_mem_hotplug_unplug(), below, unplugs any CPUs | 1876 | * If CPU and/or memory hotplug handlers, below, unplug any CPUs |
| 1815 | * or memory nodes, we need to walk over the cpuset hierarchy, | 1877 | * or memory nodes, we need to walk over the cpuset hierarchy, |
| 1816 | * removing that CPU or node from all cpusets. If this removes the | 1878 | * removing that CPU or node from all cpusets. If this removes the |
| 1817 | * last CPU or node from a cpuset, then move the tasks in the empty | 1879 | * last CPU or node from a cpuset, then move the tasks in the empty |
| @@ -1859,7 +1921,7 @@ static void remove_tasks_in_empty_cpuset(struct cpuset *cs) | |||
| 1859 | * that has tasks along with an empty 'mems'. But if we did see such | 1921 | * that has tasks along with an empty 'mems'. But if we did see such |
| 1860 | * a cpuset, we'd handle it just like we do if its 'cpus' was empty. | 1922 | * a cpuset, we'd handle it just like we do if its 'cpus' was empty. |
| 1861 | */ | 1923 | */ |
| 1862 | static void scan_for_empty_cpusets(const struct cpuset *root) | 1924 | static void scan_for_empty_cpusets(struct cpuset *root) |
| 1863 | { | 1925 | { |
| 1864 | LIST_HEAD(queue); | 1926 | LIST_HEAD(queue); |
| 1865 | struct cpuset *cp; /* scans cpusets being updated */ | 1927 | struct cpuset *cp; /* scans cpusets being updated */ |
| @@ -1896,42 +1958,13 @@ static void scan_for_empty_cpusets(const struct cpuset *root) | |||
| 1896 | nodes_empty(cp->mems_allowed)) | 1958 | nodes_empty(cp->mems_allowed)) |
| 1897 | remove_tasks_in_empty_cpuset(cp); | 1959 | remove_tasks_in_empty_cpuset(cp); |
| 1898 | else { | 1960 | else { |
| 1899 | update_tasks_cpumask(cp); | 1961 | update_tasks_cpumask(cp, NULL); |
| 1900 | update_tasks_nodemask(cp, &oldmems); | 1962 | update_tasks_nodemask(cp, &oldmems); |
| 1901 | } | 1963 | } |
| 1902 | } | 1964 | } |
| 1903 | } | 1965 | } |
| 1904 | 1966 | ||
| 1905 | /* | 1967 | /* |
| 1906 | * The cpus_allowed and mems_allowed nodemasks in the top_cpuset track | ||
| 1907 | * cpu_online_map and node_states[N_HIGH_MEMORY]. Force the top cpuset to | ||
| 1908 | * track what's online after any CPU or memory node hotplug or unplug event. | ||
| 1909 | * | ||
| 1910 | * Since there are two callers of this routine, one for CPU hotplug | ||
| 1911 | * events and one for memory node hotplug events, we could have coded | ||
| 1912 | * two separate routines here. We code it as a single common routine | ||
| 1913 | * in order to minimize text size. | ||
| 1914 | */ | ||
| 1915 | |||
| 1916 | static void common_cpu_mem_hotplug_unplug(int rebuild_sd) | ||
| 1917 | { | ||
| 1918 | cgroup_lock(); | ||
| 1919 | |||
| 1920 | top_cpuset.cpus_allowed = cpu_online_map; | ||
| 1921 | top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY]; | ||
| 1922 | scan_for_empty_cpusets(&top_cpuset); | ||
| 1923 | |||
| 1924 | /* | ||
| 1925 | * Scheduler destroys domains on hotplug events. | ||
| 1926 | * Rebuild them based on the current settings. | ||
| 1927 | */ | ||
| 1928 | if (rebuild_sd) | ||
| 1929 | rebuild_sched_domains(); | ||
| 1930 | |||
| 1931 | cgroup_unlock(); | ||
| 1932 | } | ||
| 1933 | |||
| 1934 | /* | ||
| 1935 | * The top_cpuset tracks what CPUs and Memory Nodes are online, | 1968 | * The top_cpuset tracks what CPUs and Memory Nodes are online, |
| 1936 | * period. This is necessary in order to make cpusets transparent | 1969 | * period. This is necessary in order to make cpusets transparent |
| 1937 | * (of no affect) on systems that are actively using CPU hotplug | 1970 | * (of no affect) on systems that are actively using CPU hotplug |
| @@ -1939,40 +1972,52 @@ static void common_cpu_mem_hotplug_unplug(int rebuild_sd) | |||
| 1939 | * | 1972 | * |
| 1940 | * This routine ensures that top_cpuset.cpus_allowed tracks | 1973 | * This routine ensures that top_cpuset.cpus_allowed tracks |
| 1941 | * cpu_online_map on each CPU hotplug (cpuhp) event. | 1974 | * cpu_online_map on each CPU hotplug (cpuhp) event. |
| 1975 | * | ||
| 1976 | * Called within get_online_cpus(). Needs to call cgroup_lock() | ||
| 1977 | * before calling generate_sched_domains(). | ||
| 1942 | */ | 1978 | */ |
| 1943 | 1979 | static int cpuset_track_online_cpus(struct notifier_block *unused_nb, | |
| 1944 | static int cpuset_handle_cpuhp(struct notifier_block *unused_nb, | ||
| 1945 | unsigned long phase, void *unused_cpu) | 1980 | unsigned long phase, void *unused_cpu) |
| 1946 | { | 1981 | { |
| 1982 | struct sched_domain_attr *attr; | ||
| 1983 | cpumask_t *doms; | ||
| 1984 | int ndoms; | ||
| 1985 | |||
| 1947 | switch (phase) { | 1986 | switch (phase) { |
| 1948 | case CPU_UP_CANCELED: | ||
| 1949 | case CPU_UP_CANCELED_FROZEN: | ||
| 1950 | case CPU_DOWN_FAILED: | ||
| 1951 | case CPU_DOWN_FAILED_FROZEN: | ||
| 1952 | case CPU_ONLINE: | 1987 | case CPU_ONLINE: |
| 1953 | case CPU_ONLINE_FROZEN: | 1988 | case CPU_ONLINE_FROZEN: |
| 1954 | case CPU_DEAD: | 1989 | case CPU_DEAD: |
| 1955 | case CPU_DEAD_FROZEN: | 1990 | case CPU_DEAD_FROZEN: |
| 1956 | common_cpu_mem_hotplug_unplug(1); | ||
| 1957 | break; | 1991 | break; |
| 1992 | |||
| 1958 | default: | 1993 | default: |
| 1959 | return NOTIFY_DONE; | 1994 | return NOTIFY_DONE; |
| 1960 | } | 1995 | } |
| 1961 | 1996 | ||
| 1997 | cgroup_lock(); | ||
| 1998 | top_cpuset.cpus_allowed = cpu_online_map; | ||
| 1999 | scan_for_empty_cpusets(&top_cpuset); | ||
| 2000 | ndoms = generate_sched_domains(&doms, &attr); | ||
| 2001 | cgroup_unlock(); | ||
| 2002 | |||
| 2003 | /* Have scheduler rebuild the domains */ | ||
| 2004 | partition_sched_domains(ndoms, doms, attr); | ||
| 2005 | |||
| 1962 | return NOTIFY_OK; | 2006 | return NOTIFY_OK; |
| 1963 | } | 2007 | } |
| 1964 | 2008 | ||
| 1965 | #ifdef CONFIG_MEMORY_HOTPLUG | 2009 | #ifdef CONFIG_MEMORY_HOTPLUG |
| 1966 | /* | 2010 | /* |
| 1967 | * Keep top_cpuset.mems_allowed tracking node_states[N_HIGH_MEMORY]. | 2011 | * Keep top_cpuset.mems_allowed tracking node_states[N_HIGH_MEMORY]. |
| 1968 | * Call this routine anytime after you change | 2012 | * Call this routine anytime after node_states[N_HIGH_MEMORY] changes. |
| 1969 | * node_states[N_HIGH_MEMORY]. | 2013 | * See also the previous routine cpuset_track_online_cpus(). |
| 1970 | * See also the previous routine cpuset_handle_cpuhp(). | ||
| 1971 | */ | 2014 | */ |
| 1972 | |||
| 1973 | void cpuset_track_online_nodes(void) | 2015 | void cpuset_track_online_nodes(void) |
| 1974 | { | 2016 | { |
| 1975 | common_cpu_mem_hotplug_unplug(0); | 2017 | cgroup_lock(); |
| 2018 | top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY]; | ||
| 2019 | scan_for_empty_cpusets(&top_cpuset); | ||
| 2020 | cgroup_unlock(); | ||
| 1976 | } | 2021 | } |
| 1977 | #endif | 2022 | #endif |
| 1978 | 2023 | ||
| @@ -1987,7 +2032,7 @@ void __init cpuset_init_smp(void) | |||
| 1987 | top_cpuset.cpus_allowed = cpu_online_map; | 2032 | top_cpuset.cpus_allowed = cpu_online_map; |
| 1988 | top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY]; | 2033 | top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY]; |
| 1989 | 2034 | ||
| 1990 | hotcpu_notifier(cpuset_handle_cpuhp, 0); | 2035 | hotcpu_notifier(cpuset_track_online_cpus, 0); |
| 1991 | } | 2036 | } |
| 1992 | 2037 | ||
| 1993 | /** | 2038 | /** |
diff --git a/kernel/dma-coherent.c b/kernel/dma-coherent.c index c1d4d5b4c61c..f013a0c2e111 100644 --- a/kernel/dma-coherent.c +++ b/kernel/dma-coherent.c | |||
| @@ -124,6 +124,7 @@ int dma_alloc_from_coherent(struct device *dev, ssize_t size, | |||
| 124 | } | 124 | } |
| 125 | return (mem != NULL); | 125 | return (mem != NULL); |
| 126 | } | 126 | } |
| 127 | EXPORT_SYMBOL(dma_alloc_from_coherent); | ||
| 127 | 128 | ||
| 128 | /** | 129 | /** |
| 129 | * dma_release_from_coherent() - try to free the memory allocated from per-device coherent memory pool | 130 | * dma_release_from_coherent() - try to free the memory allocated from per-device coherent memory pool |
| @@ -151,3 +152,4 @@ int dma_release_from_coherent(struct device *dev, int order, void *vaddr) | |||
| 151 | } | 152 | } |
| 152 | return 0; | 153 | return 0; |
| 153 | } | 154 | } |
| 155 | EXPORT_SYMBOL(dma_release_from_coherent); | ||
diff --git a/kernel/dma.c b/kernel/dma.c index d2c60a822790..f903189c5304 100644 --- a/kernel/dma.c +++ b/kernel/dma.c | |||
| @@ -1,4 +1,4 @@ | |||
| 1 | /* $Id: dma.c,v 1.7 1994/12/28 03:35:33 root Exp root $ | 1 | /* |
| 2 | * linux/kernel/dma.c: A DMA channel allocator. Inspired by linux/kernel/irq.c. | 2 | * linux/kernel/dma.c: A DMA channel allocator. Inspired by linux/kernel/irq.c. |
| 3 | * | 3 | * |
| 4 | * Written by Hennus Bergman, 1992. | 4 | * Written by Hennus Bergman, 1992. |
diff --git a/kernel/exit.c b/kernel/exit.c index 16395644a98f..0ef4673e351b 100644 --- a/kernel/exit.c +++ b/kernel/exit.c | |||
| @@ -583,8 +583,6 @@ mm_need_new_owner(struct mm_struct *mm, struct task_struct *p) | |||
| 583 | * If there are other users of the mm and the owner (us) is exiting | 583 | * If there are other users of the mm and the owner (us) is exiting |
| 584 | * we need to find a new owner to take on the responsibility. | 584 | * we need to find a new owner to take on the responsibility. |
| 585 | */ | 585 | */ |
| 586 | if (!mm) | ||
| 587 | return 0; | ||
| 588 | if (atomic_read(&mm->mm_users) <= 1) | 586 | if (atomic_read(&mm->mm_users) <= 1) |
| 589 | return 0; | 587 | return 0; |
| 590 | if (mm->owner != p) | 588 | if (mm->owner != p) |
| @@ -627,29 +625,38 @@ retry: | |||
| 627 | } while_each_thread(g, c); | 625 | } while_each_thread(g, c); |
| 628 | 626 | ||
| 629 | read_unlock(&tasklist_lock); | 627 | read_unlock(&tasklist_lock); |
| 628 | /* | ||
| 629 | * We found no owner yet mm_users > 1: this implies that we are | ||
| 630 | * most likely racing with swapoff (try_to_unuse()) or /proc or | ||
| 631 | * ptrace or page migration (get_task_mm()). Mark owner as NULL, | ||
| 632 | * so that subsystems can understand the callback and take action. | ||
| 633 | */ | ||
| 634 | down_write(&mm->mmap_sem); | ||
| 635 | cgroup_mm_owner_callbacks(mm->owner, NULL); | ||
| 636 | mm->owner = NULL; | ||
| 637 | up_write(&mm->mmap_sem); | ||
| 630 | return; | 638 | return; |
| 631 | 639 | ||
| 632 | assign_new_owner: | 640 | assign_new_owner: |
| 633 | BUG_ON(c == p); | 641 | BUG_ON(c == p); |
| 634 | get_task_struct(c); | 642 | get_task_struct(c); |
| 643 | read_unlock(&tasklist_lock); | ||
| 644 | down_write(&mm->mmap_sem); | ||
| 635 | /* | 645 | /* |
| 636 | * The task_lock protects c->mm from changing. | 646 | * The task_lock protects c->mm from changing. |
| 637 | * We always want mm->owner->mm == mm | 647 | * We always want mm->owner->mm == mm |
| 638 | */ | 648 | */ |
| 639 | task_lock(c); | 649 | task_lock(c); |
| 640 | /* | ||
| 641 | * Delay read_unlock() till we have the task_lock() | ||
| 642 | * to ensure that c does not slip away underneath us | ||
| 643 | */ | ||
| 644 | read_unlock(&tasklist_lock); | ||
| 645 | if (c->mm != mm) { | 650 | if (c->mm != mm) { |
| 646 | task_unlock(c); | 651 | task_unlock(c); |
| 652 | up_write(&mm->mmap_sem); | ||
| 647 | put_task_struct(c); | 653 | put_task_struct(c); |
| 648 | goto retry; | 654 | goto retry; |
| 649 | } | 655 | } |
| 650 | cgroup_mm_owner_callbacks(mm->owner, c); | 656 | cgroup_mm_owner_callbacks(mm->owner, c); |
| 651 | mm->owner = c; | 657 | mm->owner = c; |
| 652 | task_unlock(c); | 658 | task_unlock(c); |
| 659 | up_write(&mm->mmap_sem); | ||
| 653 | put_task_struct(c); | 660 | put_task_struct(c); |
| 654 | } | 661 | } |
| 655 | #endif /* CONFIG_MM_OWNER */ | 662 | #endif /* CONFIG_MM_OWNER */ |
diff --git a/kernel/fork.c b/kernel/fork.c index 4308d75f0fa5..37b3e150ae39 100644 --- a/kernel/fork.c +++ b/kernel/fork.c | |||
| @@ -802,6 +802,7 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk) | |||
| 802 | 802 | ||
| 803 | sig->leader = 0; /* session leadership doesn't inherit */ | 803 | sig->leader = 0; /* session leadership doesn't inherit */ |
| 804 | sig->tty_old_pgrp = NULL; | 804 | sig->tty_old_pgrp = NULL; |
| 805 | sig->tty = NULL; | ||
| 805 | 806 | ||
| 806 | sig->utime = sig->stime = sig->cutime = sig->cstime = cputime_zero; | 807 | sig->utime = sig->stime = sig->cutime = sig->cstime = cputime_zero; |
| 807 | sig->gtime = cputime_zero; | 808 | sig->gtime = cputime_zero; |
| @@ -838,6 +839,7 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk) | |||
| 838 | void __cleanup_signal(struct signal_struct *sig) | 839 | void __cleanup_signal(struct signal_struct *sig) |
| 839 | { | 840 | { |
| 840 | exit_thread_group_keys(sig); | 841 | exit_thread_group_keys(sig); |
| 842 | tty_kref_put(sig->tty); | ||
| 841 | kmem_cache_free(signal_cachep, sig); | 843 | kmem_cache_free(signal_cachep, sig); |
| 842 | } | 844 | } |
| 843 | 845 | ||
| @@ -1229,7 +1231,8 @@ static struct task_struct *copy_process(unsigned long clone_flags, | |||
| 1229 | p->nsproxy->pid_ns->child_reaper = p; | 1231 | p->nsproxy->pid_ns->child_reaper = p; |
| 1230 | 1232 | ||
| 1231 | p->signal->leader_pid = pid; | 1233 | p->signal->leader_pid = pid; |
| 1232 | p->signal->tty = current->signal->tty; | 1234 | tty_kref_put(p->signal->tty); |
| 1235 | p->signal->tty = tty_kref_get(current->signal->tty); | ||
| 1233 | set_task_pgrp(p, task_pgrp_nr(current)); | 1236 | set_task_pgrp(p, task_pgrp_nr(current)); |
| 1234 | set_task_session(p, task_session_nr(current)); | 1237 | set_task_session(p, task_session_nr(current)); |
| 1235 | attach_pid(p, PIDTYPE_PGID, task_pgrp(current)); | 1238 | attach_pid(p, PIDTYPE_PGID, task_pgrp(current)); |
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 2bd230be1cb5..51ee90bca2de 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c | |||
| @@ -672,13 +672,14 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, | |||
| 672 | */ | 672 | */ |
| 673 | BUG_ON(timer->function(timer) != HRTIMER_NORESTART); | 673 | BUG_ON(timer->function(timer) != HRTIMER_NORESTART); |
| 674 | return 1; | 674 | return 1; |
| 675 | case HRTIMER_CB_IRQSAFE_NO_SOFTIRQ: | 675 | case HRTIMER_CB_IRQSAFE_PERCPU: |
| 676 | case HRTIMER_CB_IRQSAFE_UNLOCKED: | ||
| 676 | /* | 677 | /* |
| 677 | * This is solely for the sched tick emulation with | 678 | * This is solely for the sched tick emulation with |
| 678 | * dynamic tick support to ensure that we do not | 679 | * dynamic tick support to ensure that we do not |
| 679 | * restart the tick right on the edge and end up with | 680 | * restart the tick right on the edge and end up with |
| 680 | * the tick timer in the softirq ! The calling site | 681 | * the tick timer in the softirq ! The calling site |
| 681 | * takes care of this. | 682 | * takes care of this. Also used for hrtimer sleeper ! |
| 682 | */ | 683 | */ |
| 683 | debug_hrtimer_deactivate(timer); | 684 | debug_hrtimer_deactivate(timer); |
| 684 | return 1; | 685 | return 1; |
| @@ -1266,7 +1267,8 @@ static void __run_hrtimer(struct hrtimer *timer) | |||
| 1266 | timer_stats_account_hrtimer(timer); | 1267 | timer_stats_account_hrtimer(timer); |
| 1267 | 1268 | ||
| 1268 | fn = timer->function; | 1269 | fn = timer->function; |
| 1269 | if (timer->cb_mode == HRTIMER_CB_IRQSAFE_NO_SOFTIRQ) { | 1270 | if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU || |
| 1271 | timer->cb_mode == HRTIMER_CB_IRQSAFE_UNLOCKED) { | ||
| 1270 | /* | 1272 | /* |
| 1271 | * Used for scheduler timers, avoid lock inversion with | 1273 | * Used for scheduler timers, avoid lock inversion with |
| 1272 | * rq->lock and tasklist_lock. | 1274 | * rq->lock and tasklist_lock. |
| @@ -1517,7 +1519,7 @@ void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task) | |||
| 1517 | sl->timer.function = hrtimer_wakeup; | 1519 | sl->timer.function = hrtimer_wakeup; |
| 1518 | sl->task = task; | 1520 | sl->task = task; |
| 1519 | #ifdef CONFIG_HIGH_RES_TIMERS | 1521 | #ifdef CONFIG_HIGH_RES_TIMERS |
| 1520 | sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ; | 1522 | sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED; |
| 1521 | #endif | 1523 | #endif |
| 1522 | } | 1524 | } |
| 1523 | 1525 | ||
| @@ -1661,29 +1663,95 @@ static void __cpuinit init_hrtimers_cpu(int cpu) | |||
| 1661 | 1663 | ||
| 1662 | #ifdef CONFIG_HOTPLUG_CPU | 1664 | #ifdef CONFIG_HOTPLUG_CPU |
| 1663 | 1665 | ||
| 1664 | static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, | 1666 | static int migrate_hrtimer_list(struct hrtimer_clock_base *old_base, |
| 1665 | struct hrtimer_clock_base *new_base) | 1667 | struct hrtimer_clock_base *new_base, int dcpu) |
| 1666 | { | 1668 | { |
| 1667 | struct hrtimer *timer; | 1669 | struct hrtimer *timer; |
| 1668 | struct rb_node *node; | 1670 | struct rb_node *node; |
| 1671 | int raise = 0; | ||
| 1669 | 1672 | ||
| 1670 | while ((node = rb_first(&old_base->active))) { | 1673 | while ((node = rb_first(&old_base->active))) { |
| 1671 | timer = rb_entry(node, struct hrtimer, node); | 1674 | timer = rb_entry(node, struct hrtimer, node); |
| 1672 | BUG_ON(hrtimer_callback_running(timer)); | 1675 | BUG_ON(hrtimer_callback_running(timer)); |
| 1673 | debug_hrtimer_deactivate(timer); | 1676 | debug_hrtimer_deactivate(timer); |
| 1674 | __remove_hrtimer(timer, old_base, HRTIMER_STATE_INACTIVE, 0); | 1677 | |
| 1678 | /* | ||
| 1679 | * Should not happen. Per CPU timers should be | ||
| 1680 | * canceled _before_ the migration code is called | ||
| 1681 | */ | ||
| 1682 | if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU) { | ||
| 1683 | __remove_hrtimer(timer, old_base, | ||
| 1684 | HRTIMER_STATE_INACTIVE, 0); | ||
| 1685 | WARN(1, "hrtimer (%p %p)active but cpu %d dead\n", | ||
| 1686 | timer, timer->function, dcpu); | ||
| 1687 | continue; | ||
| 1688 | } | ||
| 1689 | |||
| 1690 | /* | ||
| 1691 | * Mark it as STATE_MIGRATE not INACTIVE otherwise the | ||
| 1692 | * timer could be seen as !active and just vanish away | ||
| 1693 | * under us on another CPU | ||
| 1694 | */ | ||
| 1695 | __remove_hrtimer(timer, old_base, HRTIMER_STATE_MIGRATE, 0); | ||
| 1675 | timer->base = new_base; | 1696 | timer->base = new_base; |
| 1676 | /* | 1697 | /* |
| 1677 | * Enqueue the timer. Allow reprogramming of the event device | 1698 | * Enqueue the timer. Allow reprogramming of the event device |
| 1678 | */ | 1699 | */ |
| 1679 | enqueue_hrtimer(timer, new_base, 1); | 1700 | enqueue_hrtimer(timer, new_base, 1); |
| 1701 | |||
| 1702 | #ifdef CONFIG_HIGH_RES_TIMERS | ||
| 1703 | /* | ||
| 1704 | * Happens with high res enabled when the timer was | ||
| 1705 | * already expired and the callback mode is | ||
| 1706 | * HRTIMER_CB_IRQSAFE_UNLOCKED (hrtimer_sleeper). The | ||
| 1707 | * enqueue code does not move them to the soft irq | ||
| 1708 | * pending list for performance/latency reasons, but | ||
| 1709 | * in the migration state, we need to do that | ||
| 1710 | * otherwise we end up with a stale timer. | ||
| 1711 | */ | ||
| 1712 | if (timer->state == HRTIMER_STATE_MIGRATE) { | ||
| 1713 | timer->state = HRTIMER_STATE_PENDING; | ||
| 1714 | list_add_tail(&timer->cb_entry, | ||
| 1715 | &new_base->cpu_base->cb_pending); | ||
| 1716 | raise = 1; | ||
| 1717 | } | ||
| 1718 | #endif | ||
| 1719 | /* Clear the migration state bit */ | ||
| 1720 | timer->state &= ~HRTIMER_STATE_MIGRATE; | ||
| 1721 | } | ||
| 1722 | return raise; | ||
| 1723 | } | ||
| 1724 | |||
| 1725 | #ifdef CONFIG_HIGH_RES_TIMERS | ||
| 1726 | static int migrate_hrtimer_pending(struct hrtimer_cpu_base *old_base, | ||
| 1727 | struct hrtimer_cpu_base *new_base) | ||
| 1728 | { | ||
| 1729 | struct hrtimer *timer; | ||
| 1730 | int raise = 0; | ||
| 1731 | |||
| 1732 | while (!list_empty(&old_base->cb_pending)) { | ||
| 1733 | timer = list_entry(old_base->cb_pending.next, | ||
| 1734 | struct hrtimer, cb_entry); | ||
| 1735 | |||
| 1736 | __remove_hrtimer(timer, timer->base, HRTIMER_STATE_PENDING, 0); | ||
| 1737 | timer->base = &new_base->clock_base[timer->base->index]; | ||
| 1738 | list_add_tail(&timer->cb_entry, &new_base->cb_pending); | ||
| 1739 | raise = 1; | ||
| 1680 | } | 1740 | } |
| 1741 | return raise; | ||
| 1742 | } | ||
| 1743 | #else | ||
| 1744 | static int migrate_hrtimer_pending(struct hrtimer_cpu_base *old_base, | ||
| 1745 | struct hrtimer_cpu_base *new_base) | ||
| 1746 | { | ||
| 1747 | return 0; | ||
| 1681 | } | 1748 | } |
| 1749 | #endif | ||
| 1682 | 1750 | ||
| 1683 | static void migrate_hrtimers(int cpu) | 1751 | static void migrate_hrtimers(int cpu) |
| 1684 | { | 1752 | { |
| 1685 | struct hrtimer_cpu_base *old_base, *new_base; | 1753 | struct hrtimer_cpu_base *old_base, *new_base; |
| 1686 | int i; | 1754 | int i, raise = 0; |
| 1687 | 1755 | ||
| 1688 | BUG_ON(cpu_online(cpu)); | 1756 | BUG_ON(cpu_online(cpu)); |
| 1689 | old_base = &per_cpu(hrtimer_bases, cpu); | 1757 | old_base = &per_cpu(hrtimer_bases, cpu); |
| @@ -1696,14 +1764,21 @@ static void migrate_hrtimers(int cpu) | |||
| 1696 | spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); | 1764 | spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); |
| 1697 | 1765 | ||
| 1698 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { | 1766 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { |
| 1699 | migrate_hrtimer_list(&old_base->clock_base[i], | 1767 | if (migrate_hrtimer_list(&old_base->clock_base[i], |
| 1700 | &new_base->clock_base[i]); | 1768 | &new_base->clock_base[i], cpu)) |
| 1769 | raise = 1; | ||
| 1701 | } | 1770 | } |
| 1702 | 1771 | ||
| 1772 | if (migrate_hrtimer_pending(old_base, new_base)) | ||
| 1773 | raise = 1; | ||
| 1774 | |||
| 1703 | spin_unlock(&old_base->lock); | 1775 | spin_unlock(&old_base->lock); |
| 1704 | spin_unlock(&new_base->lock); | 1776 | spin_unlock(&new_base->lock); |
| 1705 | local_irq_enable(); | 1777 | local_irq_enable(); |
| 1706 | put_cpu_var(hrtimer_bases); | 1778 | put_cpu_var(hrtimer_bases); |
| 1779 | |||
| 1780 | if (raise) | ||
| 1781 | hrtimer_raise_softirq(); | ||
| 1707 | } | 1782 | } |
| 1708 | #endif /* CONFIG_HOTPLUG_CPU */ | 1783 | #endif /* CONFIG_HOTPLUG_CPU */ |
| 1709 | 1784 | ||
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 0314074fa232..60c49e324390 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c | |||
| @@ -89,7 +89,14 @@ int irq_set_affinity(unsigned int irq, cpumask_t cpumask) | |||
| 89 | set_balance_irq_affinity(irq, cpumask); | 89 | set_balance_irq_affinity(irq, cpumask); |
| 90 | 90 | ||
| 91 | #ifdef CONFIG_GENERIC_PENDING_IRQ | 91 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
| 92 | set_pending_irq(irq, cpumask); | 92 | if (desc->status & IRQ_MOVE_PCNTXT) { |
| 93 | unsigned long flags; | ||
| 94 | |||
| 95 | spin_lock_irqsave(&desc->lock, flags); | ||
| 96 | desc->chip->set_affinity(irq, cpumask); | ||
| 97 | spin_unlock_irqrestore(&desc->lock, flags); | ||
| 98 | } else | ||
| 99 | set_pending_irq(irq, cpumask); | ||
| 93 | #else | 100 | #else |
| 94 | desc->affinity = cpumask; | 101 | desc->affinity = cpumask; |
| 95 | desc->chip->set_affinity(irq, cpumask); | 102 | desc->chip->set_affinity(irq, cpumask); |
diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 38fc10ac7541..5072cf1685a2 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c | |||
| @@ -260,7 +260,6 @@ const char *kallsyms_lookup(unsigned long addr, | |||
| 260 | /* see if it's in a module */ | 260 | /* see if it's in a module */ |
| 261 | return module_address_lookup(addr, symbolsize, offset, modname, | 261 | return module_address_lookup(addr, symbolsize, offset, modname, |
| 262 | namebuf); | 262 | namebuf); |
| 263 | return NULL; | ||
| 264 | } | 263 | } |
| 265 | 264 | ||
| 266 | int lookup_symbol_name(unsigned long addr, char *symname) | 265 | int lookup_symbol_name(unsigned long addr, char *symname) |
diff --git a/kernel/kexec.c b/kernel/kexec.c index 59f3f0df35d4..aef265325cd3 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c | |||
| @@ -753,8 +753,14 @@ static struct page *kimage_alloc_page(struct kimage *image, | |||
| 753 | *old = addr | (*old & ~PAGE_MASK); | 753 | *old = addr | (*old & ~PAGE_MASK); |
| 754 | 754 | ||
| 755 | /* The old page I have found cannot be a | 755 | /* The old page I have found cannot be a |
| 756 | * destination page, so return it. | 756 | * destination page, so return it if it's |
| 757 | * gfp_flags honor the ones passed in. | ||
| 757 | */ | 758 | */ |
| 759 | if (!(gfp_mask & __GFP_HIGHMEM) && | ||
| 760 | PageHighMem(old_page)) { | ||
| 761 | kimage_free_pages(old_page); | ||
| 762 | continue; | ||
| 763 | } | ||
| 758 | addr = old_addr; | 764 | addr = old_addr; |
| 759 | page = old_page; | 765 | page = old_page; |
| 760 | break; | 766 | break; |
diff --git a/kernel/kgdb.c b/kernel/kgdb.c index eaa21fc9ad1d..e4dcfb2272a4 100644 --- a/kernel/kgdb.c +++ b/kernel/kgdb.c | |||
| @@ -488,7 +488,7 @@ static int write_mem_msg(int binary) | |||
| 488 | if (err) | 488 | if (err) |
| 489 | return err; | 489 | return err; |
| 490 | if (CACHE_FLUSH_IS_SAFE) | 490 | if (CACHE_FLUSH_IS_SAFE) |
| 491 | flush_icache_range(addr, addr + length + 1); | 491 | flush_icache_range(addr, addr + length); |
| 492 | return 0; | 492 | return 0; |
| 493 | } | 493 | } |
| 494 | 494 | ||
| @@ -590,6 +590,7 @@ static void kgdb_wait(struct pt_regs *regs) | |||
| 590 | 590 | ||
| 591 | /* Signal the primary CPU that we are done: */ | 591 | /* Signal the primary CPU that we are done: */ |
| 592 | atomic_set(&cpu_in_kgdb[cpu], 0); | 592 | atomic_set(&cpu_in_kgdb[cpu], 0); |
| 593 | touch_softlockup_watchdog(); | ||
| 593 | clocksource_touch_watchdog(); | 594 | clocksource_touch_watchdog(); |
| 594 | local_irq_restore(flags); | 595 | local_irq_restore(flags); |
| 595 | } | 596 | } |
| @@ -1432,6 +1433,7 @@ acquirelock: | |||
| 1432 | atomic_read(&kgdb_cpu_doing_single_step) != cpu) { | 1433 | atomic_read(&kgdb_cpu_doing_single_step) != cpu) { |
| 1433 | 1434 | ||
| 1434 | atomic_set(&kgdb_active, -1); | 1435 | atomic_set(&kgdb_active, -1); |
| 1436 | touch_softlockup_watchdog(); | ||
| 1435 | clocksource_touch_watchdog(); | 1437 | clocksource_touch_watchdog(); |
| 1436 | local_irq_restore(flags); | 1438 | local_irq_restore(flags); |
| 1437 | 1439 | ||
| @@ -1462,7 +1464,7 @@ acquirelock: | |||
| 1462 | * Get the passive CPU lock which will hold all the non-primary | 1464 | * Get the passive CPU lock which will hold all the non-primary |
| 1463 | * CPU in a spin state while the debugger is active | 1465 | * CPU in a spin state while the debugger is active |
| 1464 | */ | 1466 | */ |
| 1465 | if (!kgdb_single_step || !kgdb_contthread) { | 1467 | if (!kgdb_single_step) { |
| 1466 | for (i = 0; i < NR_CPUS; i++) | 1468 | for (i = 0; i < NR_CPUS; i++) |
| 1467 | atomic_set(&passive_cpu_wait[i], 1); | 1469 | atomic_set(&passive_cpu_wait[i], 1); |
| 1468 | } | 1470 | } |
| @@ -1475,7 +1477,7 @@ acquirelock: | |||
| 1475 | 1477 | ||
| 1476 | #ifdef CONFIG_SMP | 1478 | #ifdef CONFIG_SMP |
| 1477 | /* Signal the other CPUs to enter kgdb_wait() */ | 1479 | /* Signal the other CPUs to enter kgdb_wait() */ |
| 1478 | if ((!kgdb_single_step || !kgdb_contthread) && kgdb_do_roundup) | 1480 | if ((!kgdb_single_step) && kgdb_do_roundup) |
| 1479 | kgdb_roundup_cpus(flags); | 1481 | kgdb_roundup_cpus(flags); |
| 1480 | #endif | 1482 | #endif |
| 1481 | 1483 | ||
| @@ -1494,7 +1496,7 @@ acquirelock: | |||
| 1494 | kgdb_post_primary_code(ks->linux_regs, ks->ex_vector, ks->err_code); | 1496 | kgdb_post_primary_code(ks->linux_regs, ks->ex_vector, ks->err_code); |
| 1495 | kgdb_deactivate_sw_breakpoints(); | 1497 | kgdb_deactivate_sw_breakpoints(); |
| 1496 | kgdb_single_step = 0; | 1498 | kgdb_single_step = 0; |
| 1497 | kgdb_contthread = NULL; | 1499 | kgdb_contthread = current; |
| 1498 | exception_level = 0; | 1500 | exception_level = 0; |
| 1499 | 1501 | ||
| 1500 | /* Talk to debugger with gdbserial protocol */ | 1502 | /* Talk to debugger with gdbserial protocol */ |
| @@ -1508,7 +1510,7 @@ acquirelock: | |||
| 1508 | kgdb_info[ks->cpu].task = NULL; | 1510 | kgdb_info[ks->cpu].task = NULL; |
| 1509 | atomic_set(&cpu_in_kgdb[ks->cpu], 0); | 1511 | atomic_set(&cpu_in_kgdb[ks->cpu], 0); |
| 1510 | 1512 | ||
| 1511 | if (!kgdb_single_step || !kgdb_contthread) { | 1513 | if (!kgdb_single_step) { |
| 1512 | for (i = NR_CPUS-1; i >= 0; i--) | 1514 | for (i = NR_CPUS-1; i >= 0; i--) |
| 1513 | atomic_set(&passive_cpu_wait[i], 0); | 1515 | atomic_set(&passive_cpu_wait[i], 0); |
| 1514 | /* | 1516 | /* |
| @@ -1524,6 +1526,7 @@ acquirelock: | |||
| 1524 | kgdb_restore: | 1526 | kgdb_restore: |
| 1525 | /* Free kgdb_active */ | 1527 | /* Free kgdb_active */ |
| 1526 | atomic_set(&kgdb_active, -1); | 1528 | atomic_set(&kgdb_active, -1); |
| 1529 | touch_softlockup_watchdog(); | ||
| 1527 | clocksource_touch_watchdog(); | 1530 | clocksource_touch_watchdog(); |
| 1528 | local_irq_restore(flags); | 1531 | local_irq_restore(flags); |
| 1529 | 1532 | ||
diff --git a/kernel/kmod.c b/kernel/kmod.c index 2456d1a0befb..3d3c3ea3a023 100644 --- a/kernel/kmod.c +++ b/kernel/kmod.c | |||
| @@ -113,7 +113,7 @@ int request_module(const char *fmt, ...) | |||
| 113 | return ret; | 113 | return ret; |
| 114 | } | 114 | } |
| 115 | EXPORT_SYMBOL(request_module); | 115 | EXPORT_SYMBOL(request_module); |
| 116 | #endif /* CONFIG_KMOD */ | 116 | #endif /* CONFIG_MODULES */ |
| 117 | 117 | ||
| 118 | struct subprocess_info { | 118 | struct subprocess_info { |
| 119 | struct work_struct work; | 119 | struct work_struct work; |
| @@ -265,7 +265,7 @@ static void __call_usermodehelper(struct work_struct *work) | |||
| 265 | } | 265 | } |
| 266 | } | 266 | } |
| 267 | 267 | ||
| 268 | #ifdef CONFIG_PM | 268 | #ifdef CONFIG_PM_SLEEP |
| 269 | /* | 269 | /* |
| 270 | * If set, call_usermodehelper_exec() will exit immediately returning -EBUSY | 270 | * If set, call_usermodehelper_exec() will exit immediately returning -EBUSY |
| 271 | * (used for preventing user land processes from being created after the user | 271 | * (used for preventing user land processes from being created after the user |
| @@ -288,39 +288,37 @@ static DECLARE_WAIT_QUEUE_HEAD(running_helpers_waitq); | |||
| 288 | */ | 288 | */ |
| 289 | #define RUNNING_HELPERS_TIMEOUT (5 * HZ) | 289 | #define RUNNING_HELPERS_TIMEOUT (5 * HZ) |
| 290 | 290 | ||
| 291 | static int usermodehelper_pm_callback(struct notifier_block *nfb, | 291 | /** |
| 292 | unsigned long action, | 292 | * usermodehelper_disable - prevent new helpers from being started |
| 293 | void *ignored) | 293 | */ |
| 294 | int usermodehelper_disable(void) | ||
| 294 | { | 295 | { |
| 295 | long retval; | 296 | long retval; |
| 296 | 297 | ||
| 297 | switch (action) { | 298 | usermodehelper_disabled = 1; |
| 298 | case PM_HIBERNATION_PREPARE: | 299 | smp_mb(); |
| 299 | case PM_SUSPEND_PREPARE: | 300 | /* |
| 300 | usermodehelper_disabled = 1; | 301 | * From now on call_usermodehelper_exec() won't start any new |
| 301 | smp_mb(); | 302 | * helpers, so it is sufficient if running_helpers turns out to |
| 302 | /* | 303 | * be zero at one point (it may be increased later, but that |
| 303 | * From now on call_usermodehelper_exec() won't start any new | 304 | * doesn't matter). |
| 304 | * helpers, so it is sufficient if running_helpers turns out to | 305 | */ |
| 305 | * be zero at one point (it may be increased later, but that | 306 | retval = wait_event_timeout(running_helpers_waitq, |
| 306 | * doesn't matter). | ||
| 307 | */ | ||
| 308 | retval = wait_event_timeout(running_helpers_waitq, | ||
| 309 | atomic_read(&running_helpers) == 0, | 307 | atomic_read(&running_helpers) == 0, |
| 310 | RUNNING_HELPERS_TIMEOUT); | 308 | RUNNING_HELPERS_TIMEOUT); |
| 311 | if (retval) { | 309 | if (retval) |
| 312 | return NOTIFY_OK; | 310 | return 0; |
| 313 | } else { | ||
| 314 | usermodehelper_disabled = 0; | ||
| 315 | return NOTIFY_BAD; | ||
| 316 | } | ||
| 317 | case PM_POST_HIBERNATION: | ||
| 318 | case PM_POST_SUSPEND: | ||
| 319 | usermodehelper_disabled = 0; | ||
| 320 | return NOTIFY_OK; | ||
| 321 | } | ||
| 322 | 311 | ||
| 323 | return NOTIFY_DONE; | 312 | usermodehelper_disabled = 0; |
| 313 | return -EAGAIN; | ||
| 314 | } | ||
| 315 | |||
| 316 | /** | ||
| 317 | * usermodehelper_enable - allow new helpers to be started again | ||
| 318 | */ | ||
| 319 | void usermodehelper_enable(void) | ||
| 320 | { | ||
| 321 | usermodehelper_disabled = 0; | ||
| 324 | } | 322 | } |
| 325 | 323 | ||
| 326 | static void helper_lock(void) | 324 | static void helper_lock(void) |
| @@ -334,18 +332,12 @@ static void helper_unlock(void) | |||
| 334 | if (atomic_dec_and_test(&running_helpers)) | 332 | if (atomic_dec_and_test(&running_helpers)) |
| 335 | wake_up(&running_helpers_waitq); | 333 | wake_up(&running_helpers_waitq); |
| 336 | } | 334 | } |
| 337 | 335 | #else /* CONFIG_PM_SLEEP */ | |
| 338 | static void register_pm_notifier_callback(void) | ||
| 339 | { | ||
| 340 | pm_notifier(usermodehelper_pm_callback, 0); | ||
| 341 | } | ||
| 342 | #else /* CONFIG_PM */ | ||
| 343 | #define usermodehelper_disabled 0 | 336 | #define usermodehelper_disabled 0 |
| 344 | 337 | ||
| 345 | static inline void helper_lock(void) {} | 338 | static inline void helper_lock(void) {} |
| 346 | static inline void helper_unlock(void) {} | 339 | static inline void helper_unlock(void) {} |
| 347 | static inline void register_pm_notifier_callback(void) {} | 340 | #endif /* CONFIG_PM_SLEEP */ |
| 348 | #endif /* CONFIG_PM */ | ||
| 349 | 341 | ||
| 350 | /** | 342 | /** |
| 351 | * call_usermodehelper_setup - prepare to call a usermode helper | 343 | * call_usermodehelper_setup - prepare to call a usermode helper |
| @@ -515,5 +507,4 @@ void __init usermodehelper_init(void) | |||
| 515 | { | 507 | { |
| 516 | khelper_wq = create_singlethread_workqueue("khelper"); | 508 | khelper_wq = create_singlethread_workqueue("khelper"); |
| 517 | BUG_ON(!khelper_wq); | 509 | BUG_ON(!khelper_wq); |
| 518 | register_pm_notifier_callback(); | ||
| 519 | } | 510 | } |
diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 75bc2cd9ebc6..8b57a2597f21 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c | |||
| @@ -404,7 +404,7 @@ void kretprobe_hash_lock(struct task_struct *tsk, | |||
| 404 | spin_lock_irqsave(hlist_lock, *flags); | 404 | spin_lock_irqsave(hlist_lock, *flags); |
| 405 | } | 405 | } |
| 406 | 406 | ||
| 407 | void kretprobe_table_lock(unsigned long hash, unsigned long *flags) | 407 | static void kretprobe_table_lock(unsigned long hash, unsigned long *flags) |
| 408 | { | 408 | { |
| 409 | spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash); | 409 | spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash); |
| 410 | spin_lock_irqsave(hlist_lock, *flags); | 410 | spin_lock_irqsave(hlist_lock, *flags); |
diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c index e53bc30e9ba5..08dd8ed86c77 100644 --- a/kernel/ksysfs.c +++ b/kernel/ksysfs.c | |||
| @@ -14,6 +14,7 @@ | |||
| 14 | #include <linux/module.h> | 14 | #include <linux/module.h> |
| 15 | #include <linux/init.h> | 15 | #include <linux/init.h> |
| 16 | #include <linux/kexec.h> | 16 | #include <linux/kexec.h> |
| 17 | #include <linux/profile.h> | ||
| 17 | #include <linux/sched.h> | 18 | #include <linux/sched.h> |
| 18 | 19 | ||
| 19 | #define KERNEL_ATTR_RO(_name) \ | 20 | #define KERNEL_ATTR_RO(_name) \ |
| @@ -53,6 +54,37 @@ static ssize_t uevent_helper_store(struct kobject *kobj, | |||
| 53 | KERNEL_ATTR_RW(uevent_helper); | 54 | KERNEL_ATTR_RW(uevent_helper); |
| 54 | #endif | 55 | #endif |
| 55 | 56 | ||
| 57 | #ifdef CONFIG_PROFILING | ||
| 58 | static ssize_t profiling_show(struct kobject *kobj, | ||
| 59 | struct kobj_attribute *attr, char *buf) | ||
| 60 | { | ||
| 61 | return sprintf(buf, "%d\n", prof_on); | ||
| 62 | } | ||
| 63 | static ssize_t profiling_store(struct kobject *kobj, | ||
| 64 | struct kobj_attribute *attr, | ||
| 65 | const char *buf, size_t count) | ||
| 66 | { | ||
| 67 | int ret; | ||
| 68 | |||
| 69 | if (prof_on) | ||
| 70 | return -EEXIST; | ||
| 71 | /* | ||
| 72 | * This eventually calls into get_option() which | ||
| 73 | * has a ton of callers and is not const. It is | ||
| 74 | * easiest to cast it away here. | ||
| 75 | */ | ||
| 76 | profile_setup((char *)buf); | ||
| 77 | ret = profile_init(); | ||
| 78 | if (ret) | ||
| 79 | return ret; | ||
| 80 | ret = create_proc_profile(); | ||
| 81 | if (ret) | ||
| 82 | return ret; | ||
| 83 | return count; | ||
| 84 | } | ||
| 85 | KERNEL_ATTR_RW(profiling); | ||
| 86 | #endif | ||
| 87 | |||
| 56 | #ifdef CONFIG_KEXEC | 88 | #ifdef CONFIG_KEXEC |
| 57 | static ssize_t kexec_loaded_show(struct kobject *kobj, | 89 | static ssize_t kexec_loaded_show(struct kobject *kobj, |
| 58 | struct kobj_attribute *attr, char *buf) | 90 | struct kobj_attribute *attr, char *buf) |
| @@ -109,6 +141,9 @@ static struct attribute * kernel_attrs[] = { | |||
| 109 | &uevent_seqnum_attr.attr, | 141 | &uevent_seqnum_attr.attr, |
| 110 | &uevent_helper_attr.attr, | 142 | &uevent_helper_attr.attr, |
| 111 | #endif | 143 | #endif |
| 144 | #ifdef CONFIG_PROFILING | ||
| 145 | &profiling_attr.attr, | ||
| 146 | #endif | ||
| 112 | #ifdef CONFIG_KEXEC | 147 | #ifdef CONFIG_KEXEC |
| 113 | &kexec_loaded_attr.attr, | 148 | &kexec_loaded_attr.attr, |
| 114 | &kexec_crash_loaded_attr.attr, | 149 | &kexec_crash_loaded_attr.attr, |
diff --git a/kernel/module.c b/kernel/module.c index 9db11911e04b..b7205f67cfaf 100644 --- a/kernel/module.c +++ b/kernel/module.c | |||
| @@ -100,7 +100,7 @@ static inline int strong_try_module_get(struct module *mod) | |||
| 100 | static inline void add_taint_module(struct module *mod, unsigned flag) | 100 | static inline void add_taint_module(struct module *mod, unsigned flag) |
| 101 | { | 101 | { |
| 102 | add_taint(flag); | 102 | add_taint(flag); |
| 103 | mod->taints |= flag; | 103 | mod->taints |= (1U << flag); |
| 104 | } | 104 | } |
| 105 | 105 | ||
| 106 | /* | 106 | /* |
| @@ -784,6 +784,7 @@ sys_delete_module(const char __user *name_user, unsigned int flags) | |||
| 784 | mutex_lock(&module_mutex); | 784 | mutex_lock(&module_mutex); |
| 785 | /* Store the name of the last unloaded module for diagnostic purposes */ | 785 | /* Store the name of the last unloaded module for diagnostic purposes */ |
| 786 | strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module)); | 786 | strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module)); |
| 787 | unregister_dynamic_debug_module(mod->name); | ||
| 787 | free_module(mod); | 788 | free_module(mod); |
| 788 | 789 | ||
| 789 | out: | 790 | out: |
| @@ -923,7 +924,7 @@ static const char vermagic[] = VERMAGIC_STRING; | |||
| 923 | static int try_to_force_load(struct module *mod, const char *symname) | 924 | static int try_to_force_load(struct module *mod, const char *symname) |
| 924 | { | 925 | { |
| 925 | #ifdef CONFIG_MODULE_FORCE_LOAD | 926 | #ifdef CONFIG_MODULE_FORCE_LOAD |
| 926 | if (!(tainted & TAINT_FORCED_MODULE)) | 927 | if (!test_taint(TAINT_FORCED_MODULE)) |
| 927 | printk("%s: no version for \"%s\" found: kernel tainted.\n", | 928 | printk("%s: no version for \"%s\" found: kernel tainted.\n", |
| 928 | mod->name, symname); | 929 | mod->name, symname); |
| 929 | add_taint_module(mod, TAINT_FORCED_MODULE); | 930 | add_taint_module(mod, TAINT_FORCED_MODULE); |
| @@ -1033,7 +1034,7 @@ static unsigned long resolve_symbol(Elf_Shdr *sechdrs, | |||
| 1033 | const unsigned long *crc; | 1034 | const unsigned long *crc; |
| 1034 | 1035 | ||
| 1035 | ret = find_symbol(name, &owner, &crc, | 1036 | ret = find_symbol(name, &owner, &crc, |
| 1036 | !(mod->taints & TAINT_PROPRIETARY_MODULE), true); | 1037 | !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), true); |
| 1037 | if (!IS_ERR_VALUE(ret)) { | 1038 | if (!IS_ERR_VALUE(ret)) { |
| 1038 | /* use_module can fail due to OOM, | 1039 | /* use_module can fail due to OOM, |
| 1039 | or module initialization or unloading */ | 1040 | or module initialization or unloading */ |
| @@ -1173,7 +1174,7 @@ static void free_notes_attrs(struct module_notes_attrs *notes_attrs, | |||
| 1173 | while (i-- > 0) | 1174 | while (i-- > 0) |
| 1174 | sysfs_remove_bin_file(notes_attrs->dir, | 1175 | sysfs_remove_bin_file(notes_attrs->dir, |
| 1175 | ¬es_attrs->attrs[i]); | 1176 | ¬es_attrs->attrs[i]); |
| 1176 | kobject_del(notes_attrs->dir); | 1177 | kobject_put(notes_attrs->dir); |
| 1177 | } | 1178 | } |
| 1178 | kfree(notes_attrs); | 1179 | kfree(notes_attrs); |
| 1179 | } | 1180 | } |
| @@ -1634,7 +1635,7 @@ static void set_license(struct module *mod, const char *license) | |||
| 1634 | license = "unspecified"; | 1635 | license = "unspecified"; |
| 1635 | 1636 | ||
| 1636 | if (!license_is_gpl_compatible(license)) { | 1637 | if (!license_is_gpl_compatible(license)) { |
| 1637 | if (!(tainted & TAINT_PROPRIETARY_MODULE)) | 1638 | if (!test_taint(TAINT_PROPRIETARY_MODULE)) |
| 1638 | printk(KERN_WARNING "%s: module license '%s' taints " | 1639 | printk(KERN_WARNING "%s: module license '%s' taints " |
| 1639 | "kernel.\n", mod->name, license); | 1640 | "kernel.\n", mod->name, license); |
| 1640 | add_taint_module(mod, TAINT_PROPRIETARY_MODULE); | 1641 | add_taint_module(mod, TAINT_PROPRIETARY_MODULE); |
| @@ -1783,6 +1784,33 @@ static inline void add_kallsyms(struct module *mod, | |||
| 1783 | } | 1784 | } |
| 1784 | #endif /* CONFIG_KALLSYMS */ | 1785 | #endif /* CONFIG_KALLSYMS */ |
| 1785 | 1786 | ||
| 1787 | #ifdef CONFIG_DYNAMIC_PRINTK_DEBUG | ||
| 1788 | static void dynamic_printk_setup(Elf_Shdr *sechdrs, unsigned int verboseindex) | ||
| 1789 | { | ||
| 1790 | struct mod_debug *debug_info; | ||
| 1791 | unsigned long pos, end; | ||
| 1792 | unsigned int num_verbose; | ||
| 1793 | |||
| 1794 | pos = sechdrs[verboseindex].sh_addr; | ||
| 1795 | num_verbose = sechdrs[verboseindex].sh_size / | ||
| 1796 | sizeof(struct mod_debug); | ||
| 1797 | end = pos + (num_verbose * sizeof(struct mod_debug)); | ||
| 1798 | |||
| 1799 | for (; pos < end; pos += sizeof(struct mod_debug)) { | ||
| 1800 | debug_info = (struct mod_debug *)pos; | ||
| 1801 | register_dynamic_debug_module(debug_info->modname, | ||
| 1802 | debug_info->type, debug_info->logical_modname, | ||
| 1803 | debug_info->flag_names, debug_info->hash, | ||
| 1804 | debug_info->hash2); | ||
| 1805 | } | ||
| 1806 | } | ||
| 1807 | #else | ||
| 1808 | static inline void dynamic_printk_setup(Elf_Shdr *sechdrs, | ||
| 1809 | unsigned int verboseindex) | ||
| 1810 | { | ||
| 1811 | } | ||
| 1812 | #endif /* CONFIG_DYNAMIC_PRINTK_DEBUG */ | ||
| 1813 | |||
| 1786 | static void *module_alloc_update_bounds(unsigned long size) | 1814 | static void *module_alloc_update_bounds(unsigned long size) |
| 1787 | { | 1815 | { |
| 1788 | void *ret = module_alloc(size); | 1816 | void *ret = module_alloc(size); |
| @@ -1831,6 +1859,7 @@ static noinline struct module *load_module(void __user *umod, | |||
| 1831 | #endif | 1859 | #endif |
| 1832 | unsigned int markersindex; | 1860 | unsigned int markersindex; |
| 1833 | unsigned int markersstringsindex; | 1861 | unsigned int markersstringsindex; |
| 1862 | unsigned int verboseindex; | ||
| 1834 | struct module *mod; | 1863 | struct module *mod; |
| 1835 | long err = 0; | 1864 | long err = 0; |
| 1836 | void *percpu = NULL, *ptr = NULL; /* Stops spurious gcc warning */ | 1865 | void *percpu = NULL, *ptr = NULL; /* Stops spurious gcc warning */ |
| @@ -2117,6 +2146,7 @@ static noinline struct module *load_module(void __user *umod, | |||
| 2117 | markersindex = find_sec(hdr, sechdrs, secstrings, "__markers"); | 2146 | markersindex = find_sec(hdr, sechdrs, secstrings, "__markers"); |
| 2118 | markersstringsindex = find_sec(hdr, sechdrs, secstrings, | 2147 | markersstringsindex = find_sec(hdr, sechdrs, secstrings, |
| 2119 | "__markers_strings"); | 2148 | "__markers_strings"); |
| 2149 | verboseindex = find_sec(hdr, sechdrs, secstrings, "__verbose"); | ||
| 2120 | 2150 | ||
| 2121 | /* Now do relocations. */ | 2151 | /* Now do relocations. */ |
| 2122 | for (i = 1; i < hdr->e_shnum; i++) { | 2152 | for (i = 1; i < hdr->e_shnum; i++) { |
| @@ -2167,6 +2197,7 @@ static noinline struct module *load_module(void __user *umod, | |||
| 2167 | marker_update_probe_range(mod->markers, | 2197 | marker_update_probe_range(mod->markers, |
| 2168 | mod->markers + mod->num_markers); | 2198 | mod->markers + mod->num_markers); |
| 2169 | #endif | 2199 | #endif |
| 2200 | dynamic_printk_setup(sechdrs, verboseindex); | ||
| 2170 | err = module_finalize(hdr, sechdrs, mod); | 2201 | err = module_finalize(hdr, sechdrs, mod); |
| 2171 | if (err < 0) | 2202 | if (err < 0) |
| 2172 | goto cleanup; | 2203 | goto cleanup; |
| @@ -2552,9 +2583,9 @@ static char *module_flags(struct module *mod, char *buf) | |||
| 2552 | mod->state == MODULE_STATE_GOING || | 2583 | mod->state == MODULE_STATE_GOING || |
| 2553 | mod->state == MODULE_STATE_COMING) { | 2584 | mod->state == MODULE_STATE_COMING) { |
| 2554 | buf[bx++] = '('; | 2585 | buf[bx++] = '('; |
| 2555 | if (mod->taints & TAINT_PROPRIETARY_MODULE) | 2586 | if (mod->taints & (1 << TAINT_PROPRIETARY_MODULE)) |
| 2556 | buf[bx++] = 'P'; | 2587 | buf[bx++] = 'P'; |
| 2557 | if (mod->taints & TAINT_FORCED_MODULE) | 2588 | if (mod->taints & (1 << TAINT_FORCED_MODULE)) |
| 2558 | buf[bx++] = 'F'; | 2589 | buf[bx++] = 'F'; |
| 2559 | /* | 2590 | /* |
| 2560 | * TAINT_FORCED_RMMOD: could be added. | 2591 | * TAINT_FORCED_RMMOD: could be added. |
diff --git a/kernel/panic.c b/kernel/panic.c index 12c5a0a6c89b..f290e8e866f6 100644 --- a/kernel/panic.c +++ b/kernel/panic.c | |||
| @@ -23,7 +23,7 @@ | |||
| 23 | #include <linux/kallsyms.h> | 23 | #include <linux/kallsyms.h> |
| 24 | 24 | ||
| 25 | int panic_on_oops; | 25 | int panic_on_oops; |
| 26 | int tainted; | 26 | static unsigned long tainted_mask; |
| 27 | static int pause_on_oops; | 27 | static int pause_on_oops; |
| 28 | static int pause_on_oops_flag; | 28 | static int pause_on_oops_flag; |
| 29 | static DEFINE_SPINLOCK(pause_on_oops_lock); | 29 | static DEFINE_SPINLOCK(pause_on_oops_lock); |
| @@ -143,6 +143,26 @@ NORET_TYPE void panic(const char * fmt, ...) | |||
| 143 | 143 | ||
| 144 | EXPORT_SYMBOL(panic); | 144 | EXPORT_SYMBOL(panic); |
| 145 | 145 | ||
| 146 | |||
| 147 | struct tnt { | ||
| 148 | u8 bit; | ||
| 149 | char true; | ||
| 150 | char false; | ||
| 151 | }; | ||
| 152 | |||
| 153 | static const struct tnt tnts[] = { | ||
| 154 | { TAINT_PROPRIETARY_MODULE, 'P', 'G' }, | ||
| 155 | { TAINT_FORCED_MODULE, 'F', ' ' }, | ||
| 156 | { TAINT_UNSAFE_SMP, 'S', ' ' }, | ||
| 157 | { TAINT_FORCED_RMMOD, 'R', ' ' }, | ||
| 158 | { TAINT_MACHINE_CHECK, 'M', ' ' }, | ||
| 159 | { TAINT_BAD_PAGE, 'B', ' ' }, | ||
| 160 | { TAINT_USER, 'U', ' ' }, | ||
| 161 | { TAINT_DIE, 'D', ' ' }, | ||
| 162 | { TAINT_OVERRIDDEN_ACPI_TABLE, 'A', ' ' }, | ||
| 163 | { TAINT_WARN, 'W', ' ' }, | ||
| 164 | }; | ||
| 165 | |||
| 146 | /** | 166 | /** |
| 147 | * print_tainted - return a string to represent the kernel taint state. | 167 | * print_tainted - return a string to represent the kernel taint state. |
| 148 | * | 168 | * |
| @@ -158,32 +178,41 @@ EXPORT_SYMBOL(panic); | |||
| 158 | * | 178 | * |
| 159 | * The string is overwritten by the next call to print_taint(). | 179 | * The string is overwritten by the next call to print_taint(). |
| 160 | */ | 180 | */ |
| 161 | |||
| 162 | const char *print_tainted(void) | 181 | const char *print_tainted(void) |
| 163 | { | 182 | { |
| 164 | static char buf[20]; | 183 | static char buf[ARRAY_SIZE(tnts) + sizeof("Tainted: ") + 1]; |
| 165 | if (tainted) { | 184 | |
| 166 | snprintf(buf, sizeof(buf), "Tainted: %c%c%c%c%c%c%c%c%c%c", | 185 | if (tainted_mask) { |
| 167 | tainted & TAINT_PROPRIETARY_MODULE ? 'P' : 'G', | 186 | char *s; |
| 168 | tainted & TAINT_FORCED_MODULE ? 'F' : ' ', | 187 | int i; |
| 169 | tainted & TAINT_UNSAFE_SMP ? 'S' : ' ', | 188 | |
| 170 | tainted & TAINT_FORCED_RMMOD ? 'R' : ' ', | 189 | s = buf + sprintf(buf, "Tainted: "); |
| 171 | tainted & TAINT_MACHINE_CHECK ? 'M' : ' ', | 190 | for (i = 0; i < ARRAY_SIZE(tnts); i++) { |
| 172 | tainted & TAINT_BAD_PAGE ? 'B' : ' ', | 191 | const struct tnt *t = &tnts[i]; |
| 173 | tainted & TAINT_USER ? 'U' : ' ', | 192 | *s++ = test_bit(t->bit, &tainted_mask) ? |
| 174 | tainted & TAINT_DIE ? 'D' : ' ', | 193 | t->true : t->false; |
| 175 | tainted & TAINT_OVERRIDDEN_ACPI_TABLE ? 'A' : ' ', | 194 | } |
| 176 | tainted & TAINT_WARN ? 'W' : ' '); | 195 | *s = 0; |
| 177 | } | 196 | } else |
| 178 | else | ||
| 179 | snprintf(buf, sizeof(buf), "Not tainted"); | 197 | snprintf(buf, sizeof(buf), "Not tainted"); |
| 180 | return(buf); | 198 | return(buf); |
| 181 | } | 199 | } |
| 182 | 200 | ||
| 201 | int test_taint(unsigned flag) | ||
| 202 | { | ||
| 203 | return test_bit(flag, &tainted_mask); | ||
| 204 | } | ||
| 205 | EXPORT_SYMBOL(test_taint); | ||
| 206 | |||
| 207 | unsigned long get_taint(void) | ||
| 208 | { | ||
| 209 | return tainted_mask; | ||
| 210 | } | ||
| 211 | |||
| 183 | void add_taint(unsigned flag) | 212 | void add_taint(unsigned flag) |
| 184 | { | 213 | { |
| 185 | debug_locks = 0; /* can't trust the integrity of the kernel anymore */ | 214 | debug_locks = 0; /* can't trust the integrity of the kernel anymore */ |
| 186 | tainted |= flag; | 215 | set_bit(flag, &tainted_mask); |
| 187 | } | 216 | } |
| 188 | EXPORT_SYMBOL(add_taint); | 217 | EXPORT_SYMBOL(add_taint); |
| 189 | 218 | ||
diff --git a/kernel/posix-timers.c b/kernel/posix-timers.c index f85efcdcab2d..ee204586149a 100644 --- a/kernel/posix-timers.c +++ b/kernel/posix-timers.c | |||
| @@ -441,7 +441,7 @@ static struct k_itimer * alloc_posix_timer(void) | |||
| 441 | return tmr; | 441 | return tmr; |
| 442 | if (unlikely(!(tmr->sigq = sigqueue_alloc()))) { | 442 | if (unlikely(!(tmr->sigq = sigqueue_alloc()))) { |
| 443 | kmem_cache_free(posix_timers_cache, tmr); | 443 | kmem_cache_free(posix_timers_cache, tmr); |
| 444 | tmr = NULL; | 444 | return NULL; |
| 445 | } | 445 | } |
| 446 | memset(&tmr->sigq->info, 0, sizeof(siginfo_t)); | 446 | memset(&tmr->sigq->info, 0, sizeof(siginfo_t)); |
| 447 | return tmr; | 447 | return tmr; |
diff --git a/kernel/power/disk.c b/kernel/power/disk.c index bbd85c60f741..331f9836383f 100644 --- a/kernel/power/disk.c +++ b/kernel/power/disk.c | |||
| @@ -14,6 +14,7 @@ | |||
| 14 | #include <linux/reboot.h> | 14 | #include <linux/reboot.h> |
| 15 | #include <linux/string.h> | 15 | #include <linux/string.h> |
| 16 | #include <linux/device.h> | 16 | #include <linux/device.h> |
| 17 | #include <linux/kmod.h> | ||
| 17 | #include <linux/delay.h> | 18 | #include <linux/delay.h> |
| 18 | #include <linux/fs.h> | 19 | #include <linux/fs.h> |
| 19 | #include <linux/mount.h> | 20 | #include <linux/mount.h> |
| @@ -520,6 +521,10 @@ int hibernate(void) | |||
| 520 | if (error) | 521 | if (error) |
| 521 | goto Exit; | 522 | goto Exit; |
| 522 | 523 | ||
| 524 | error = usermodehelper_disable(); | ||
| 525 | if (error) | ||
| 526 | goto Exit; | ||
| 527 | |||
| 523 | /* Allocate memory management structures */ | 528 | /* Allocate memory management structures */ |
| 524 | error = create_basic_memory_bitmaps(); | 529 | error = create_basic_memory_bitmaps(); |
| 525 | if (error) | 530 | if (error) |
| @@ -558,6 +563,7 @@ int hibernate(void) | |||
| 558 | thaw_processes(); | 563 | thaw_processes(); |
| 559 | Finish: | 564 | Finish: |
| 560 | free_basic_memory_bitmaps(); | 565 | free_basic_memory_bitmaps(); |
| 566 | usermodehelper_enable(); | ||
| 561 | Exit: | 567 | Exit: |
| 562 | pm_notifier_call_chain(PM_POST_HIBERNATION); | 568 | pm_notifier_call_chain(PM_POST_HIBERNATION); |
| 563 | pm_restore_console(); | 569 | pm_restore_console(); |
| @@ -634,6 +640,10 @@ static int software_resume(void) | |||
| 634 | if (error) | 640 | if (error) |
| 635 | goto Finish; | 641 | goto Finish; |
| 636 | 642 | ||
| 643 | error = usermodehelper_disable(); | ||
| 644 | if (error) | ||
| 645 | goto Finish; | ||
| 646 | |||
| 637 | error = create_basic_memory_bitmaps(); | 647 | error = create_basic_memory_bitmaps(); |
| 638 | if (error) | 648 | if (error) |
| 639 | goto Finish; | 649 | goto Finish; |
| @@ -656,6 +666,7 @@ static int software_resume(void) | |||
| 656 | thaw_processes(); | 666 | thaw_processes(); |
| 657 | Done: | 667 | Done: |
| 658 | free_basic_memory_bitmaps(); | 668 | free_basic_memory_bitmaps(); |
| 669 | usermodehelper_enable(); | ||
| 659 | Finish: | 670 | Finish: |
| 660 | pm_notifier_call_chain(PM_POST_RESTORE); | 671 | pm_notifier_call_chain(PM_POST_RESTORE); |
| 661 | pm_restore_console(); | 672 | pm_restore_console(); |
diff --git a/kernel/power/main.c b/kernel/power/main.c index 540b16b68565..19122cf6d827 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c | |||
| @@ -14,6 +14,7 @@ | |||
| 14 | #include <linux/string.h> | 14 | #include <linux/string.h> |
| 15 | #include <linux/delay.h> | 15 | #include <linux/delay.h> |
| 16 | #include <linux/errno.h> | 16 | #include <linux/errno.h> |
| 17 | #include <linux/kmod.h> | ||
| 17 | #include <linux/init.h> | 18 | #include <linux/init.h> |
| 18 | #include <linux/console.h> | 19 | #include <linux/console.h> |
| 19 | #include <linux/cpu.h> | 20 | #include <linux/cpu.h> |
| @@ -237,6 +238,10 @@ static int suspend_prepare(void) | |||
| 237 | if (error) | 238 | if (error) |
| 238 | goto Finish; | 239 | goto Finish; |
| 239 | 240 | ||
| 241 | error = usermodehelper_disable(); | ||
| 242 | if (error) | ||
| 243 | goto Finish; | ||
| 244 | |||
| 240 | if (suspend_freeze_processes()) { | 245 | if (suspend_freeze_processes()) { |
| 241 | error = -EAGAIN; | 246 | error = -EAGAIN; |
| 242 | goto Thaw; | 247 | goto Thaw; |
| @@ -256,6 +261,7 @@ static int suspend_prepare(void) | |||
| 256 | 261 | ||
| 257 | Thaw: | 262 | Thaw: |
| 258 | suspend_thaw_processes(); | 263 | suspend_thaw_processes(); |
| 264 | usermodehelper_enable(); | ||
| 259 | Finish: | 265 | Finish: |
| 260 | pm_notifier_call_chain(PM_POST_SUSPEND); | 266 | pm_notifier_call_chain(PM_POST_SUSPEND); |
| 261 | pm_restore_console(); | 267 | pm_restore_console(); |
| @@ -376,6 +382,7 @@ int suspend_devices_and_enter(suspend_state_t state) | |||
| 376 | static void suspend_finish(void) | 382 | static void suspend_finish(void) |
| 377 | { | 383 | { |
| 378 | suspend_thaw_processes(); | 384 | suspend_thaw_processes(); |
| 385 | usermodehelper_enable(); | ||
| 379 | pm_notifier_call_chain(PM_POST_SUSPEND); | 386 | pm_notifier_call_chain(PM_POST_SUSPEND); |
| 380 | pm_restore_console(); | 387 | pm_restore_console(); |
| 381 | } | 388 | } |
diff --git a/kernel/power/user.c b/kernel/power/user.c index a6332a313262..005b93d839ba 100644 --- a/kernel/power/user.c +++ b/kernel/power/user.c | |||
| @@ -212,13 +212,20 @@ static long snapshot_ioctl(struct file *filp, unsigned int cmd, | |||
| 212 | case SNAPSHOT_FREEZE: | 212 | case SNAPSHOT_FREEZE: |
| 213 | if (data->frozen) | 213 | if (data->frozen) |
| 214 | break; | 214 | break; |
| 215 | |||
| 215 | printk("Syncing filesystems ... "); | 216 | printk("Syncing filesystems ... "); |
| 216 | sys_sync(); | 217 | sys_sync(); |
| 217 | printk("done.\n"); | 218 | printk("done.\n"); |
| 218 | 219 | ||
| 219 | error = freeze_processes(); | 220 | error = usermodehelper_disable(); |
| 220 | if (error) | 221 | if (error) |
| 222 | break; | ||
| 223 | |||
| 224 | error = freeze_processes(); | ||
| 225 | if (error) { | ||
| 221 | thaw_processes(); | 226 | thaw_processes(); |
| 227 | usermodehelper_enable(); | ||
| 228 | } | ||
| 222 | if (!error) | 229 | if (!error) |
| 223 | data->frozen = 1; | 230 | data->frozen = 1; |
| 224 | break; | 231 | break; |
| @@ -227,6 +234,7 @@ static long snapshot_ioctl(struct file *filp, unsigned int cmd, | |||
| 227 | if (!data->frozen || data->ready) | 234 | if (!data->frozen || data->ready) |
| 228 | break; | 235 | break; |
| 229 | thaw_processes(); | 236 | thaw_processes(); |
| 237 | usermodehelper_enable(); | ||
| 230 | data->frozen = 0; | 238 | data->frozen = 0; |
| 231 | break; | 239 | break; |
| 232 | 240 | ||
diff --git a/kernel/printk.c b/kernel/printk.c index b51b1567bb55..6341af77eb65 100644 --- a/kernel/printk.c +++ b/kernel/printk.c | |||
| @@ -13,7 +13,7 @@ | |||
| 13 | * Fixed SMP synchronization, 08/08/99, Manfred Spraul | 13 | * Fixed SMP synchronization, 08/08/99, Manfred Spraul |
| 14 | * manfred@colorfullife.com | 14 | * manfred@colorfullife.com |
| 15 | * Rewrote bits to get rid of console_lock | 15 | * Rewrote bits to get rid of console_lock |
| 16 | * 01Mar01 Andrew Morton <andrewm@uow.edu.au> | 16 | * 01Mar01 Andrew Morton |
| 17 | */ | 17 | */ |
| 18 | 18 | ||
| 19 | #include <linux/kernel.h> | 19 | #include <linux/kernel.h> |
| @@ -577,9 +577,6 @@ static int have_callable_console(void) | |||
| 577 | * @fmt: format string | 577 | * @fmt: format string |
| 578 | * | 578 | * |
| 579 | * This is printk(). It can be called from any context. We want it to work. | 579 | * This is printk(). It can be called from any context. We want it to work. |
| 580 | * Be aware of the fact that if oops_in_progress is not set, we might try to | ||
| 581 | * wake klogd up which could deadlock on runqueue lock if printk() is called | ||
| 582 | * from scheduler code. | ||
| 583 | * | 580 | * |
| 584 | * We try to grab the console_sem. If we succeed, it's easy - we log the output and | 581 | * We try to grab the console_sem. If we succeed, it's easy - we log the output and |
| 585 | * call the console drivers. If we fail to get the semaphore we place the output | 582 | * call the console drivers. If we fail to get the semaphore we place the output |
| @@ -593,6 +590,8 @@ static int have_callable_console(void) | |||
| 593 | * | 590 | * |
| 594 | * See also: | 591 | * See also: |
| 595 | * printf(3) | 592 | * printf(3) |
| 593 | * | ||
| 594 | * See the vsnprintf() documentation for format string extensions over C99. | ||
| 596 | */ | 595 | */ |
| 597 | 596 | ||
| 598 | asmlinkage int printk(const char *fmt, ...) | 597 | asmlinkage int printk(const char *fmt, ...) |
| @@ -982,10 +981,25 @@ int is_console_locked(void) | |||
| 982 | return console_locked; | 981 | return console_locked; |
| 983 | } | 982 | } |
| 984 | 983 | ||
| 985 | void wake_up_klogd(void) | 984 | static DEFINE_PER_CPU(int, printk_pending); |
| 985 | |||
| 986 | void printk_tick(void) | ||
| 986 | { | 987 | { |
| 987 | if (!oops_in_progress && waitqueue_active(&log_wait)) | 988 | if (__get_cpu_var(printk_pending)) { |
| 989 | __get_cpu_var(printk_pending) = 0; | ||
| 988 | wake_up_interruptible(&log_wait); | 990 | wake_up_interruptible(&log_wait); |
| 991 | } | ||
| 992 | } | ||
| 993 | |||
| 994 | int printk_needs_cpu(int cpu) | ||
| 995 | { | ||
| 996 | return per_cpu(printk_pending, cpu); | ||
| 997 | } | ||
| 998 | |||
| 999 | void wake_up_klogd(void) | ||
| 1000 | { | ||
| 1001 | if (waitqueue_active(&log_wait)) | ||
| 1002 | __raw_get_cpu_var(printk_pending) = 1; | ||
| 989 | } | 1003 | } |
| 990 | 1004 | ||
| 991 | /** | 1005 | /** |
| @@ -1291,22 +1305,6 @@ static int __init disable_boot_consoles(void) | |||
| 1291 | } | 1305 | } |
| 1292 | late_initcall(disable_boot_consoles); | 1306 | late_initcall(disable_boot_consoles); |
| 1293 | 1307 | ||
| 1294 | /** | ||
| 1295 | * tty_write_message - write a message to a certain tty, not just the console. | ||
| 1296 | * @tty: the destination tty_struct | ||
| 1297 | * @msg: the message to write | ||
| 1298 | * | ||
| 1299 | * This is used for messages that need to be redirected to a specific tty. | ||
| 1300 | * We don't put it into the syslog queue right now maybe in the future if | ||
| 1301 | * really needed. | ||
| 1302 | */ | ||
| 1303 | void tty_write_message(struct tty_struct *tty, char *msg) | ||
| 1304 | { | ||
| 1305 | if (tty && tty->ops->write) | ||
| 1306 | tty->ops->write(tty, msg, strlen(msg)); | ||
| 1307 | return; | ||
| 1308 | } | ||
| 1309 | |||
| 1310 | #if defined CONFIG_PRINTK | 1308 | #if defined CONFIG_PRINTK |
| 1311 | 1309 | ||
| 1312 | /* | 1310 | /* |
diff --git a/kernel/profile.c b/kernel/profile.c index cd26bed4cc26..a9e422df6bf6 100644 --- a/kernel/profile.c +++ b/kernel/profile.c | |||
| @@ -22,6 +22,8 @@ | |||
| 22 | #include <linux/cpu.h> | 22 | #include <linux/cpu.h> |
| 23 | #include <linux/highmem.h> | 23 | #include <linux/highmem.h> |
| 24 | #include <linux/mutex.h> | 24 | #include <linux/mutex.h> |
| 25 | #include <linux/slab.h> | ||
| 26 | #include <linux/vmalloc.h> | ||
| 25 | #include <asm/sections.h> | 27 | #include <asm/sections.h> |
| 26 | #include <asm/irq_regs.h> | 28 | #include <asm/irq_regs.h> |
| 27 | #include <asm/ptrace.h> | 29 | #include <asm/ptrace.h> |
| @@ -50,11 +52,11 @@ static DEFINE_PER_CPU(int, cpu_profile_flip); | |||
| 50 | static DEFINE_MUTEX(profile_flip_mutex); | 52 | static DEFINE_MUTEX(profile_flip_mutex); |
| 51 | #endif /* CONFIG_SMP */ | 53 | #endif /* CONFIG_SMP */ |
| 52 | 54 | ||
| 53 | static int __init profile_setup(char *str) | 55 | int profile_setup(char *str) |
| 54 | { | 56 | { |
| 55 | static char __initdata schedstr[] = "schedule"; | 57 | static char schedstr[] = "schedule"; |
| 56 | static char __initdata sleepstr[] = "sleep"; | 58 | static char sleepstr[] = "sleep"; |
| 57 | static char __initdata kvmstr[] = "kvm"; | 59 | static char kvmstr[] = "kvm"; |
| 58 | int par; | 60 | int par; |
| 59 | 61 | ||
| 60 | if (!strncmp(str, sleepstr, strlen(sleepstr))) { | 62 | if (!strncmp(str, sleepstr, strlen(sleepstr))) { |
| @@ -100,14 +102,33 @@ static int __init profile_setup(char *str) | |||
| 100 | __setup("profile=", profile_setup); | 102 | __setup("profile=", profile_setup); |
| 101 | 103 | ||
| 102 | 104 | ||
| 103 | void __init profile_init(void) | 105 | int profile_init(void) |
| 104 | { | 106 | { |
| 107 | int buffer_bytes; | ||
| 105 | if (!prof_on) | 108 | if (!prof_on) |
| 106 | return; | 109 | return 0; |
| 107 | 110 | ||
| 108 | /* only text is profiled */ | 111 | /* only text is profiled */ |
| 109 | prof_len = (_etext - _stext) >> prof_shift; | 112 | prof_len = (_etext - _stext) >> prof_shift; |
| 110 | prof_buffer = alloc_bootmem(prof_len*sizeof(atomic_t)); | 113 | buffer_bytes = prof_len*sizeof(atomic_t); |
| 114 | if (!slab_is_available()) { | ||
| 115 | prof_buffer = alloc_bootmem(buffer_bytes); | ||
| 116 | return 0; | ||
| 117 | } | ||
| 118 | |||
| 119 | prof_buffer = kzalloc(buffer_bytes, GFP_KERNEL); | ||
| 120 | if (prof_buffer) | ||
| 121 | return 0; | ||
| 122 | |||
| 123 | prof_buffer = alloc_pages_exact(buffer_bytes, GFP_KERNEL|__GFP_ZERO); | ||
| 124 | if (prof_buffer) | ||
| 125 | return 0; | ||
| 126 | |||
| 127 | prof_buffer = vmalloc(buffer_bytes); | ||
| 128 | if (prof_buffer) | ||
| 129 | return 0; | ||
| 130 | |||
| 131 | return -ENOMEM; | ||
| 111 | } | 132 | } |
| 112 | 133 | ||
| 113 | /* Profile event notifications */ | 134 | /* Profile event notifications */ |
| @@ -527,7 +548,7 @@ static void __init profile_nop(void *unused) | |||
| 527 | { | 548 | { |
| 528 | } | 549 | } |
| 529 | 550 | ||
| 530 | static int __init create_hash_tables(void) | 551 | static int create_hash_tables(void) |
| 531 | { | 552 | { |
| 532 | int cpu; | 553 | int cpu; |
| 533 | 554 | ||
| @@ -575,14 +596,14 @@ out_cleanup: | |||
| 575 | #define create_hash_tables() ({ 0; }) | 596 | #define create_hash_tables() ({ 0; }) |
| 576 | #endif | 597 | #endif |
| 577 | 598 | ||
| 578 | static int __init create_proc_profile(void) | 599 | int create_proc_profile(void) |
| 579 | { | 600 | { |
| 580 | struct proc_dir_entry *entry; | 601 | struct proc_dir_entry *entry; |
| 581 | 602 | ||
| 582 | if (!prof_on) | 603 | if (!prof_on) |
| 583 | return 0; | 604 | return 0; |
| 584 | if (create_hash_tables()) | 605 | if (create_hash_tables()) |
| 585 | return -1; | 606 | return -ENOMEM; |
| 586 | entry = proc_create("profile", S_IWUSR | S_IRUGO, | 607 | entry = proc_create("profile", S_IWUSR | S_IRUGO, |
| 587 | NULL, &proc_profile_operations); | 608 | NULL, &proc_profile_operations); |
| 588 | if (!entry) | 609 | if (!entry) |
diff --git a/kernel/rcuclassic.c b/kernel/rcuclassic.c index aad93cdc9f68..37f72e551542 100644 --- a/kernel/rcuclassic.c +++ b/kernel/rcuclassic.c | |||
| @@ -47,6 +47,7 @@ | |||
| 47 | #include <linux/notifier.h> | 47 | #include <linux/notifier.h> |
| 48 | #include <linux/cpu.h> | 48 | #include <linux/cpu.h> |
| 49 | #include <linux/mutex.h> | 49 | #include <linux/mutex.h> |
| 50 | #include <linux/time.h> | ||
| 50 | 51 | ||
| 51 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 52 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
| 52 | static struct lock_class_key rcu_lock_key; | 53 | static struct lock_class_key rcu_lock_key; |
| @@ -60,12 +61,14 @@ EXPORT_SYMBOL_GPL(rcu_lock_map); | |||
| 60 | static struct rcu_ctrlblk rcu_ctrlblk = { | 61 | static struct rcu_ctrlblk rcu_ctrlblk = { |
| 61 | .cur = -300, | 62 | .cur = -300, |
| 62 | .completed = -300, | 63 | .completed = -300, |
| 64 | .pending = -300, | ||
| 63 | .lock = __SPIN_LOCK_UNLOCKED(&rcu_ctrlblk.lock), | 65 | .lock = __SPIN_LOCK_UNLOCKED(&rcu_ctrlblk.lock), |
| 64 | .cpumask = CPU_MASK_NONE, | 66 | .cpumask = CPU_MASK_NONE, |
| 65 | }; | 67 | }; |
| 66 | static struct rcu_ctrlblk rcu_bh_ctrlblk = { | 68 | static struct rcu_ctrlblk rcu_bh_ctrlblk = { |
| 67 | .cur = -300, | 69 | .cur = -300, |
| 68 | .completed = -300, | 70 | .completed = -300, |
| 71 | .pending = -300, | ||
| 69 | .lock = __SPIN_LOCK_UNLOCKED(&rcu_bh_ctrlblk.lock), | 72 | .lock = __SPIN_LOCK_UNLOCKED(&rcu_bh_ctrlblk.lock), |
| 70 | .cpumask = CPU_MASK_NONE, | 73 | .cpumask = CPU_MASK_NONE, |
| 71 | }; | 74 | }; |
| @@ -83,7 +86,10 @@ static void force_quiescent_state(struct rcu_data *rdp, | |||
| 83 | { | 86 | { |
| 84 | int cpu; | 87 | int cpu; |
| 85 | cpumask_t cpumask; | 88 | cpumask_t cpumask; |
| 89 | unsigned long flags; | ||
| 90 | |||
| 86 | set_need_resched(); | 91 | set_need_resched(); |
| 92 | spin_lock_irqsave(&rcp->lock, flags); | ||
| 87 | if (unlikely(!rcp->signaled)) { | 93 | if (unlikely(!rcp->signaled)) { |
| 88 | rcp->signaled = 1; | 94 | rcp->signaled = 1; |
| 89 | /* | 95 | /* |
| @@ -109,6 +115,7 @@ static void force_quiescent_state(struct rcu_data *rdp, | |||
| 109 | for_each_cpu_mask_nr(cpu, cpumask) | 115 | for_each_cpu_mask_nr(cpu, cpumask) |
| 110 | smp_send_reschedule(cpu); | 116 | smp_send_reschedule(cpu); |
| 111 | } | 117 | } |
| 118 | spin_unlock_irqrestore(&rcp->lock, flags); | ||
| 112 | } | 119 | } |
| 113 | #else | 120 | #else |
| 114 | static inline void force_quiescent_state(struct rcu_data *rdp, | 121 | static inline void force_quiescent_state(struct rcu_data *rdp, |
| @@ -118,6 +125,126 @@ static inline void force_quiescent_state(struct rcu_data *rdp, | |||
| 118 | } | 125 | } |
| 119 | #endif | 126 | #endif |
| 120 | 127 | ||
| 128 | static void __call_rcu(struct rcu_head *head, struct rcu_ctrlblk *rcp, | ||
| 129 | struct rcu_data *rdp) | ||
| 130 | { | ||
| 131 | long batch; | ||
| 132 | |||
| 133 | head->next = NULL; | ||
| 134 | smp_mb(); /* Read of rcu->cur must happen after any change by caller. */ | ||
| 135 | |||
| 136 | /* | ||
| 137 | * Determine the batch number of this callback. | ||
| 138 | * | ||
| 139 | * Using ACCESS_ONCE to avoid the following error when gcc eliminates | ||
| 140 | * local variable "batch" and emits codes like this: | ||
| 141 | * 1) rdp->batch = rcp->cur + 1 # gets old value | ||
| 142 | * ...... | ||
| 143 | * 2)rcu_batch_after(rcp->cur + 1, rdp->batch) # gets new value | ||
| 144 | * then [*nxttail[0], *nxttail[1]) may contain callbacks | ||
| 145 | * that batch# = rdp->batch, see the comment of struct rcu_data. | ||
| 146 | */ | ||
| 147 | batch = ACCESS_ONCE(rcp->cur) + 1; | ||
| 148 | |||
| 149 | if (rdp->nxtlist && rcu_batch_after(batch, rdp->batch)) { | ||
| 150 | /* process callbacks */ | ||
| 151 | rdp->nxttail[0] = rdp->nxttail[1]; | ||
| 152 | rdp->nxttail[1] = rdp->nxttail[2]; | ||
| 153 | if (rcu_batch_after(batch - 1, rdp->batch)) | ||
| 154 | rdp->nxttail[0] = rdp->nxttail[2]; | ||
| 155 | } | ||
| 156 | |||
| 157 | rdp->batch = batch; | ||
| 158 | *rdp->nxttail[2] = head; | ||
| 159 | rdp->nxttail[2] = &head->next; | ||
| 160 | |||
| 161 | if (unlikely(++rdp->qlen > qhimark)) { | ||
| 162 | rdp->blimit = INT_MAX; | ||
| 163 | force_quiescent_state(rdp, &rcu_ctrlblk); | ||
| 164 | } | ||
| 165 | } | ||
| 166 | |||
| 167 | #ifdef CONFIG_RCU_CPU_STALL_DETECTOR | ||
| 168 | |||
| 169 | static void record_gp_stall_check_time(struct rcu_ctrlblk *rcp) | ||
| 170 | { | ||
| 171 | rcp->gp_start = jiffies; | ||
| 172 | rcp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_CHECK; | ||
| 173 | } | ||
| 174 | |||
| 175 | static void print_other_cpu_stall(struct rcu_ctrlblk *rcp) | ||
| 176 | { | ||
| 177 | int cpu; | ||
| 178 | long delta; | ||
| 179 | unsigned long flags; | ||
| 180 | |||
| 181 | /* Only let one CPU complain about others per time interval. */ | ||
| 182 | |||
| 183 | spin_lock_irqsave(&rcp->lock, flags); | ||
| 184 | delta = jiffies - rcp->jiffies_stall; | ||
| 185 | if (delta < 2 || rcp->cur != rcp->completed) { | ||
| 186 | spin_unlock_irqrestore(&rcp->lock, flags); | ||
| 187 | return; | ||
| 188 | } | ||
| 189 | rcp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_RECHECK; | ||
| 190 | spin_unlock_irqrestore(&rcp->lock, flags); | ||
| 191 | |||
| 192 | /* OK, time to rat on our buddy... */ | ||
| 193 | |||
| 194 | printk(KERN_ERR "RCU detected CPU stalls:"); | ||
| 195 | for_each_possible_cpu(cpu) { | ||
| 196 | if (cpu_isset(cpu, rcp->cpumask)) | ||
| 197 | printk(" %d", cpu); | ||
| 198 | } | ||
| 199 | printk(" (detected by %d, t=%ld jiffies)\n", | ||
| 200 | smp_processor_id(), (long)(jiffies - rcp->gp_start)); | ||
| 201 | } | ||
| 202 | |||
| 203 | static void print_cpu_stall(struct rcu_ctrlblk *rcp) | ||
| 204 | { | ||
| 205 | unsigned long flags; | ||
| 206 | |||
| 207 | printk(KERN_ERR "RCU detected CPU %d stall (t=%lu/%lu jiffies)\n", | ||
| 208 | smp_processor_id(), jiffies, | ||
| 209 | jiffies - rcp->gp_start); | ||
| 210 | dump_stack(); | ||
| 211 | spin_lock_irqsave(&rcp->lock, flags); | ||
| 212 | if ((long)(jiffies - rcp->jiffies_stall) >= 0) | ||
| 213 | rcp->jiffies_stall = | ||
| 214 | jiffies + RCU_SECONDS_TILL_STALL_RECHECK; | ||
| 215 | spin_unlock_irqrestore(&rcp->lock, flags); | ||
| 216 | set_need_resched(); /* kick ourselves to get things going. */ | ||
| 217 | } | ||
| 218 | |||
| 219 | static void check_cpu_stall(struct rcu_ctrlblk *rcp) | ||
| 220 | { | ||
| 221 | long delta; | ||
| 222 | |||
| 223 | delta = jiffies - rcp->jiffies_stall; | ||
| 224 | if (cpu_isset(smp_processor_id(), rcp->cpumask) && delta >= 0) { | ||
| 225 | |||
| 226 | /* We haven't checked in, so go dump stack. */ | ||
| 227 | print_cpu_stall(rcp); | ||
| 228 | |||
| 229 | } else if (rcp->cur != rcp->completed && delta >= 2) { | ||
| 230 | |||
| 231 | /* They had two seconds to dump stack, so complain. */ | ||
| 232 | print_other_cpu_stall(rcp); | ||
| 233 | } | ||
| 234 | } | ||
| 235 | |||
| 236 | #else /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ | ||
| 237 | |||
| 238 | static void record_gp_stall_check_time(struct rcu_ctrlblk *rcp) | ||
| 239 | { | ||
| 240 | } | ||
| 241 | |||
| 242 | static inline void check_cpu_stall(struct rcu_ctrlblk *rcp) | ||
| 243 | { | ||
| 244 | } | ||
| 245 | |||
| 246 | #endif /* #else #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ | ||
| 247 | |||
| 121 | /** | 248 | /** |
| 122 | * call_rcu - Queue an RCU callback for invocation after a grace period. | 249 | * call_rcu - Queue an RCU callback for invocation after a grace period. |
| 123 | * @head: structure to be used for queueing the RCU updates. | 250 | * @head: structure to be used for queueing the RCU updates. |
| @@ -133,18 +260,10 @@ void call_rcu(struct rcu_head *head, | |||
| 133 | void (*func)(struct rcu_head *rcu)) | 260 | void (*func)(struct rcu_head *rcu)) |
| 134 | { | 261 | { |
| 135 | unsigned long flags; | 262 | unsigned long flags; |
| 136 | struct rcu_data *rdp; | ||
| 137 | 263 | ||
| 138 | head->func = func; | 264 | head->func = func; |
| 139 | head->next = NULL; | ||
| 140 | local_irq_save(flags); | 265 | local_irq_save(flags); |
| 141 | rdp = &__get_cpu_var(rcu_data); | 266 | __call_rcu(head, &rcu_ctrlblk, &__get_cpu_var(rcu_data)); |
| 142 | *rdp->nxttail = head; | ||
| 143 | rdp->nxttail = &head->next; | ||
| 144 | if (unlikely(++rdp->qlen > qhimark)) { | ||
| 145 | rdp->blimit = INT_MAX; | ||
| 146 | force_quiescent_state(rdp, &rcu_ctrlblk); | ||
| 147 | } | ||
| 148 | local_irq_restore(flags); | 267 | local_irq_restore(flags); |
| 149 | } | 268 | } |
| 150 | EXPORT_SYMBOL_GPL(call_rcu); | 269 | EXPORT_SYMBOL_GPL(call_rcu); |
| @@ -169,20 +288,10 @@ void call_rcu_bh(struct rcu_head *head, | |||
| 169 | void (*func)(struct rcu_head *rcu)) | 288 | void (*func)(struct rcu_head *rcu)) |
| 170 | { | 289 | { |
| 171 | unsigned long flags; | 290 | unsigned long flags; |
| 172 | struct rcu_data *rdp; | ||
| 173 | 291 | ||
| 174 | head->func = func; | 292 | head->func = func; |
| 175 | head->next = NULL; | ||
| 176 | local_irq_save(flags); | 293 | local_irq_save(flags); |
| 177 | rdp = &__get_cpu_var(rcu_bh_data); | 294 | __call_rcu(head, &rcu_bh_ctrlblk, &__get_cpu_var(rcu_bh_data)); |
| 178 | *rdp->nxttail = head; | ||
| 179 | rdp->nxttail = &head->next; | ||
| 180 | |||
| 181 | if (unlikely(++rdp->qlen > qhimark)) { | ||
| 182 | rdp->blimit = INT_MAX; | ||
| 183 | force_quiescent_state(rdp, &rcu_bh_ctrlblk); | ||
| 184 | } | ||
| 185 | |||
| 186 | local_irq_restore(flags); | 295 | local_irq_restore(flags); |
| 187 | } | 296 | } |
| 188 | EXPORT_SYMBOL_GPL(call_rcu_bh); | 297 | EXPORT_SYMBOL_GPL(call_rcu_bh); |
| @@ -211,12 +320,6 @@ EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); | |||
| 211 | static inline void raise_rcu_softirq(void) | 320 | static inline void raise_rcu_softirq(void) |
| 212 | { | 321 | { |
| 213 | raise_softirq(RCU_SOFTIRQ); | 322 | raise_softirq(RCU_SOFTIRQ); |
| 214 | /* | ||
| 215 | * The smp_mb() here is required to ensure that this cpu's | ||
| 216 | * __rcu_process_callbacks() reads the most recently updated | ||
| 217 | * value of rcu->cur. | ||
| 218 | */ | ||
| 219 | smp_mb(); | ||
| 220 | } | 323 | } |
| 221 | 324 | ||
| 222 | /* | 325 | /* |
| @@ -225,6 +328,7 @@ static inline void raise_rcu_softirq(void) | |||
| 225 | */ | 328 | */ |
| 226 | static void rcu_do_batch(struct rcu_data *rdp) | 329 | static void rcu_do_batch(struct rcu_data *rdp) |
| 227 | { | 330 | { |
| 331 | unsigned long flags; | ||
| 228 | struct rcu_head *next, *list; | 332 | struct rcu_head *next, *list; |
| 229 | int count = 0; | 333 | int count = 0; |
| 230 | 334 | ||
| @@ -239,9 +343,9 @@ static void rcu_do_batch(struct rcu_data *rdp) | |||
| 239 | } | 343 | } |
| 240 | rdp->donelist = list; | 344 | rdp->donelist = list; |
| 241 | 345 | ||
| 242 | local_irq_disable(); | 346 | local_irq_save(flags); |
| 243 | rdp->qlen -= count; | 347 | rdp->qlen -= count; |
| 244 | local_irq_enable(); | 348 | local_irq_restore(flags); |
| 245 | if (rdp->blimit == INT_MAX && rdp->qlen <= qlowmark) | 349 | if (rdp->blimit == INT_MAX && rdp->qlen <= qlowmark) |
| 246 | rdp->blimit = blimit; | 350 | rdp->blimit = blimit; |
| 247 | 351 | ||
| @@ -269,6 +373,7 @@ static void rcu_do_batch(struct rcu_data *rdp) | |||
| 269 | * rcu_check_quiescent_state calls rcu_start_batch(0) to start the next grace | 373 | * rcu_check_quiescent_state calls rcu_start_batch(0) to start the next grace |
| 270 | * period (if necessary). | 374 | * period (if necessary). |
| 271 | */ | 375 | */ |
| 376 | |||
| 272 | /* | 377 | /* |
| 273 | * Register a new batch of callbacks, and start it up if there is currently no | 378 | * Register a new batch of callbacks, and start it up if there is currently no |
| 274 | * active batch and the batch to be registered has not already occurred. | 379 | * active batch and the batch to be registered has not already occurred. |
| @@ -276,15 +381,10 @@ static void rcu_do_batch(struct rcu_data *rdp) | |||
| 276 | */ | 381 | */ |
| 277 | static void rcu_start_batch(struct rcu_ctrlblk *rcp) | 382 | static void rcu_start_batch(struct rcu_ctrlblk *rcp) |
| 278 | { | 383 | { |
| 279 | if (rcp->next_pending && | 384 | if (rcp->cur != rcp->pending && |
| 280 | rcp->completed == rcp->cur) { | 385 | rcp->completed == rcp->cur) { |
| 281 | rcp->next_pending = 0; | ||
| 282 | /* | ||
| 283 | * next_pending == 0 must be visible in | ||
| 284 | * __rcu_process_callbacks() before it can see new value of cur. | ||
| 285 | */ | ||
| 286 | smp_wmb(); | ||
| 287 | rcp->cur++; | 386 | rcp->cur++; |
| 387 | record_gp_stall_check_time(rcp); | ||
| 288 | 388 | ||
| 289 | /* | 389 | /* |
| 290 | * Accessing nohz_cpu_mask before incrementing rcp->cur needs a | 390 | * Accessing nohz_cpu_mask before incrementing rcp->cur needs a |
| @@ -322,6 +422,8 @@ static void cpu_quiet(int cpu, struct rcu_ctrlblk *rcp) | |||
| 322 | static void rcu_check_quiescent_state(struct rcu_ctrlblk *rcp, | 422 | static void rcu_check_quiescent_state(struct rcu_ctrlblk *rcp, |
| 323 | struct rcu_data *rdp) | 423 | struct rcu_data *rdp) |
| 324 | { | 424 | { |
| 425 | unsigned long flags; | ||
| 426 | |||
| 325 | if (rdp->quiescbatch != rcp->cur) { | 427 | if (rdp->quiescbatch != rcp->cur) { |
| 326 | /* start new grace period: */ | 428 | /* start new grace period: */ |
| 327 | rdp->qs_pending = 1; | 429 | rdp->qs_pending = 1; |
| @@ -345,7 +447,7 @@ static void rcu_check_quiescent_state(struct rcu_ctrlblk *rcp, | |||
| 345 | return; | 447 | return; |
| 346 | rdp->qs_pending = 0; | 448 | rdp->qs_pending = 0; |
| 347 | 449 | ||
| 348 | spin_lock(&rcp->lock); | 450 | spin_lock_irqsave(&rcp->lock, flags); |
| 349 | /* | 451 | /* |
| 350 | * rdp->quiescbatch/rcp->cur and the cpu bitmap can come out of sync | 452 | * rdp->quiescbatch/rcp->cur and the cpu bitmap can come out of sync |
| 351 | * during cpu startup. Ignore the quiescent state. | 453 | * during cpu startup. Ignore the quiescent state. |
| @@ -353,7 +455,7 @@ static void rcu_check_quiescent_state(struct rcu_ctrlblk *rcp, | |||
| 353 | if (likely(rdp->quiescbatch == rcp->cur)) | 455 | if (likely(rdp->quiescbatch == rcp->cur)) |
| 354 | cpu_quiet(rdp->cpu, rcp); | 456 | cpu_quiet(rdp->cpu, rcp); |
| 355 | 457 | ||
| 356 | spin_unlock(&rcp->lock); | 458 | spin_unlock_irqrestore(&rcp->lock, flags); |
| 357 | } | 459 | } |
| 358 | 460 | ||
| 359 | 461 | ||
| @@ -364,33 +466,38 @@ static void rcu_check_quiescent_state(struct rcu_ctrlblk *rcp, | |||
| 364 | * which is dead and hence not processing interrupts. | 466 | * which is dead and hence not processing interrupts. |
| 365 | */ | 467 | */ |
| 366 | static void rcu_move_batch(struct rcu_data *this_rdp, struct rcu_head *list, | 468 | static void rcu_move_batch(struct rcu_data *this_rdp, struct rcu_head *list, |
| 367 | struct rcu_head **tail) | 469 | struct rcu_head **tail, long batch) |
| 368 | { | 470 | { |
| 369 | local_irq_disable(); | 471 | unsigned long flags; |
| 370 | *this_rdp->nxttail = list; | 472 | |
| 371 | if (list) | 473 | if (list) { |
| 372 | this_rdp->nxttail = tail; | 474 | local_irq_save(flags); |
| 373 | local_irq_enable(); | 475 | this_rdp->batch = batch; |
| 476 | *this_rdp->nxttail[2] = list; | ||
| 477 | this_rdp->nxttail[2] = tail; | ||
| 478 | local_irq_restore(flags); | ||
| 479 | } | ||
| 374 | } | 480 | } |
| 375 | 481 | ||
| 376 | static void __rcu_offline_cpu(struct rcu_data *this_rdp, | 482 | static void __rcu_offline_cpu(struct rcu_data *this_rdp, |
| 377 | struct rcu_ctrlblk *rcp, struct rcu_data *rdp) | 483 | struct rcu_ctrlblk *rcp, struct rcu_data *rdp) |
| 378 | { | 484 | { |
| 379 | /* if the cpu going offline owns the grace period | 485 | unsigned long flags; |
| 486 | |||
| 487 | /* | ||
| 488 | * if the cpu going offline owns the grace period | ||
| 380 | * we can block indefinitely waiting for it, so flush | 489 | * we can block indefinitely waiting for it, so flush |
| 381 | * it here | 490 | * it here |
| 382 | */ | 491 | */ |
| 383 | spin_lock_bh(&rcp->lock); | 492 | spin_lock_irqsave(&rcp->lock, flags); |
| 384 | if (rcp->cur != rcp->completed) | 493 | if (rcp->cur != rcp->completed) |
| 385 | cpu_quiet(rdp->cpu, rcp); | 494 | cpu_quiet(rdp->cpu, rcp); |
| 386 | spin_unlock_bh(&rcp->lock); | 495 | rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail, rcp->cur + 1); |
| 387 | rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail); | 496 | rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail[2], rcp->cur + 1); |
| 388 | rcu_move_batch(this_rdp, rdp->curlist, rdp->curtail); | 497 | spin_unlock(&rcp->lock); |
| 389 | rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail); | ||
| 390 | 498 | ||
| 391 | local_irq_disable(); | ||
| 392 | this_rdp->qlen += rdp->qlen; | 499 | this_rdp->qlen += rdp->qlen; |
| 393 | local_irq_enable(); | 500 | local_irq_restore(flags); |
| 394 | } | 501 | } |
| 395 | 502 | ||
| 396 | static void rcu_offline_cpu(int cpu) | 503 | static void rcu_offline_cpu(int cpu) |
| @@ -420,38 +527,52 @@ static void rcu_offline_cpu(int cpu) | |||
| 420 | static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp, | 527 | static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp, |
| 421 | struct rcu_data *rdp) | 528 | struct rcu_data *rdp) |
| 422 | { | 529 | { |
| 423 | if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch)) { | 530 | unsigned long flags; |
| 424 | *rdp->donetail = rdp->curlist; | 531 | long completed_snap; |
| 425 | rdp->donetail = rdp->curtail; | ||
| 426 | rdp->curlist = NULL; | ||
| 427 | rdp->curtail = &rdp->curlist; | ||
| 428 | } | ||
| 429 | 532 | ||
| 430 | if (rdp->nxtlist && !rdp->curlist) { | 533 | if (rdp->nxtlist) { |
| 431 | local_irq_disable(); | 534 | local_irq_save(flags); |
| 432 | rdp->curlist = rdp->nxtlist; | 535 | completed_snap = ACCESS_ONCE(rcp->completed); |
| 433 | rdp->curtail = rdp->nxttail; | ||
| 434 | rdp->nxtlist = NULL; | ||
| 435 | rdp->nxttail = &rdp->nxtlist; | ||
| 436 | local_irq_enable(); | ||
| 437 | 536 | ||
| 438 | /* | 537 | /* |
| 439 | * start the next batch of callbacks | 538 | * move the other grace-period-completed entries to |
| 539 | * [rdp->nxtlist, *rdp->nxttail[0]) temporarily | ||
| 440 | */ | 540 | */ |
| 541 | if (!rcu_batch_before(completed_snap, rdp->batch)) | ||
| 542 | rdp->nxttail[0] = rdp->nxttail[1] = rdp->nxttail[2]; | ||
| 543 | else if (!rcu_batch_before(completed_snap, rdp->batch - 1)) | ||
| 544 | rdp->nxttail[0] = rdp->nxttail[1]; | ||
| 441 | 545 | ||
| 442 | /* determine batch number */ | 546 | /* |
| 443 | rdp->batch = rcp->cur + 1; | 547 | * the grace period for entries in |
| 444 | /* see the comment and corresponding wmb() in | 548 | * [rdp->nxtlist, *rdp->nxttail[0]) has completed and |
| 445 | * the rcu_start_batch() | 549 | * move these entries to donelist |
| 446 | */ | 550 | */ |
| 447 | smp_rmb(); | 551 | if (rdp->nxttail[0] != &rdp->nxtlist) { |
| 552 | *rdp->donetail = rdp->nxtlist; | ||
| 553 | rdp->donetail = rdp->nxttail[0]; | ||
| 554 | rdp->nxtlist = *rdp->nxttail[0]; | ||
| 555 | *rdp->donetail = NULL; | ||
| 556 | |||
| 557 | if (rdp->nxttail[1] == rdp->nxttail[0]) | ||
| 558 | rdp->nxttail[1] = &rdp->nxtlist; | ||
| 559 | if (rdp->nxttail[2] == rdp->nxttail[0]) | ||
| 560 | rdp->nxttail[2] = &rdp->nxtlist; | ||
| 561 | rdp->nxttail[0] = &rdp->nxtlist; | ||
| 562 | } | ||
| 563 | |||
| 564 | local_irq_restore(flags); | ||
| 565 | |||
| 566 | if (rcu_batch_after(rdp->batch, rcp->pending)) { | ||
| 567 | unsigned long flags2; | ||
| 448 | 568 | ||
| 449 | if (!rcp->next_pending) { | ||
| 450 | /* and start it/schedule start if it's a new batch */ | 569 | /* and start it/schedule start if it's a new batch */ |
| 451 | spin_lock(&rcp->lock); | 570 | spin_lock_irqsave(&rcp->lock, flags2); |
| 452 | rcp->next_pending = 1; | 571 | if (rcu_batch_after(rdp->batch, rcp->pending)) { |
| 453 | rcu_start_batch(rcp); | 572 | rcp->pending = rdp->batch; |
| 454 | spin_unlock(&rcp->lock); | 573 | rcu_start_batch(rcp); |
| 574 | } | ||
| 575 | spin_unlock_irqrestore(&rcp->lock, flags2); | ||
| 455 | } | 576 | } |
| 456 | } | 577 | } |
| 457 | 578 | ||
| @@ -462,21 +583,53 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp, | |||
| 462 | 583 | ||
| 463 | static void rcu_process_callbacks(struct softirq_action *unused) | 584 | static void rcu_process_callbacks(struct softirq_action *unused) |
| 464 | { | 585 | { |
| 586 | /* | ||
| 587 | * Memory references from any prior RCU read-side critical sections | ||
| 588 | * executed by the interrupted code must be see before any RCU | ||
| 589 | * grace-period manupulations below. | ||
| 590 | */ | ||
| 591 | |||
| 592 | smp_mb(); /* See above block comment. */ | ||
| 593 | |||
| 465 | __rcu_process_callbacks(&rcu_ctrlblk, &__get_cpu_var(rcu_data)); | 594 | __rcu_process_callbacks(&rcu_ctrlblk, &__get_cpu_var(rcu_data)); |
| 466 | __rcu_process_callbacks(&rcu_bh_ctrlblk, &__get_cpu_var(rcu_bh_data)); | 595 | __rcu_process_callbacks(&rcu_bh_ctrlblk, &__get_cpu_var(rcu_bh_data)); |
| 596 | |||
| 597 | /* | ||
| 598 | * Memory references from any later RCU read-side critical sections | ||
| 599 | * executed by the interrupted code must be see after any RCU | ||
| 600 | * grace-period manupulations above. | ||
| 601 | */ | ||
| 602 | |||
| 603 | smp_mb(); /* See above block comment. */ | ||
| 467 | } | 604 | } |
| 468 | 605 | ||
| 469 | static int __rcu_pending(struct rcu_ctrlblk *rcp, struct rcu_data *rdp) | 606 | static int __rcu_pending(struct rcu_ctrlblk *rcp, struct rcu_data *rdp) |
| 470 | { | 607 | { |
| 471 | /* This cpu has pending rcu entries and the grace period | 608 | /* Check for CPU stalls, if enabled. */ |
| 472 | * for them has completed. | 609 | check_cpu_stall(rcp); |
| 473 | */ | ||
| 474 | if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch)) | ||
| 475 | return 1; | ||
| 476 | 610 | ||
| 477 | /* This cpu has no pending entries, but there are new entries */ | 611 | if (rdp->nxtlist) { |
| 478 | if (!rdp->curlist && rdp->nxtlist) | 612 | long completed_snap = ACCESS_ONCE(rcp->completed); |
| 479 | return 1; | 613 | |
| 614 | /* | ||
| 615 | * This cpu has pending rcu entries and the grace period | ||
| 616 | * for them has completed. | ||
| 617 | */ | ||
| 618 | if (!rcu_batch_before(completed_snap, rdp->batch)) | ||
| 619 | return 1; | ||
| 620 | if (!rcu_batch_before(completed_snap, rdp->batch - 1) && | ||
| 621 | rdp->nxttail[0] != rdp->nxttail[1]) | ||
| 622 | return 1; | ||
| 623 | if (rdp->nxttail[0] != &rdp->nxtlist) | ||
| 624 | return 1; | ||
| 625 | |||
| 626 | /* | ||
| 627 | * This cpu has pending rcu entries and the new batch | ||
| 628 | * for then hasn't been started nor scheduled start | ||
| 629 | */ | ||
| 630 | if (rcu_batch_after(rdp->batch, rcp->pending)) | ||
| 631 | return 1; | ||
| 632 | } | ||
| 480 | 633 | ||
| 481 | /* This cpu has finished callbacks to invoke */ | 634 | /* This cpu has finished callbacks to invoke */ |
| 482 | if (rdp->donelist) | 635 | if (rdp->donelist) |
| @@ -512,9 +665,15 @@ int rcu_needs_cpu(int cpu) | |||
| 512 | struct rcu_data *rdp = &per_cpu(rcu_data, cpu); | 665 | struct rcu_data *rdp = &per_cpu(rcu_data, cpu); |
| 513 | struct rcu_data *rdp_bh = &per_cpu(rcu_bh_data, cpu); | 666 | struct rcu_data *rdp_bh = &per_cpu(rcu_bh_data, cpu); |
| 514 | 667 | ||
| 515 | return (!!rdp->curlist || !!rdp_bh->curlist || rcu_pending(cpu)); | 668 | return !!rdp->nxtlist || !!rdp_bh->nxtlist || rcu_pending(cpu); |
| 516 | } | 669 | } |
| 517 | 670 | ||
| 671 | /* | ||
| 672 | * Top-level function driving RCU grace-period detection, normally | ||
| 673 | * invoked from the scheduler-clock interrupt. This function simply | ||
| 674 | * increments counters that are read only from softirq by this same | ||
| 675 | * CPU, so there are no memory barriers required. | ||
| 676 | */ | ||
| 518 | void rcu_check_callbacks(int cpu, int user) | 677 | void rcu_check_callbacks(int cpu, int user) |
| 519 | { | 678 | { |
| 520 | if (user || | 679 | if (user || |
| @@ -558,14 +717,17 @@ void rcu_check_callbacks(int cpu, int user) | |||
| 558 | static void rcu_init_percpu_data(int cpu, struct rcu_ctrlblk *rcp, | 717 | static void rcu_init_percpu_data(int cpu, struct rcu_ctrlblk *rcp, |
| 559 | struct rcu_data *rdp) | 718 | struct rcu_data *rdp) |
| 560 | { | 719 | { |
| 720 | unsigned long flags; | ||
| 721 | |||
| 722 | spin_lock_irqsave(&rcp->lock, flags); | ||
| 561 | memset(rdp, 0, sizeof(*rdp)); | 723 | memset(rdp, 0, sizeof(*rdp)); |
| 562 | rdp->curtail = &rdp->curlist; | 724 | rdp->nxttail[0] = rdp->nxttail[1] = rdp->nxttail[2] = &rdp->nxtlist; |
| 563 | rdp->nxttail = &rdp->nxtlist; | ||
| 564 | rdp->donetail = &rdp->donelist; | 725 | rdp->donetail = &rdp->donelist; |
| 565 | rdp->quiescbatch = rcp->completed; | 726 | rdp->quiescbatch = rcp->completed; |
| 566 | rdp->qs_pending = 0; | 727 | rdp->qs_pending = 0; |
| 567 | rdp->cpu = cpu; | 728 | rdp->cpu = cpu; |
| 568 | rdp->blimit = blimit; | 729 | rdp->blimit = blimit; |
| 730 | spin_unlock_irqrestore(&rcp->lock, flags); | ||
| 569 | } | 731 | } |
| 570 | 732 | ||
| 571 | static void __cpuinit rcu_online_cpu(int cpu) | 733 | static void __cpuinit rcu_online_cpu(int cpu) |
| @@ -610,6 +772,9 @@ static struct notifier_block __cpuinitdata rcu_nb = { | |||
| 610 | */ | 772 | */ |
| 611 | void __init __rcu_init(void) | 773 | void __init __rcu_init(void) |
| 612 | { | 774 | { |
| 775 | #ifdef CONFIG_RCU_CPU_STALL_DETECTOR | ||
| 776 | printk(KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n"); | ||
| 777 | #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ | ||
| 613 | rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, | 778 | rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, |
| 614 | (void *)(long)smp_processor_id()); | 779 | (void *)(long)smp_processor_id()); |
| 615 | /* Register notifier for non-boot CPUs */ | 780 | /* Register notifier for non-boot CPUs */ |
diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c index 27827931ca0d..ca4bbbe04aa4 100644 --- a/kernel/rcupreempt.c +++ b/kernel/rcupreempt.c | |||
| @@ -59,14 +59,6 @@ | |||
| 59 | #include <linux/rcupreempt_trace.h> | 59 | #include <linux/rcupreempt_trace.h> |
| 60 | 60 | ||
| 61 | /* | 61 | /* |
| 62 | * Macro that prevents the compiler from reordering accesses, but does | ||
| 63 | * absolutely -nothing- to prevent CPUs from reordering. This is used | ||
| 64 | * only to mediate communication between mainline code and hardware | ||
| 65 | * interrupt and NMI handlers. | ||
| 66 | */ | ||
| 67 | #define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x)) | ||
| 68 | |||
| 69 | /* | ||
| 70 | * PREEMPT_RCU data structures. | 62 | * PREEMPT_RCU data structures. |
| 71 | */ | 63 | */ |
| 72 | 64 | ||
diff --git a/kernel/rcupreempt_trace.c b/kernel/rcupreempt_trace.c index 5edf82c34bbc..35c2d3360ecf 100644 --- a/kernel/rcupreempt_trace.c +++ b/kernel/rcupreempt_trace.c | |||
| @@ -308,11 +308,16 @@ out: | |||
| 308 | 308 | ||
| 309 | static int __init rcupreempt_trace_init(void) | 309 | static int __init rcupreempt_trace_init(void) |
| 310 | { | 310 | { |
| 311 | int ret; | ||
| 312 | |||
| 311 | mutex_init(&rcupreempt_trace_mutex); | 313 | mutex_init(&rcupreempt_trace_mutex); |
| 312 | rcupreempt_trace_buf = kmalloc(RCUPREEMPT_TRACE_BUF_SIZE, GFP_KERNEL); | 314 | rcupreempt_trace_buf = kmalloc(RCUPREEMPT_TRACE_BUF_SIZE, GFP_KERNEL); |
| 313 | if (!rcupreempt_trace_buf) | 315 | if (!rcupreempt_trace_buf) |
| 314 | return 1; | 316 | return 1; |
| 315 | return rcupreempt_debugfs_init(); | 317 | ret = rcupreempt_debugfs_init(); |
| 318 | if (ret) | ||
| 319 | kfree(rcupreempt_trace_buf); | ||
| 320 | return ret; | ||
| 316 | } | 321 | } |
| 317 | 322 | ||
| 318 | static void __exit rcupreempt_trace_cleanup(void) | 323 | static void __exit rcupreempt_trace_cleanup(void) |
diff --git a/kernel/resource.c b/kernel/resource.c index 03d796c1b2e9..4089d12af6e0 100644 --- a/kernel/resource.c +++ b/kernel/resource.c | |||
| @@ -38,10 +38,6 @@ EXPORT_SYMBOL(iomem_resource); | |||
| 38 | 38 | ||
| 39 | static DEFINE_RWLOCK(resource_lock); | 39 | static DEFINE_RWLOCK(resource_lock); |
| 40 | 40 | ||
| 41 | #ifdef CONFIG_PROC_FS | ||
| 42 | |||
| 43 | enum { MAX_IORES_LEVEL = 5 }; | ||
| 44 | |||
| 45 | static void *r_next(struct seq_file *m, void *v, loff_t *pos) | 41 | static void *r_next(struct seq_file *m, void *v, loff_t *pos) |
| 46 | { | 42 | { |
| 47 | struct resource *p = v; | 43 | struct resource *p = v; |
| @@ -53,6 +49,10 @@ static void *r_next(struct seq_file *m, void *v, loff_t *pos) | |||
| 53 | return p->sibling; | 49 | return p->sibling; |
| 54 | } | 50 | } |
| 55 | 51 | ||
| 52 | #ifdef CONFIG_PROC_FS | ||
| 53 | |||
| 54 | enum { MAX_IORES_LEVEL = 5 }; | ||
| 55 | |||
| 56 | static void *r_start(struct seq_file *m, loff_t *pos) | 56 | static void *r_start(struct seq_file *m, loff_t *pos) |
| 57 | __acquires(resource_lock) | 57 | __acquires(resource_lock) |
| 58 | { | 58 | { |
| @@ -516,6 +516,70 @@ int adjust_resource(struct resource *res, resource_size_t start, resource_size_t | |||
| 516 | return result; | 516 | return result; |
| 517 | } | 517 | } |
| 518 | 518 | ||
| 519 | static void __init __reserve_region_with_split(struct resource *root, | ||
| 520 | resource_size_t start, resource_size_t end, | ||
| 521 | const char *name) | ||
| 522 | { | ||
| 523 | struct resource *parent = root; | ||
| 524 | struct resource *conflict; | ||
| 525 | struct resource *res = kzalloc(sizeof(*res), GFP_KERNEL); | ||
| 526 | |||
| 527 | if (!res) | ||
| 528 | return; | ||
| 529 | |||
| 530 | res->name = name; | ||
| 531 | res->start = start; | ||
| 532 | res->end = end; | ||
| 533 | res->flags = IORESOURCE_BUSY; | ||
| 534 | |||
| 535 | for (;;) { | ||
| 536 | conflict = __request_resource(parent, res); | ||
| 537 | if (!conflict) | ||
| 538 | break; | ||
| 539 | if (conflict != parent) { | ||
| 540 | parent = conflict; | ||
| 541 | if (!(conflict->flags & IORESOURCE_BUSY)) | ||
| 542 | continue; | ||
| 543 | } | ||
| 544 | |||
| 545 | /* Uhhuh, that didn't work out.. */ | ||
| 546 | kfree(res); | ||
| 547 | res = NULL; | ||
| 548 | break; | ||
| 549 | } | ||
| 550 | |||
| 551 | if (!res) { | ||
| 552 | /* failed, split and try again */ | ||
| 553 | |||
| 554 | /* conflict covered whole area */ | ||
| 555 | if (conflict->start <= start && conflict->end >= end) | ||
| 556 | return; | ||
| 557 | |||
| 558 | if (conflict->start > start) | ||
| 559 | __reserve_region_with_split(root, start, conflict->start-1, name); | ||
| 560 | if (!(conflict->flags & IORESOURCE_BUSY)) { | ||
| 561 | resource_size_t common_start, common_end; | ||
| 562 | |||
| 563 | common_start = max(conflict->start, start); | ||
| 564 | common_end = min(conflict->end, end); | ||
| 565 | if (common_start < common_end) | ||
| 566 | __reserve_region_with_split(root, common_start, common_end, name); | ||
| 567 | } | ||
| 568 | if (conflict->end < end) | ||
| 569 | __reserve_region_with_split(root, conflict->end+1, end, name); | ||
| 570 | } | ||
| 571 | |||
| 572 | } | ||
| 573 | |||
| 574 | void reserve_region_with_split(struct resource *root, | ||
| 575 | resource_size_t start, resource_size_t end, | ||
| 576 | const char *name) | ||
| 577 | { | ||
| 578 | write_lock(&resource_lock); | ||
| 579 | __reserve_region_with_split(root, start, end, name); | ||
| 580 | write_unlock(&resource_lock); | ||
| 581 | } | ||
| 582 | |||
| 519 | EXPORT_SYMBOL(adjust_resource); | 583 | EXPORT_SYMBOL(adjust_resource); |
| 520 | 584 | ||
| 521 | /** | 585 | /** |
| @@ -562,33 +626,34 @@ struct resource * __request_region(struct resource *parent, | |||
| 562 | { | 626 | { |
| 563 | struct resource *res = kzalloc(sizeof(*res), GFP_KERNEL); | 627 | struct resource *res = kzalloc(sizeof(*res), GFP_KERNEL); |
| 564 | 628 | ||
| 565 | if (res) { | 629 | if (!res) |
| 566 | res->name = name; | 630 | return NULL; |
| 567 | res->start = start; | ||
| 568 | res->end = start + n - 1; | ||
| 569 | res->flags = IORESOURCE_BUSY; | ||
| 570 | 631 | ||
| 571 | write_lock(&resource_lock); | 632 | res->name = name; |
| 633 | res->start = start; | ||
| 634 | res->end = start + n - 1; | ||
| 635 | res->flags = IORESOURCE_BUSY; | ||
| 572 | 636 | ||
| 573 | for (;;) { | 637 | write_lock(&resource_lock); |
| 574 | struct resource *conflict; | ||
| 575 | 638 | ||
| 576 | conflict = __request_resource(parent, res); | 639 | for (;;) { |
| 577 | if (!conflict) | 640 | struct resource *conflict; |
| 578 | break; | ||
| 579 | if (conflict != parent) { | ||
| 580 | parent = conflict; | ||
| 581 | if (!(conflict->flags & IORESOURCE_BUSY)) | ||
| 582 | continue; | ||
| 583 | } | ||
| 584 | 641 | ||
| 585 | /* Uhhuh, that didn't work out.. */ | 642 | conflict = __request_resource(parent, res); |
| 586 | kfree(res); | 643 | if (!conflict) |
| 587 | res = NULL; | ||
| 588 | break; | 644 | break; |
| 645 | if (conflict != parent) { | ||
| 646 | parent = conflict; | ||
| 647 | if (!(conflict->flags & IORESOURCE_BUSY)) | ||
| 648 | continue; | ||
| 589 | } | 649 | } |
| 590 | write_unlock(&resource_lock); | 650 | |
| 651 | /* Uhhuh, that didn't work out.. */ | ||
| 652 | kfree(res); | ||
| 653 | res = NULL; | ||
| 654 | break; | ||
| 591 | } | 655 | } |
| 656 | write_unlock(&resource_lock); | ||
| 592 | return res; | 657 | return res; |
| 593 | } | 658 | } |
| 594 | EXPORT_SYMBOL(__request_region); | 659 | EXPORT_SYMBOL(__request_region); |
| @@ -763,3 +828,40 @@ static int __init reserve_setup(char *str) | |||
| 763 | } | 828 | } |
| 764 | 829 | ||
| 765 | __setup("reserve=", reserve_setup); | 830 | __setup("reserve=", reserve_setup); |
| 831 | |||
| 832 | /* | ||
| 833 | * Check if the requested addr and size spans more than any slot in the | ||
| 834 | * iomem resource tree. | ||
| 835 | */ | ||
| 836 | int iomem_map_sanity_check(resource_size_t addr, unsigned long size) | ||
| 837 | { | ||
| 838 | struct resource *p = &iomem_resource; | ||
| 839 | int err = 0; | ||
| 840 | loff_t l; | ||
| 841 | |||
| 842 | read_lock(&resource_lock); | ||
| 843 | for (p = p->child; p ; p = r_next(NULL, p, &l)) { | ||
| 844 | /* | ||
| 845 | * We can probably skip the resources without | ||
| 846 | * IORESOURCE_IO attribute? | ||
| 847 | */ | ||
| 848 | if (p->start >= addr + size) | ||
| 849 | continue; | ||
| 850 | if (p->end < addr) | ||
| 851 | continue; | ||
| 852 | if (p->start <= addr && (p->end >= addr + size - 1)) | ||
| 853 | continue; | ||
| 854 | printk(KERN_WARNING "resource map sanity check conflict: " | ||
| 855 | "0x%llx 0x%llx 0x%llx 0x%llx %s\n", | ||
| 856 | (unsigned long long)addr, | ||
| 857 | (unsigned long long)(addr + size - 1), | ||
| 858 | (unsigned long long)p->start, | ||
| 859 | (unsigned long long)p->end, | ||
| 860 | p->name); | ||
| 861 | err = -1; | ||
| 862 | break; | ||
| 863 | } | ||
| 864 | read_unlock(&resource_lock); | ||
| 865 | |||
| 866 | return err; | ||
| 867 | } | ||
diff --git a/kernel/sched.c b/kernel/sched.c index e46b5afa200d..eb3c72953615 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
| @@ -201,14 +201,19 @@ void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime) | |||
| 201 | hrtimer_init(&rt_b->rt_period_timer, | 201 | hrtimer_init(&rt_b->rt_period_timer, |
| 202 | CLOCK_MONOTONIC, HRTIMER_MODE_REL); | 202 | CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| 203 | rt_b->rt_period_timer.function = sched_rt_period_timer; | 203 | rt_b->rt_period_timer.function = sched_rt_period_timer; |
| 204 | rt_b->rt_period_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ; | 204 | rt_b->rt_period_timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED; |
| 205 | } | ||
| 206 | |||
| 207 | static inline int rt_bandwidth_enabled(void) | ||
| 208 | { | ||
| 209 | return sysctl_sched_rt_runtime >= 0; | ||
| 205 | } | 210 | } |
| 206 | 211 | ||
| 207 | static void start_rt_bandwidth(struct rt_bandwidth *rt_b) | 212 | static void start_rt_bandwidth(struct rt_bandwidth *rt_b) |
| 208 | { | 213 | { |
| 209 | ktime_t now; | 214 | ktime_t now; |
| 210 | 215 | ||
| 211 | if (rt_b->rt_runtime == RUNTIME_INF) | 216 | if (rt_bandwidth_enabled() && rt_b->rt_runtime == RUNTIME_INF) |
| 212 | return; | 217 | return; |
| 213 | 218 | ||
| 214 | if (hrtimer_active(&rt_b->rt_period_timer)) | 219 | if (hrtimer_active(&rt_b->rt_period_timer)) |
| @@ -297,9 +302,9 @@ static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp; | |||
| 297 | static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity); | 302 | static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity); |
| 298 | static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp; | 303 | static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp; |
| 299 | #endif /* CONFIG_RT_GROUP_SCHED */ | 304 | #endif /* CONFIG_RT_GROUP_SCHED */ |
| 300 | #else /* !CONFIG_FAIR_GROUP_SCHED */ | 305 | #else /* !CONFIG_USER_SCHED */ |
| 301 | #define root_task_group init_task_group | 306 | #define root_task_group init_task_group |
| 302 | #endif /* CONFIG_FAIR_GROUP_SCHED */ | 307 | #endif /* CONFIG_USER_SCHED */ |
| 303 | 308 | ||
| 304 | /* task_group_lock serializes add/remove of task groups and also changes to | 309 | /* task_group_lock serializes add/remove of task groups and also changes to |
| 305 | * a task group's cpu shares. | 310 | * a task group's cpu shares. |
| @@ -603,9 +608,9 @@ struct rq { | |||
| 603 | 608 | ||
| 604 | static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); | 609 | static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); |
| 605 | 610 | ||
| 606 | static inline void check_preempt_curr(struct rq *rq, struct task_struct *p) | 611 | static inline void check_preempt_curr(struct rq *rq, struct task_struct *p, int sync) |
| 607 | { | 612 | { |
| 608 | rq->curr->sched_class->check_preempt_curr(rq, p); | 613 | rq->curr->sched_class->check_preempt_curr(rq, p, sync); |
| 609 | } | 614 | } |
| 610 | 615 | ||
| 611 | static inline int cpu_of(struct rq *rq) | 616 | static inline int cpu_of(struct rq *rq) |
| @@ -1086,7 +1091,7 @@ hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu) | |||
| 1086 | return NOTIFY_DONE; | 1091 | return NOTIFY_DONE; |
| 1087 | } | 1092 | } |
| 1088 | 1093 | ||
| 1089 | static void init_hrtick(void) | 1094 | static __init void init_hrtick(void) |
| 1090 | { | 1095 | { |
| 1091 | hotcpu_notifier(hotplug_hrtick, 0); | 1096 | hotcpu_notifier(hotplug_hrtick, 0); |
| 1092 | } | 1097 | } |
| @@ -1101,7 +1106,7 @@ static void hrtick_start(struct rq *rq, u64 delay) | |||
| 1101 | hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), HRTIMER_MODE_REL); | 1106 | hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), HRTIMER_MODE_REL); |
| 1102 | } | 1107 | } |
| 1103 | 1108 | ||
| 1104 | static void init_hrtick(void) | 1109 | static inline void init_hrtick(void) |
| 1105 | { | 1110 | { |
| 1106 | } | 1111 | } |
| 1107 | #endif /* CONFIG_SMP */ | 1112 | #endif /* CONFIG_SMP */ |
| @@ -1118,9 +1123,9 @@ static void init_rq_hrtick(struct rq *rq) | |||
| 1118 | 1123 | ||
| 1119 | hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | 1124 | hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| 1120 | rq->hrtick_timer.function = hrtick; | 1125 | rq->hrtick_timer.function = hrtick; |
| 1121 | rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ; | 1126 | rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU; |
| 1122 | } | 1127 | } |
| 1123 | #else | 1128 | #else /* CONFIG_SCHED_HRTICK */ |
| 1124 | static inline void hrtick_clear(struct rq *rq) | 1129 | static inline void hrtick_clear(struct rq *rq) |
| 1125 | { | 1130 | { |
| 1126 | } | 1131 | } |
| @@ -1132,7 +1137,7 @@ static inline void init_rq_hrtick(struct rq *rq) | |||
| 1132 | static inline void init_hrtick(void) | 1137 | static inline void init_hrtick(void) |
| 1133 | { | 1138 | { |
| 1134 | } | 1139 | } |
| 1135 | #endif | 1140 | #endif /* CONFIG_SCHED_HRTICK */ |
| 1136 | 1141 | ||
| 1137 | /* | 1142 | /* |
| 1138 | * resched_task - mark a task 'to be rescheduled now'. | 1143 | * resched_task - mark a task 'to be rescheduled now'. |
| @@ -1379,38 +1384,24 @@ static inline void dec_cpu_load(struct rq *rq, unsigned long load) | |||
| 1379 | update_load_sub(&rq->load, load); | 1384 | update_load_sub(&rq->load, load); |
| 1380 | } | 1385 | } |
| 1381 | 1386 | ||
| 1382 | #ifdef CONFIG_SMP | 1387 | #if (defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)) || defined(CONFIG_RT_GROUP_SCHED) |
| 1383 | static unsigned long source_load(int cpu, int type); | 1388 | typedef int (*tg_visitor)(struct task_group *, void *); |
| 1384 | static unsigned long target_load(int cpu, int type); | ||
| 1385 | static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd); | ||
| 1386 | |||
| 1387 | static unsigned long cpu_avg_load_per_task(int cpu) | ||
| 1388 | { | ||
| 1389 | struct rq *rq = cpu_rq(cpu); | ||
| 1390 | |||
| 1391 | if (rq->nr_running) | ||
| 1392 | rq->avg_load_per_task = rq->load.weight / rq->nr_running; | ||
| 1393 | |||
| 1394 | return rq->avg_load_per_task; | ||
| 1395 | } | ||
| 1396 | |||
| 1397 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
| 1398 | |||
| 1399 | typedef void (*tg_visitor)(struct task_group *, int, struct sched_domain *); | ||
| 1400 | 1389 | ||
| 1401 | /* | 1390 | /* |
| 1402 | * Iterate the full tree, calling @down when first entering a node and @up when | 1391 | * Iterate the full tree, calling @down when first entering a node and @up when |
| 1403 | * leaving it for the final time. | 1392 | * leaving it for the final time. |
| 1404 | */ | 1393 | */ |
| 1405 | static void | 1394 | static int walk_tg_tree(tg_visitor down, tg_visitor up, void *data) |
| 1406 | walk_tg_tree(tg_visitor down, tg_visitor up, int cpu, struct sched_domain *sd) | ||
| 1407 | { | 1395 | { |
| 1408 | struct task_group *parent, *child; | 1396 | struct task_group *parent, *child; |
| 1397 | int ret; | ||
| 1409 | 1398 | ||
| 1410 | rcu_read_lock(); | 1399 | rcu_read_lock(); |
| 1411 | parent = &root_task_group; | 1400 | parent = &root_task_group; |
| 1412 | down: | 1401 | down: |
| 1413 | (*down)(parent, cpu, sd); | 1402 | ret = (*down)(parent, data); |
| 1403 | if (ret) | ||
| 1404 | goto out_unlock; | ||
| 1414 | list_for_each_entry_rcu(child, &parent->children, siblings) { | 1405 | list_for_each_entry_rcu(child, &parent->children, siblings) { |
| 1415 | parent = child; | 1406 | parent = child; |
| 1416 | goto down; | 1407 | goto down; |
| @@ -1418,15 +1409,43 @@ down: | |||
| 1418 | up: | 1409 | up: |
| 1419 | continue; | 1410 | continue; |
| 1420 | } | 1411 | } |
| 1421 | (*up)(parent, cpu, sd); | 1412 | ret = (*up)(parent, data); |
| 1413 | if (ret) | ||
| 1414 | goto out_unlock; | ||
| 1422 | 1415 | ||
| 1423 | child = parent; | 1416 | child = parent; |
| 1424 | parent = parent->parent; | 1417 | parent = parent->parent; |
| 1425 | if (parent) | 1418 | if (parent) |
| 1426 | goto up; | 1419 | goto up; |
| 1420 | out_unlock: | ||
| 1427 | rcu_read_unlock(); | 1421 | rcu_read_unlock(); |
| 1422 | |||
| 1423 | return ret; | ||
| 1424 | } | ||
| 1425 | |||
| 1426 | static int tg_nop(struct task_group *tg, void *data) | ||
| 1427 | { | ||
| 1428 | return 0; | ||
| 1429 | } | ||
| 1430 | #endif | ||
| 1431 | |||
| 1432 | #ifdef CONFIG_SMP | ||
| 1433 | static unsigned long source_load(int cpu, int type); | ||
| 1434 | static unsigned long target_load(int cpu, int type); | ||
| 1435 | static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd); | ||
| 1436 | |||
| 1437 | static unsigned long cpu_avg_load_per_task(int cpu) | ||
| 1438 | { | ||
| 1439 | struct rq *rq = cpu_rq(cpu); | ||
| 1440 | |||
| 1441 | if (rq->nr_running) | ||
| 1442 | rq->avg_load_per_task = rq->load.weight / rq->nr_running; | ||
| 1443 | |||
| 1444 | return rq->avg_load_per_task; | ||
| 1428 | } | 1445 | } |
| 1429 | 1446 | ||
| 1447 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
| 1448 | |||
| 1430 | static void __set_se_shares(struct sched_entity *se, unsigned long shares); | 1449 | static void __set_se_shares(struct sched_entity *se, unsigned long shares); |
| 1431 | 1450 | ||
| 1432 | /* | 1451 | /* |
| @@ -1485,11 +1504,11 @@ __update_group_shares_cpu(struct task_group *tg, int cpu, | |||
| 1485 | * This needs to be done in a bottom-up fashion because the rq weight of a | 1504 | * This needs to be done in a bottom-up fashion because the rq weight of a |
| 1486 | * parent group depends on the shares of its child groups. | 1505 | * parent group depends on the shares of its child groups. |
| 1487 | */ | 1506 | */ |
| 1488 | static void | 1507 | static int tg_shares_up(struct task_group *tg, void *data) |
| 1489 | tg_shares_up(struct task_group *tg, int cpu, struct sched_domain *sd) | ||
| 1490 | { | 1508 | { |
| 1491 | unsigned long rq_weight = 0; | 1509 | unsigned long rq_weight = 0; |
| 1492 | unsigned long shares = 0; | 1510 | unsigned long shares = 0; |
| 1511 | struct sched_domain *sd = data; | ||
| 1493 | int i; | 1512 | int i; |
| 1494 | 1513 | ||
| 1495 | for_each_cpu_mask(i, sd->span) { | 1514 | for_each_cpu_mask(i, sd->span) { |
| @@ -1514,6 +1533,8 @@ tg_shares_up(struct task_group *tg, int cpu, struct sched_domain *sd) | |||
| 1514 | __update_group_shares_cpu(tg, i, shares, rq_weight); | 1533 | __update_group_shares_cpu(tg, i, shares, rq_weight); |
| 1515 | spin_unlock_irqrestore(&rq->lock, flags); | 1534 | spin_unlock_irqrestore(&rq->lock, flags); |
| 1516 | } | 1535 | } |
| 1536 | |||
| 1537 | return 0; | ||
| 1517 | } | 1538 | } |
| 1518 | 1539 | ||
| 1519 | /* | 1540 | /* |
| @@ -1521,10 +1542,10 @@ tg_shares_up(struct task_group *tg, int cpu, struct sched_domain *sd) | |||
| 1521 | * This needs to be done in a top-down fashion because the load of a child | 1542 | * This needs to be done in a top-down fashion because the load of a child |
| 1522 | * group is a fraction of its parents load. | 1543 | * group is a fraction of its parents load. |
| 1523 | */ | 1544 | */ |
| 1524 | static void | 1545 | static int tg_load_down(struct task_group *tg, void *data) |
| 1525 | tg_load_down(struct task_group *tg, int cpu, struct sched_domain *sd) | ||
| 1526 | { | 1546 | { |
| 1527 | unsigned long load; | 1547 | unsigned long load; |
| 1548 | long cpu = (long)data; | ||
| 1528 | 1549 | ||
| 1529 | if (!tg->parent) { | 1550 | if (!tg->parent) { |
| 1530 | load = cpu_rq(cpu)->load.weight; | 1551 | load = cpu_rq(cpu)->load.weight; |
| @@ -1535,11 +1556,8 @@ tg_load_down(struct task_group *tg, int cpu, struct sched_domain *sd) | |||
| 1535 | } | 1556 | } |
| 1536 | 1557 | ||
| 1537 | tg->cfs_rq[cpu]->h_load = load; | 1558 | tg->cfs_rq[cpu]->h_load = load; |
| 1538 | } | ||
| 1539 | 1559 | ||
| 1540 | static void | 1560 | return 0; |
| 1541 | tg_nop(struct task_group *tg, int cpu, struct sched_domain *sd) | ||
| 1542 | { | ||
| 1543 | } | 1561 | } |
| 1544 | 1562 | ||
| 1545 | static void update_shares(struct sched_domain *sd) | 1563 | static void update_shares(struct sched_domain *sd) |
| @@ -1549,7 +1567,7 @@ static void update_shares(struct sched_domain *sd) | |||
| 1549 | 1567 | ||
| 1550 | if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) { | 1568 | if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) { |
| 1551 | sd->last_update = now; | 1569 | sd->last_update = now; |
| 1552 | walk_tg_tree(tg_nop, tg_shares_up, 0, sd); | 1570 | walk_tg_tree(tg_nop, tg_shares_up, sd); |
| 1553 | } | 1571 | } |
| 1554 | } | 1572 | } |
| 1555 | 1573 | ||
| @@ -1560,9 +1578,9 @@ static void update_shares_locked(struct rq *rq, struct sched_domain *sd) | |||
| 1560 | spin_lock(&rq->lock); | 1578 | spin_lock(&rq->lock); |
| 1561 | } | 1579 | } |
| 1562 | 1580 | ||
| 1563 | static void update_h_load(int cpu) | 1581 | static void update_h_load(long cpu) |
| 1564 | { | 1582 | { |
| 1565 | walk_tg_tree(tg_load_down, tg_nop, cpu, NULL); | 1583 | walk_tg_tree(tg_load_down, tg_nop, (void *)cpu); |
| 1566 | } | 1584 | } |
| 1567 | 1585 | ||
| 1568 | #else | 1586 | #else |
| @@ -1920,11 +1938,8 @@ unsigned long wait_task_inactive(struct task_struct *p, long match_state) | |||
| 1920 | running = task_running(rq, p); | 1938 | running = task_running(rq, p); |
| 1921 | on_rq = p->se.on_rq; | 1939 | on_rq = p->se.on_rq; |
| 1922 | ncsw = 0; | 1940 | ncsw = 0; |
| 1923 | if (!match_state || p->state == match_state) { | 1941 | if (!match_state || p->state == match_state) |
| 1924 | ncsw = p->nivcsw + p->nvcsw; | 1942 | ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ |
| 1925 | if (unlikely(!ncsw)) | ||
| 1926 | ncsw = 1; | ||
| 1927 | } | ||
| 1928 | task_rq_unlock(rq, &flags); | 1943 | task_rq_unlock(rq, &flags); |
| 1929 | 1944 | ||
| 1930 | /* | 1945 | /* |
| @@ -2284,7 +2299,7 @@ out_running: | |||
| 2284 | trace_mark(kernel_sched_wakeup, | 2299 | trace_mark(kernel_sched_wakeup, |
| 2285 | "pid %d state %ld ## rq %p task %p rq->curr %p", | 2300 | "pid %d state %ld ## rq %p task %p rq->curr %p", |
| 2286 | p->pid, p->state, rq, p, rq->curr); | 2301 | p->pid, p->state, rq, p, rq->curr); |
| 2287 | check_preempt_curr(rq, p); | 2302 | check_preempt_curr(rq, p, sync); |
| 2288 | 2303 | ||
| 2289 | p->state = TASK_RUNNING; | 2304 | p->state = TASK_RUNNING; |
| 2290 | #ifdef CONFIG_SMP | 2305 | #ifdef CONFIG_SMP |
| @@ -2419,7 +2434,7 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags) | |||
| 2419 | trace_mark(kernel_sched_wakeup_new, | 2434 | trace_mark(kernel_sched_wakeup_new, |
| 2420 | "pid %d state %ld ## rq %p task %p rq->curr %p", | 2435 | "pid %d state %ld ## rq %p task %p rq->curr %p", |
| 2421 | p->pid, p->state, rq, p, rq->curr); | 2436 | p->pid, p->state, rq, p, rq->curr); |
| 2422 | check_preempt_curr(rq, p); | 2437 | check_preempt_curr(rq, p, 0); |
| 2423 | #ifdef CONFIG_SMP | 2438 | #ifdef CONFIG_SMP |
| 2424 | if (p->sched_class->task_wake_up) | 2439 | if (p->sched_class->task_wake_up) |
| 2425 | p->sched_class->task_wake_up(rq, p); | 2440 | p->sched_class->task_wake_up(rq, p); |
| @@ -2879,7 +2894,7 @@ static void pull_task(struct rq *src_rq, struct task_struct *p, | |||
| 2879 | * Note that idle threads have a prio of MAX_PRIO, for this test | 2894 | * Note that idle threads have a prio of MAX_PRIO, for this test |
| 2880 | * to be always true for them. | 2895 | * to be always true for them. |
| 2881 | */ | 2896 | */ |
| 2882 | check_preempt_curr(this_rq, p); | 2897 | check_preempt_curr(this_rq, p, 0); |
| 2883 | } | 2898 | } |
| 2884 | 2899 | ||
| 2885 | /* | 2900 | /* |
| @@ -4626,6 +4641,15 @@ __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) | |||
| 4626 | } | 4641 | } |
| 4627 | EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ | 4642 | EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ |
| 4628 | 4643 | ||
| 4644 | /** | ||
| 4645 | * complete: - signals a single thread waiting on this completion | ||
| 4646 | * @x: holds the state of this particular completion | ||
| 4647 | * | ||
| 4648 | * This will wake up a single thread waiting on this completion. Threads will be | ||
| 4649 | * awakened in the same order in which they were queued. | ||
| 4650 | * | ||
| 4651 | * See also complete_all(), wait_for_completion() and related routines. | ||
| 4652 | */ | ||
| 4629 | void complete(struct completion *x) | 4653 | void complete(struct completion *x) |
| 4630 | { | 4654 | { |
| 4631 | unsigned long flags; | 4655 | unsigned long flags; |
| @@ -4637,6 +4661,12 @@ void complete(struct completion *x) | |||
| 4637 | } | 4661 | } |
| 4638 | EXPORT_SYMBOL(complete); | 4662 | EXPORT_SYMBOL(complete); |
| 4639 | 4663 | ||
| 4664 | /** | ||
| 4665 | * complete_all: - signals all threads waiting on this completion | ||
| 4666 | * @x: holds the state of this particular completion | ||
| 4667 | * | ||
| 4668 | * This will wake up all threads waiting on this particular completion event. | ||
| 4669 | */ | ||
| 4640 | void complete_all(struct completion *x) | 4670 | void complete_all(struct completion *x) |
| 4641 | { | 4671 | { |
| 4642 | unsigned long flags; | 4672 | unsigned long flags; |
| @@ -4657,10 +4687,7 @@ do_wait_for_common(struct completion *x, long timeout, int state) | |||
| 4657 | wait.flags |= WQ_FLAG_EXCLUSIVE; | 4687 | wait.flags |= WQ_FLAG_EXCLUSIVE; |
| 4658 | __add_wait_queue_tail(&x->wait, &wait); | 4688 | __add_wait_queue_tail(&x->wait, &wait); |
| 4659 | do { | 4689 | do { |
| 4660 | if ((state == TASK_INTERRUPTIBLE && | 4690 | if (signal_pending_state(state, current)) { |
| 4661 | signal_pending(current)) || | ||
| 4662 | (state == TASK_KILLABLE && | ||
| 4663 | fatal_signal_pending(current))) { | ||
| 4664 | timeout = -ERESTARTSYS; | 4691 | timeout = -ERESTARTSYS; |
| 4665 | break; | 4692 | break; |
| 4666 | } | 4693 | } |
| @@ -4688,12 +4715,31 @@ wait_for_common(struct completion *x, long timeout, int state) | |||
| 4688 | return timeout; | 4715 | return timeout; |
| 4689 | } | 4716 | } |
| 4690 | 4717 | ||
| 4718 | /** | ||
| 4719 | * wait_for_completion: - waits for completion of a task | ||
| 4720 | * @x: holds the state of this particular completion | ||
| 4721 | * | ||
| 4722 | * This waits to be signaled for completion of a specific task. It is NOT | ||
| 4723 | * interruptible and there is no timeout. | ||
| 4724 | * | ||
| 4725 | * See also similar routines (i.e. wait_for_completion_timeout()) with timeout | ||
| 4726 | * and interrupt capability. Also see complete(). | ||
| 4727 | */ | ||
| 4691 | void __sched wait_for_completion(struct completion *x) | 4728 | void __sched wait_for_completion(struct completion *x) |
| 4692 | { | 4729 | { |
| 4693 | wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); | 4730 | wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); |
| 4694 | } | 4731 | } |
| 4695 | EXPORT_SYMBOL(wait_for_completion); | 4732 | EXPORT_SYMBOL(wait_for_completion); |
| 4696 | 4733 | ||
| 4734 | /** | ||
| 4735 | * wait_for_completion_timeout: - waits for completion of a task (w/timeout) | ||
| 4736 | * @x: holds the state of this particular completion | ||
| 4737 | * @timeout: timeout value in jiffies | ||
| 4738 | * | ||
| 4739 | * This waits for either a completion of a specific task to be signaled or for a | ||
| 4740 | * specified timeout to expire. The timeout is in jiffies. It is not | ||
| 4741 | * interruptible. | ||
| 4742 | */ | ||
| 4697 | unsigned long __sched | 4743 | unsigned long __sched |
| 4698 | wait_for_completion_timeout(struct completion *x, unsigned long timeout) | 4744 | wait_for_completion_timeout(struct completion *x, unsigned long timeout) |
| 4699 | { | 4745 | { |
| @@ -4701,6 +4747,13 @@ wait_for_completion_timeout(struct completion *x, unsigned long timeout) | |||
| 4701 | } | 4747 | } |
| 4702 | EXPORT_SYMBOL(wait_for_completion_timeout); | 4748 | EXPORT_SYMBOL(wait_for_completion_timeout); |
| 4703 | 4749 | ||
| 4750 | /** | ||
| 4751 | * wait_for_completion_interruptible: - waits for completion of a task (w/intr) | ||
| 4752 | * @x: holds the state of this particular completion | ||
| 4753 | * | ||
| 4754 | * This waits for completion of a specific task to be signaled. It is | ||
| 4755 | * interruptible. | ||
| 4756 | */ | ||
| 4704 | int __sched wait_for_completion_interruptible(struct completion *x) | 4757 | int __sched wait_for_completion_interruptible(struct completion *x) |
| 4705 | { | 4758 | { |
| 4706 | long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE); | 4759 | long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE); |
| @@ -4710,6 +4763,14 @@ int __sched wait_for_completion_interruptible(struct completion *x) | |||
| 4710 | } | 4763 | } |
| 4711 | EXPORT_SYMBOL(wait_for_completion_interruptible); | 4764 | EXPORT_SYMBOL(wait_for_completion_interruptible); |
| 4712 | 4765 | ||
| 4766 | /** | ||
| 4767 | * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr)) | ||
| 4768 | * @x: holds the state of this particular completion | ||
| 4769 | * @timeout: timeout value in jiffies | ||
| 4770 | * | ||
| 4771 | * This waits for either a completion of a specific task to be signaled or for a | ||
| 4772 | * specified timeout to expire. It is interruptible. The timeout is in jiffies. | ||
| 4773 | */ | ||
| 4713 | unsigned long __sched | 4774 | unsigned long __sched |
| 4714 | wait_for_completion_interruptible_timeout(struct completion *x, | 4775 | wait_for_completion_interruptible_timeout(struct completion *x, |
| 4715 | unsigned long timeout) | 4776 | unsigned long timeout) |
| @@ -4718,6 +4779,13 @@ wait_for_completion_interruptible_timeout(struct completion *x, | |||
| 4718 | } | 4779 | } |
| 4719 | EXPORT_SYMBOL(wait_for_completion_interruptible_timeout); | 4780 | EXPORT_SYMBOL(wait_for_completion_interruptible_timeout); |
| 4720 | 4781 | ||
| 4782 | /** | ||
| 4783 | * wait_for_completion_killable: - waits for completion of a task (killable) | ||
| 4784 | * @x: holds the state of this particular completion | ||
| 4785 | * | ||
| 4786 | * This waits to be signaled for completion of a specific task. It can be | ||
| 4787 | * interrupted by a kill signal. | ||
| 4788 | */ | ||
| 4721 | int __sched wait_for_completion_killable(struct completion *x) | 4789 | int __sched wait_for_completion_killable(struct completion *x) |
| 4722 | { | 4790 | { |
| 4723 | long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE); | 4791 | long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE); |
| @@ -5120,7 +5188,8 @@ recheck: | |||
| 5120 | * Do not allow realtime tasks into groups that have no runtime | 5188 | * Do not allow realtime tasks into groups that have no runtime |
| 5121 | * assigned. | 5189 | * assigned. |
| 5122 | */ | 5190 | */ |
| 5123 | if (rt_policy(policy) && task_group(p)->rt_bandwidth.rt_runtime == 0) | 5191 | if (rt_bandwidth_enabled() && rt_policy(policy) && |
| 5192 | task_group(p)->rt_bandwidth.rt_runtime == 0) | ||
| 5124 | return -EPERM; | 5193 | return -EPERM; |
| 5125 | #endif | 5194 | #endif |
| 5126 | 5195 | ||
| @@ -5956,7 +6025,7 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) | |||
| 5956 | set_task_cpu(p, dest_cpu); | 6025 | set_task_cpu(p, dest_cpu); |
| 5957 | if (on_rq) { | 6026 | if (on_rq) { |
| 5958 | activate_task(rq_dest, p, 0); | 6027 | activate_task(rq_dest, p, 0); |
| 5959 | check_preempt_curr(rq_dest, p); | 6028 | check_preempt_curr(rq_dest, p, 0); |
| 5960 | } | 6029 | } |
| 5961 | done: | 6030 | done: |
| 5962 | ret = 1; | 6031 | ret = 1; |
| @@ -6281,7 +6350,7 @@ set_table_entry(struct ctl_table *entry, | |||
| 6281 | static struct ctl_table * | 6350 | static struct ctl_table * |
| 6282 | sd_alloc_ctl_domain_table(struct sched_domain *sd) | 6351 | sd_alloc_ctl_domain_table(struct sched_domain *sd) |
| 6283 | { | 6352 | { |
| 6284 | struct ctl_table *table = sd_alloc_ctl_entry(12); | 6353 | struct ctl_table *table = sd_alloc_ctl_entry(13); |
| 6285 | 6354 | ||
| 6286 | if (table == NULL) | 6355 | if (table == NULL) |
| 6287 | return NULL; | 6356 | return NULL; |
| @@ -6309,7 +6378,9 @@ sd_alloc_ctl_domain_table(struct sched_domain *sd) | |||
| 6309 | sizeof(int), 0644, proc_dointvec_minmax); | 6378 | sizeof(int), 0644, proc_dointvec_minmax); |
| 6310 | set_table_entry(&table[10], "flags", &sd->flags, | 6379 | set_table_entry(&table[10], "flags", &sd->flags, |
| 6311 | sizeof(int), 0644, proc_dointvec_minmax); | 6380 | sizeof(int), 0644, proc_dointvec_minmax); |
| 6312 | /* &table[11] is terminator */ | 6381 | set_table_entry(&table[11], "name", sd->name, |
| 6382 | CORENAME_MAX_SIZE, 0444, proc_dostring); | ||
| 6383 | /* &table[12] is terminator */ | ||
| 6313 | 6384 | ||
| 6314 | return table; | 6385 | return table; |
| 6315 | } | 6386 | } |
| @@ -7193,13 +7264,21 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd) | |||
| 7193 | * Non-inlined to reduce accumulated stack pressure in build_sched_domains() | 7264 | * Non-inlined to reduce accumulated stack pressure in build_sched_domains() |
| 7194 | */ | 7265 | */ |
| 7195 | 7266 | ||
| 7267 | #ifdef CONFIG_SCHED_DEBUG | ||
| 7268 | # define SD_INIT_NAME(sd, type) sd->name = #type | ||
| 7269 | #else | ||
| 7270 | # define SD_INIT_NAME(sd, type) do { } while (0) | ||
| 7271 | #endif | ||
| 7272 | |||
| 7196 | #define SD_INIT(sd, type) sd_init_##type(sd) | 7273 | #define SD_INIT(sd, type) sd_init_##type(sd) |
| 7274 | |||
| 7197 | #define SD_INIT_FUNC(type) \ | 7275 | #define SD_INIT_FUNC(type) \ |
| 7198 | static noinline void sd_init_##type(struct sched_domain *sd) \ | 7276 | static noinline void sd_init_##type(struct sched_domain *sd) \ |
| 7199 | { \ | 7277 | { \ |
| 7200 | memset(sd, 0, sizeof(*sd)); \ | 7278 | memset(sd, 0, sizeof(*sd)); \ |
| 7201 | *sd = SD_##type##_INIT; \ | 7279 | *sd = SD_##type##_INIT; \ |
| 7202 | sd->level = SD_LV_##type; \ | 7280 | sd->level = SD_LV_##type; \ |
| 7281 | SD_INIT_NAME(sd, type); \ | ||
| 7203 | } | 7282 | } |
| 7204 | 7283 | ||
| 7205 | SD_INIT_FUNC(CPU) | 7284 | SD_INIT_FUNC(CPU) |
| @@ -7695,24 +7774,27 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur, | |||
| 7695 | * and partition_sched_domains() will fallback to the single partition | 7774 | * and partition_sched_domains() will fallback to the single partition |
| 7696 | * 'fallback_doms', it also forces the domains to be rebuilt. | 7775 | * 'fallback_doms', it also forces the domains to be rebuilt. |
| 7697 | * | 7776 | * |
| 7777 | * If doms_new==NULL it will be replaced with cpu_online_map. | ||
| 7778 | * ndoms_new==0 is a special case for destroying existing domains. | ||
| 7779 | * It will not create the default domain. | ||
| 7780 | * | ||
| 7698 | * Call with hotplug lock held | 7781 | * Call with hotplug lock held |
| 7699 | */ | 7782 | */ |
| 7700 | void partition_sched_domains(int ndoms_new, cpumask_t *doms_new, | 7783 | void partition_sched_domains(int ndoms_new, cpumask_t *doms_new, |
| 7701 | struct sched_domain_attr *dattr_new) | 7784 | struct sched_domain_attr *dattr_new) |
| 7702 | { | 7785 | { |
| 7703 | int i, j; | 7786 | int i, j, n; |
| 7704 | 7787 | ||
| 7705 | mutex_lock(&sched_domains_mutex); | 7788 | mutex_lock(&sched_domains_mutex); |
| 7706 | 7789 | ||
| 7707 | /* always unregister in case we don't destroy any domains */ | 7790 | /* always unregister in case we don't destroy any domains */ |
| 7708 | unregister_sched_domain_sysctl(); | 7791 | unregister_sched_domain_sysctl(); |
| 7709 | 7792 | ||
| 7710 | if (doms_new == NULL) | 7793 | n = doms_new ? ndoms_new : 0; |
| 7711 | ndoms_new = 0; | ||
| 7712 | 7794 | ||
| 7713 | /* Destroy deleted domains */ | 7795 | /* Destroy deleted domains */ |
| 7714 | for (i = 0; i < ndoms_cur; i++) { | 7796 | for (i = 0; i < ndoms_cur; i++) { |
| 7715 | for (j = 0; j < ndoms_new; j++) { | 7797 | for (j = 0; j < n; j++) { |
| 7716 | if (cpus_equal(doms_cur[i], doms_new[j]) | 7798 | if (cpus_equal(doms_cur[i], doms_new[j]) |
| 7717 | && dattrs_equal(dattr_cur, i, dattr_new, j)) | 7799 | && dattrs_equal(dattr_cur, i, dattr_new, j)) |
| 7718 | goto match1; | 7800 | goto match1; |
| @@ -7725,7 +7807,6 @@ match1: | |||
| 7725 | 7807 | ||
| 7726 | if (doms_new == NULL) { | 7808 | if (doms_new == NULL) { |
| 7727 | ndoms_cur = 0; | 7809 | ndoms_cur = 0; |
| 7728 | ndoms_new = 1; | ||
| 7729 | doms_new = &fallback_doms; | 7810 | doms_new = &fallback_doms; |
| 7730 | cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map); | 7811 | cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map); |
| 7731 | dattr_new = NULL; | 7812 | dattr_new = NULL; |
| @@ -7762,8 +7843,13 @@ match2: | |||
| 7762 | int arch_reinit_sched_domains(void) | 7843 | int arch_reinit_sched_domains(void) |
| 7763 | { | 7844 | { |
| 7764 | get_online_cpus(); | 7845 | get_online_cpus(); |
| 7846 | |||
| 7847 | /* Destroy domains first to force the rebuild */ | ||
| 7848 | partition_sched_domains(0, NULL, NULL); | ||
| 7849 | |||
| 7765 | rebuild_sched_domains(); | 7850 | rebuild_sched_domains(); |
| 7766 | put_online_cpus(); | 7851 | put_online_cpus(); |
| 7852 | |||
| 7767 | return 0; | 7853 | return 0; |
| 7768 | } | 7854 | } |
| 7769 | 7855 | ||
| @@ -7847,7 +7933,7 @@ static int update_sched_domains(struct notifier_block *nfb, | |||
| 7847 | case CPU_ONLINE_FROZEN: | 7933 | case CPU_ONLINE_FROZEN: |
| 7848 | case CPU_DEAD: | 7934 | case CPU_DEAD: |
| 7849 | case CPU_DEAD_FROZEN: | 7935 | case CPU_DEAD_FROZEN: |
| 7850 | partition_sched_domains(0, NULL, NULL); | 7936 | partition_sched_domains(1, NULL, NULL); |
| 7851 | return NOTIFY_OK; | 7937 | return NOTIFY_OK; |
| 7852 | 7938 | ||
| 7853 | default: | 7939 | default: |
| @@ -8234,20 +8320,25 @@ void __might_sleep(char *file, int line) | |||
| 8234 | #ifdef in_atomic | 8320 | #ifdef in_atomic |
| 8235 | static unsigned long prev_jiffy; /* ratelimiting */ | 8321 | static unsigned long prev_jiffy; /* ratelimiting */ |
| 8236 | 8322 | ||
| 8237 | if ((in_atomic() || irqs_disabled()) && | 8323 | if ((!in_atomic() && !irqs_disabled()) || |
| 8238 | system_state == SYSTEM_RUNNING && !oops_in_progress) { | 8324 | system_state != SYSTEM_RUNNING || oops_in_progress) |
| 8239 | if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) | 8325 | return; |
| 8240 | return; | 8326 | if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) |
| 8241 | prev_jiffy = jiffies; | 8327 | return; |
| 8242 | printk(KERN_ERR "BUG: sleeping function called from invalid" | 8328 | prev_jiffy = jiffies; |
| 8243 | " context at %s:%d\n", file, line); | 8329 | |
| 8244 | printk("in_atomic():%d, irqs_disabled():%d\n", | 8330 | printk(KERN_ERR |
| 8245 | in_atomic(), irqs_disabled()); | 8331 | "BUG: sleeping function called from invalid context at %s:%d\n", |
| 8246 | debug_show_held_locks(current); | 8332 | file, line); |
| 8247 | if (irqs_disabled()) | 8333 | printk(KERN_ERR |
| 8248 | print_irqtrace_events(current); | 8334 | "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", |
| 8249 | dump_stack(); | 8335 | in_atomic(), irqs_disabled(), |
| 8250 | } | 8336 | current->pid, current->comm); |
| 8337 | |||
| 8338 | debug_show_held_locks(current); | ||
| 8339 | if (irqs_disabled()) | ||
| 8340 | print_irqtrace_events(current); | ||
| 8341 | dump_stack(); | ||
| 8251 | #endif | 8342 | #endif |
| 8252 | } | 8343 | } |
| 8253 | EXPORT_SYMBOL(__might_sleep); | 8344 | EXPORT_SYMBOL(__might_sleep); |
| @@ -8745,73 +8836,95 @@ static DEFINE_MUTEX(rt_constraints_mutex); | |||
| 8745 | static unsigned long to_ratio(u64 period, u64 runtime) | 8836 | static unsigned long to_ratio(u64 period, u64 runtime) |
| 8746 | { | 8837 | { |
| 8747 | if (runtime == RUNTIME_INF) | 8838 | if (runtime == RUNTIME_INF) |
| 8748 | return 1ULL << 16; | 8839 | return 1ULL << 20; |
| 8749 | 8840 | ||
| 8750 | return div64_u64(runtime << 16, period); | 8841 | return div64_u64(runtime << 20, period); |
| 8751 | } | 8842 | } |
| 8752 | 8843 | ||
| 8753 | #ifdef CONFIG_CGROUP_SCHED | 8844 | /* Must be called with tasklist_lock held */ |
| 8754 | static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime) | 8845 | static inline int tg_has_rt_tasks(struct task_group *tg) |
| 8755 | { | 8846 | { |
| 8756 | struct task_group *tgi, *parent = tg->parent; | 8847 | struct task_struct *g, *p; |
| 8757 | unsigned long total = 0; | ||
| 8758 | 8848 | ||
| 8759 | if (!parent) { | 8849 | do_each_thread(g, p) { |
| 8760 | if (global_rt_period() < period) | 8850 | if (rt_task(p) && rt_rq_of_se(&p->rt)->tg == tg) |
| 8761 | return 0; | 8851 | return 1; |
| 8852 | } while_each_thread(g, p); | ||
| 8762 | 8853 | ||
| 8763 | return to_ratio(period, runtime) < | 8854 | return 0; |
| 8764 | to_ratio(global_rt_period(), global_rt_runtime()); | 8855 | } |
| 8765 | } | ||
| 8766 | 8856 | ||
| 8767 | if (ktime_to_ns(parent->rt_bandwidth.rt_period) < period) | 8857 | struct rt_schedulable_data { |
| 8768 | return 0; | 8858 | struct task_group *tg; |
| 8859 | u64 rt_period; | ||
| 8860 | u64 rt_runtime; | ||
| 8861 | }; | ||
| 8769 | 8862 | ||
| 8770 | rcu_read_lock(); | 8863 | static int tg_schedulable(struct task_group *tg, void *data) |
| 8771 | list_for_each_entry_rcu(tgi, &parent->children, siblings) { | 8864 | { |
| 8772 | if (tgi == tg) | 8865 | struct rt_schedulable_data *d = data; |
| 8773 | continue; | 8866 | struct task_group *child; |
| 8867 | unsigned long total, sum = 0; | ||
| 8868 | u64 period, runtime; | ||
| 8869 | |||
| 8870 | period = ktime_to_ns(tg->rt_bandwidth.rt_period); | ||
| 8871 | runtime = tg->rt_bandwidth.rt_runtime; | ||
| 8774 | 8872 | ||
| 8775 | total += to_ratio(ktime_to_ns(tgi->rt_bandwidth.rt_period), | 8873 | if (tg == d->tg) { |
| 8776 | tgi->rt_bandwidth.rt_runtime); | 8874 | period = d->rt_period; |
| 8875 | runtime = d->rt_runtime; | ||
| 8777 | } | 8876 | } |
| 8778 | rcu_read_unlock(); | ||
| 8779 | 8877 | ||
| 8780 | return total + to_ratio(period, runtime) <= | 8878 | /* |
| 8781 | to_ratio(ktime_to_ns(parent->rt_bandwidth.rt_period), | 8879 | * Cannot have more runtime than the period. |
| 8782 | parent->rt_bandwidth.rt_runtime); | 8880 | */ |
| 8783 | } | 8881 | if (runtime > period && runtime != RUNTIME_INF) |
| 8784 | #elif defined CONFIG_USER_SCHED | 8882 | return -EINVAL; |
| 8785 | static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime) | ||
| 8786 | { | ||
| 8787 | struct task_group *tgi; | ||
| 8788 | unsigned long total = 0; | ||
| 8789 | unsigned long global_ratio = | ||
| 8790 | to_ratio(global_rt_period(), global_rt_runtime()); | ||
| 8791 | 8883 | ||
| 8792 | rcu_read_lock(); | 8884 | /* |
| 8793 | list_for_each_entry_rcu(tgi, &task_groups, list) { | 8885 | * Ensure we don't starve existing RT tasks. |
| 8794 | if (tgi == tg) | 8886 | */ |
| 8795 | continue; | 8887 | if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg)) |
| 8888 | return -EBUSY; | ||
| 8796 | 8889 | ||
| 8797 | total += to_ratio(ktime_to_ns(tgi->rt_bandwidth.rt_period), | 8890 | total = to_ratio(period, runtime); |
| 8798 | tgi->rt_bandwidth.rt_runtime); | 8891 | |
| 8892 | /* | ||
| 8893 | * Nobody can have more than the global setting allows. | ||
| 8894 | */ | ||
| 8895 | if (total > to_ratio(global_rt_period(), global_rt_runtime())) | ||
| 8896 | return -EINVAL; | ||
| 8897 | |||
| 8898 | /* | ||
| 8899 | * The sum of our children's runtime should not exceed our own. | ||
| 8900 | */ | ||
| 8901 | list_for_each_entry_rcu(child, &tg->children, siblings) { | ||
| 8902 | period = ktime_to_ns(child->rt_bandwidth.rt_period); | ||
| 8903 | runtime = child->rt_bandwidth.rt_runtime; | ||
| 8904 | |||
| 8905 | if (child == d->tg) { | ||
| 8906 | period = d->rt_period; | ||
| 8907 | runtime = d->rt_runtime; | ||
| 8908 | } | ||
| 8909 | |||
| 8910 | sum += to_ratio(period, runtime); | ||
| 8799 | } | 8911 | } |
| 8800 | rcu_read_unlock(); | ||
| 8801 | 8912 | ||
| 8802 | return total + to_ratio(period, runtime) < global_ratio; | 8913 | if (sum > total) |
| 8914 | return -EINVAL; | ||
| 8915 | |||
| 8916 | return 0; | ||
| 8803 | } | 8917 | } |
| 8804 | #endif | ||
| 8805 | 8918 | ||
| 8806 | /* Must be called with tasklist_lock held */ | 8919 | static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime) |
| 8807 | static inline int tg_has_rt_tasks(struct task_group *tg) | ||
| 8808 | { | 8920 | { |
| 8809 | struct task_struct *g, *p; | 8921 | struct rt_schedulable_data data = { |
| 8810 | do_each_thread(g, p) { | 8922 | .tg = tg, |
| 8811 | if (rt_task(p) && rt_rq_of_se(&p->rt)->tg == tg) | 8923 | .rt_period = period, |
| 8812 | return 1; | 8924 | .rt_runtime = runtime, |
| 8813 | } while_each_thread(g, p); | 8925 | }; |
| 8814 | return 0; | 8926 | |
| 8927 | return walk_tg_tree(tg_schedulable, tg_nop, &data); | ||
| 8815 | } | 8928 | } |
| 8816 | 8929 | ||
| 8817 | static int tg_set_bandwidth(struct task_group *tg, | 8930 | static int tg_set_bandwidth(struct task_group *tg, |
| @@ -8821,14 +8934,9 @@ static int tg_set_bandwidth(struct task_group *tg, | |||
| 8821 | 8934 | ||
| 8822 | mutex_lock(&rt_constraints_mutex); | 8935 | mutex_lock(&rt_constraints_mutex); |
| 8823 | read_lock(&tasklist_lock); | 8936 | read_lock(&tasklist_lock); |
| 8824 | if (rt_runtime == 0 && tg_has_rt_tasks(tg)) { | 8937 | err = __rt_schedulable(tg, rt_period, rt_runtime); |
| 8825 | err = -EBUSY; | 8938 | if (err) |
| 8826 | goto unlock; | ||
| 8827 | } | ||
| 8828 | if (!__rt_schedulable(tg, rt_period, rt_runtime)) { | ||
| 8829 | err = -EINVAL; | ||
| 8830 | goto unlock; | 8939 | goto unlock; |
| 8831 | } | ||
| 8832 | 8940 | ||
| 8833 | spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock); | 8941 | spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock); |
| 8834 | tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period); | 8942 | tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period); |
| @@ -8897,16 +9005,25 @@ long sched_group_rt_period(struct task_group *tg) | |||
| 8897 | 9005 | ||
| 8898 | static int sched_rt_global_constraints(void) | 9006 | static int sched_rt_global_constraints(void) |
| 8899 | { | 9007 | { |
| 8900 | struct task_group *tg = &root_task_group; | 9008 | u64 runtime, period; |
| 8901 | u64 rt_runtime, rt_period; | ||
| 8902 | int ret = 0; | 9009 | int ret = 0; |
| 8903 | 9010 | ||
| 8904 | rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period); | 9011 | if (sysctl_sched_rt_period <= 0) |
| 8905 | rt_runtime = tg->rt_bandwidth.rt_runtime; | 9012 | return -EINVAL; |
| 9013 | |||
| 9014 | runtime = global_rt_runtime(); | ||
| 9015 | period = global_rt_period(); | ||
| 9016 | |||
| 9017 | /* | ||
| 9018 | * Sanity check on the sysctl variables. | ||
| 9019 | */ | ||
| 9020 | if (runtime > period && runtime != RUNTIME_INF) | ||
| 9021 | return -EINVAL; | ||
| 8906 | 9022 | ||
| 8907 | mutex_lock(&rt_constraints_mutex); | 9023 | mutex_lock(&rt_constraints_mutex); |
| 8908 | if (!__rt_schedulable(tg, rt_period, rt_runtime)) | 9024 | read_lock(&tasklist_lock); |
| 8909 | ret = -EINVAL; | 9025 | ret = __rt_schedulable(NULL, 0, 0); |
| 9026 | read_unlock(&tasklist_lock); | ||
| 8910 | mutex_unlock(&rt_constraints_mutex); | 9027 | mutex_unlock(&rt_constraints_mutex); |
| 8911 | 9028 | ||
| 8912 | return ret; | 9029 | return ret; |
| @@ -8917,6 +9034,9 @@ static int sched_rt_global_constraints(void) | |||
| 8917 | unsigned long flags; | 9034 | unsigned long flags; |
| 8918 | int i; | 9035 | int i; |
| 8919 | 9036 | ||
| 9037 | if (sysctl_sched_rt_period <= 0) | ||
| 9038 | return -EINVAL; | ||
| 9039 | |||
| 8920 | spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); | 9040 | spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); |
| 8921 | for_each_possible_cpu(i) { | 9041 | for_each_possible_cpu(i) { |
| 8922 | struct rt_rq *rt_rq = &cpu_rq(i)->rt; | 9042 | struct rt_rq *rt_rq = &cpu_rq(i)->rt; |
| @@ -8977,7 +9097,6 @@ cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp) | |||
| 8977 | 9097 | ||
| 8978 | if (!cgrp->parent) { | 9098 | if (!cgrp->parent) { |
| 8979 | /* This is early initialization for the top cgroup */ | 9099 | /* This is early initialization for the top cgroup */ |
| 8980 | init_task_group.css.cgroup = cgrp; | ||
| 8981 | return &init_task_group.css; | 9100 | return &init_task_group.css; |
| 8982 | } | 9101 | } |
| 8983 | 9102 | ||
| @@ -8986,9 +9105,6 @@ cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp) | |||
| 8986 | if (IS_ERR(tg)) | 9105 | if (IS_ERR(tg)) |
| 8987 | return ERR_PTR(-ENOMEM); | 9106 | return ERR_PTR(-ENOMEM); |
| 8988 | 9107 | ||
| 8989 | /* Bind the cgroup to task_group object we just created */ | ||
| 8990 | tg->css.cgroup = cgrp; | ||
| 8991 | |||
| 8992 | return &tg->css; | 9108 | return &tg->css; |
| 8993 | } | 9109 | } |
| 8994 | 9110 | ||
diff --git a/kernel/sched_clock.c b/kernel/sched_clock.c index e8ab096ddfe3..81787248b60f 100644 --- a/kernel/sched_clock.c +++ b/kernel/sched_clock.c | |||
| @@ -118,13 +118,13 @@ static u64 __update_sched_clock(struct sched_clock_data *scd, u64 now) | |||
| 118 | 118 | ||
| 119 | /* | 119 | /* |
| 120 | * scd->clock = clamp(scd->tick_gtod + delta, | 120 | * scd->clock = clamp(scd->tick_gtod + delta, |
| 121 | * max(scd->tick_gtod, scd->clock), | 121 | * max(scd->tick_gtod, scd->clock), |
| 122 | * scd->tick_gtod + TICK_NSEC); | 122 | * max(scd->clock, scd->tick_gtod + TICK_NSEC)); |
| 123 | */ | 123 | */ |
| 124 | 124 | ||
| 125 | clock = scd->tick_gtod + delta; | 125 | clock = scd->tick_gtod + delta; |
| 126 | min_clock = wrap_max(scd->tick_gtod, scd->clock); | 126 | min_clock = wrap_max(scd->tick_gtod, scd->clock); |
| 127 | max_clock = scd->tick_gtod + TICK_NSEC; | 127 | max_clock = wrap_max(scd->clock, scd->tick_gtod + TICK_NSEC); |
| 128 | 128 | ||
| 129 | clock = wrap_max(clock, min_clock); | 129 | clock = wrap_max(clock, min_clock); |
| 130 | clock = wrap_min(clock, max_clock); | 130 | clock = wrap_min(clock, max_clock); |
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c index bbe6b31c3c56..ad958c1ec708 100644 --- a/kernel/sched_debug.c +++ b/kernel/sched_debug.c | |||
| @@ -333,12 +333,10 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) | |||
| 333 | unsigned long flags; | 333 | unsigned long flags; |
| 334 | int num_threads = 1; | 334 | int num_threads = 1; |
| 335 | 335 | ||
| 336 | rcu_read_lock(); | ||
| 337 | if (lock_task_sighand(p, &flags)) { | 336 | if (lock_task_sighand(p, &flags)) { |
| 338 | num_threads = atomic_read(&p->signal->count); | 337 | num_threads = atomic_read(&p->signal->count); |
| 339 | unlock_task_sighand(p, &flags); | 338 | unlock_task_sighand(p, &flags); |
| 340 | } | 339 | } |
| 341 | rcu_read_unlock(); | ||
| 342 | 340 | ||
| 343 | SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, p->pid, num_threads); | 341 | SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, p->pid, num_threads); |
| 344 | SEQ_printf(m, | 342 | SEQ_printf(m, |
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index fb8994c6d4bb..18fd17172eb6 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c | |||
| @@ -409,64 +409,6 @@ static u64 sched_vslice_add(struct cfs_rq *cfs_rq, struct sched_entity *se) | |||
| 409 | } | 409 | } |
| 410 | 410 | ||
| 411 | /* | 411 | /* |
| 412 | * The goal of calc_delta_asym() is to be asymmetrically around NICE_0_LOAD, in | ||
| 413 | * that it favours >=0 over <0. | ||
| 414 | * | ||
| 415 | * -20 | | ||
| 416 | * | | ||
| 417 | * 0 --------+------- | ||
| 418 | * .' | ||
| 419 | * 19 .' | ||
| 420 | * | ||
| 421 | */ | ||
| 422 | static unsigned long | ||
| 423 | calc_delta_asym(unsigned long delta, struct sched_entity *se) | ||
| 424 | { | ||
| 425 | struct load_weight lw = { | ||
| 426 | .weight = NICE_0_LOAD, | ||
| 427 | .inv_weight = 1UL << (WMULT_SHIFT-NICE_0_SHIFT) | ||
| 428 | }; | ||
| 429 | |||
| 430 | for_each_sched_entity(se) { | ||
| 431 | struct load_weight *se_lw = &se->load; | ||
| 432 | unsigned long rw = cfs_rq_of(se)->load.weight; | ||
| 433 | |||
| 434 | #ifdef CONFIG_FAIR_SCHED_GROUP | ||
| 435 | struct cfs_rq *cfs_rq = se->my_q; | ||
| 436 | struct task_group *tg = NULL | ||
| 437 | |||
| 438 | if (cfs_rq) | ||
| 439 | tg = cfs_rq->tg; | ||
| 440 | |||
| 441 | if (tg && tg->shares < NICE_0_LOAD) { | ||
| 442 | /* | ||
| 443 | * scale shares to what it would have been had | ||
| 444 | * tg->weight been NICE_0_LOAD: | ||
| 445 | * | ||
| 446 | * weight = 1024 * shares / tg->weight | ||
| 447 | */ | ||
| 448 | lw.weight *= se->load.weight; | ||
| 449 | lw.weight /= tg->shares; | ||
| 450 | |||
| 451 | lw.inv_weight = 0; | ||
| 452 | |||
| 453 | se_lw = &lw; | ||
| 454 | rw += lw.weight - se->load.weight; | ||
| 455 | } else | ||
| 456 | #endif | ||
| 457 | |||
| 458 | if (se->load.weight < NICE_0_LOAD) { | ||
| 459 | se_lw = &lw; | ||
| 460 | rw += NICE_0_LOAD - se->load.weight; | ||
| 461 | } | ||
| 462 | |||
| 463 | delta = calc_delta_mine(delta, rw, se_lw); | ||
| 464 | } | ||
| 465 | |||
| 466 | return delta; | ||
| 467 | } | ||
| 468 | |||
| 469 | /* | ||
| 470 | * Update the current task's runtime statistics. Skip current tasks that | 412 | * Update the current task's runtime statistics. Skip current tasks that |
| 471 | * are not in our scheduling class. | 413 | * are not in our scheduling class. |
| 472 | */ | 414 | */ |
| @@ -586,11 +528,12 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) | |||
| 586 | update_load_add(&cfs_rq->load, se->load.weight); | 528 | update_load_add(&cfs_rq->load, se->load.weight); |
| 587 | if (!parent_entity(se)) | 529 | if (!parent_entity(se)) |
| 588 | inc_cpu_load(rq_of(cfs_rq), se->load.weight); | 530 | inc_cpu_load(rq_of(cfs_rq), se->load.weight); |
| 589 | if (entity_is_task(se)) | 531 | if (entity_is_task(se)) { |
| 590 | add_cfs_task_weight(cfs_rq, se->load.weight); | 532 | add_cfs_task_weight(cfs_rq, se->load.weight); |
| 533 | list_add(&se->group_node, &cfs_rq->tasks); | ||
| 534 | } | ||
| 591 | cfs_rq->nr_running++; | 535 | cfs_rq->nr_running++; |
| 592 | se->on_rq = 1; | 536 | se->on_rq = 1; |
| 593 | list_add(&se->group_node, &cfs_rq->tasks); | ||
| 594 | } | 537 | } |
| 595 | 538 | ||
| 596 | static void | 539 | static void |
| @@ -599,11 +542,12 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) | |||
| 599 | update_load_sub(&cfs_rq->load, se->load.weight); | 542 | update_load_sub(&cfs_rq->load, se->load.weight); |
| 600 | if (!parent_entity(se)) | 543 | if (!parent_entity(se)) |
| 601 | dec_cpu_load(rq_of(cfs_rq), se->load.weight); | 544 | dec_cpu_load(rq_of(cfs_rq), se->load.weight); |
| 602 | if (entity_is_task(se)) | 545 | if (entity_is_task(se)) { |
| 603 | add_cfs_task_weight(cfs_rq, -se->load.weight); | 546 | add_cfs_task_weight(cfs_rq, -se->load.weight); |
| 547 | list_del_init(&se->group_node); | ||
| 548 | } | ||
| 604 | cfs_rq->nr_running--; | 549 | cfs_rq->nr_running--; |
| 605 | se->on_rq = 0; | 550 | se->on_rq = 0; |
| 606 | list_del_init(&se->group_node); | ||
| 607 | } | 551 | } |
| 608 | 552 | ||
| 609 | static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) | 553 | static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| @@ -1085,7 +1029,6 @@ static long effective_load(struct task_group *tg, int cpu, | |||
| 1085 | long wl, long wg) | 1029 | long wl, long wg) |
| 1086 | { | 1030 | { |
| 1087 | struct sched_entity *se = tg->se[cpu]; | 1031 | struct sched_entity *se = tg->se[cpu]; |
| 1088 | long more_w; | ||
| 1089 | 1032 | ||
| 1090 | if (!tg->parent) | 1033 | if (!tg->parent) |
| 1091 | return wl; | 1034 | return wl; |
| @@ -1097,18 +1040,17 @@ static long effective_load(struct task_group *tg, int cpu, | |||
| 1097 | if (!wl && sched_feat(ASYM_EFF_LOAD)) | 1040 | if (!wl && sched_feat(ASYM_EFF_LOAD)) |
| 1098 | return wl; | 1041 | return wl; |
| 1099 | 1042 | ||
| 1100 | /* | ||
| 1101 | * Instead of using this increment, also add the difference | ||
| 1102 | * between when the shares were last updated and now. | ||
| 1103 | */ | ||
| 1104 | more_w = se->my_q->load.weight - se->my_q->rq_weight; | ||
| 1105 | wl += more_w; | ||
| 1106 | wg += more_w; | ||
| 1107 | |||
| 1108 | for_each_sched_entity(se) { | 1043 | for_each_sched_entity(se) { |
| 1109 | #define D(n) (likely(n) ? (n) : 1) | ||
| 1110 | |||
| 1111 | long S, rw, s, a, b; | 1044 | long S, rw, s, a, b; |
| 1045 | long more_w; | ||
| 1046 | |||
| 1047 | /* | ||
| 1048 | * Instead of using this increment, also add the difference | ||
| 1049 | * between when the shares were last updated and now. | ||
| 1050 | */ | ||
| 1051 | more_w = se->my_q->load.weight - se->my_q->rq_weight; | ||
| 1052 | wl += more_w; | ||
| 1053 | wg += more_w; | ||
| 1112 | 1054 | ||
| 1113 | S = se->my_q->tg->shares; | 1055 | S = se->my_q->tg->shares; |
| 1114 | s = se->my_q->shares; | 1056 | s = se->my_q->shares; |
| @@ -1117,7 +1059,11 @@ static long effective_load(struct task_group *tg, int cpu, | |||
| 1117 | a = S*(rw + wl); | 1059 | a = S*(rw + wl); |
| 1118 | b = S*rw + s*wg; | 1060 | b = S*rw + s*wg; |
| 1119 | 1061 | ||
| 1120 | wl = s*(a-b)/D(b); | 1062 | wl = s*(a-b); |
| 1063 | |||
| 1064 | if (likely(b)) | ||
| 1065 | wl /= b; | ||
| 1066 | |||
| 1121 | /* | 1067 | /* |
| 1122 | * Assume the group is already running and will | 1068 | * Assume the group is already running and will |
| 1123 | * thus already be accounted for in the weight. | 1069 | * thus already be accounted for in the weight. |
| @@ -1126,7 +1072,6 @@ static long effective_load(struct task_group *tg, int cpu, | |||
| 1126 | * alter the group weight. | 1072 | * alter the group weight. |
| 1127 | */ | 1073 | */ |
| 1128 | wg = 0; | 1074 | wg = 0; |
| 1129 | #undef D | ||
| 1130 | } | 1075 | } |
| 1131 | 1076 | ||
| 1132 | return wl; | 1077 | return wl; |
| @@ -1143,7 +1088,7 @@ static inline unsigned long effective_load(struct task_group *tg, int cpu, | |||
| 1143 | #endif | 1088 | #endif |
| 1144 | 1089 | ||
| 1145 | static int | 1090 | static int |
| 1146 | wake_affine(struct rq *rq, struct sched_domain *this_sd, struct rq *this_rq, | 1091 | wake_affine(struct sched_domain *this_sd, struct rq *this_rq, |
| 1147 | struct task_struct *p, int prev_cpu, int this_cpu, int sync, | 1092 | struct task_struct *p, int prev_cpu, int this_cpu, int sync, |
| 1148 | int idx, unsigned long load, unsigned long this_load, | 1093 | int idx, unsigned long load, unsigned long this_load, |
| 1149 | unsigned int imbalance) | 1094 | unsigned int imbalance) |
| @@ -1158,6 +1103,11 @@ wake_affine(struct rq *rq, struct sched_domain *this_sd, struct rq *this_rq, | |||
| 1158 | if (!(this_sd->flags & SD_WAKE_AFFINE) || !sched_feat(AFFINE_WAKEUPS)) | 1103 | if (!(this_sd->flags & SD_WAKE_AFFINE) || !sched_feat(AFFINE_WAKEUPS)) |
| 1159 | return 0; | 1104 | return 0; |
| 1160 | 1105 | ||
| 1106 | if (!sync && sched_feat(SYNC_WAKEUPS) && | ||
| 1107 | curr->se.avg_overlap < sysctl_sched_migration_cost && | ||
| 1108 | p->se.avg_overlap < sysctl_sched_migration_cost) | ||
| 1109 | sync = 1; | ||
| 1110 | |||
| 1161 | /* | 1111 | /* |
| 1162 | * If sync wakeup then subtract the (maximum possible) | 1112 | * If sync wakeup then subtract the (maximum possible) |
| 1163 | * effect of the currently running task from the load | 1113 | * effect of the currently running task from the load |
| @@ -1182,17 +1132,14 @@ wake_affine(struct rq *rq, struct sched_domain *this_sd, struct rq *this_rq, | |||
| 1182 | * a reasonable amount of time then attract this newly | 1132 | * a reasonable amount of time then attract this newly |
| 1183 | * woken task: | 1133 | * woken task: |
| 1184 | */ | 1134 | */ |
| 1185 | if (sync && balanced) { | 1135 | if (sync && balanced) |
| 1186 | if (curr->se.avg_overlap < sysctl_sched_migration_cost && | 1136 | return 1; |
| 1187 | p->se.avg_overlap < sysctl_sched_migration_cost) | ||
| 1188 | return 1; | ||
| 1189 | } | ||
| 1190 | 1137 | ||
| 1191 | schedstat_inc(p, se.nr_wakeups_affine_attempts); | 1138 | schedstat_inc(p, se.nr_wakeups_affine_attempts); |
| 1192 | tl_per_task = cpu_avg_load_per_task(this_cpu); | 1139 | tl_per_task = cpu_avg_load_per_task(this_cpu); |
| 1193 | 1140 | ||
| 1194 | if ((tl <= load && tl + target_load(prev_cpu, idx) <= tl_per_task) || | 1141 | if (balanced || (tl <= load && tl + target_load(prev_cpu, idx) <= |
| 1195 | balanced) { | 1142 | tl_per_task)) { |
| 1196 | /* | 1143 | /* |
| 1197 | * This domain has SD_WAKE_AFFINE and | 1144 | * This domain has SD_WAKE_AFFINE and |
| 1198 | * p is cache cold in this domain, and | 1145 | * p is cache cold in this domain, and |
| @@ -1211,16 +1158,17 @@ static int select_task_rq_fair(struct task_struct *p, int sync) | |||
| 1211 | struct sched_domain *sd, *this_sd = NULL; | 1158 | struct sched_domain *sd, *this_sd = NULL; |
| 1212 | int prev_cpu, this_cpu, new_cpu; | 1159 | int prev_cpu, this_cpu, new_cpu; |
| 1213 | unsigned long load, this_load; | 1160 | unsigned long load, this_load; |
| 1214 | struct rq *rq, *this_rq; | 1161 | struct rq *this_rq; |
| 1215 | unsigned int imbalance; | 1162 | unsigned int imbalance; |
| 1216 | int idx; | 1163 | int idx; |
| 1217 | 1164 | ||
| 1218 | prev_cpu = task_cpu(p); | 1165 | prev_cpu = task_cpu(p); |
| 1219 | rq = task_rq(p); | ||
| 1220 | this_cpu = smp_processor_id(); | 1166 | this_cpu = smp_processor_id(); |
| 1221 | this_rq = cpu_rq(this_cpu); | 1167 | this_rq = cpu_rq(this_cpu); |
| 1222 | new_cpu = prev_cpu; | 1168 | new_cpu = prev_cpu; |
| 1223 | 1169 | ||
| 1170 | if (prev_cpu == this_cpu) | ||
| 1171 | goto out; | ||
| 1224 | /* | 1172 | /* |
| 1225 | * 'this_sd' is the first domain that both | 1173 | * 'this_sd' is the first domain that both |
| 1226 | * this_cpu and prev_cpu are present in: | 1174 | * this_cpu and prev_cpu are present in: |
| @@ -1248,13 +1196,10 @@ static int select_task_rq_fair(struct task_struct *p, int sync) | |||
| 1248 | load = source_load(prev_cpu, idx); | 1196 | load = source_load(prev_cpu, idx); |
| 1249 | this_load = target_load(this_cpu, idx); | 1197 | this_load = target_load(this_cpu, idx); |
| 1250 | 1198 | ||
| 1251 | if (wake_affine(rq, this_sd, this_rq, p, prev_cpu, this_cpu, sync, idx, | 1199 | if (wake_affine(this_sd, this_rq, p, prev_cpu, this_cpu, sync, idx, |
| 1252 | load, this_load, imbalance)) | 1200 | load, this_load, imbalance)) |
| 1253 | return this_cpu; | 1201 | return this_cpu; |
| 1254 | 1202 | ||
| 1255 | if (prev_cpu == this_cpu) | ||
| 1256 | goto out; | ||
| 1257 | |||
| 1258 | /* | 1203 | /* |
| 1259 | * Start passive balancing when half the imbalance_pct | 1204 | * Start passive balancing when half the imbalance_pct |
| 1260 | * limit is reached. | 1205 | * limit is reached. |
| @@ -1281,62 +1226,20 @@ static unsigned long wakeup_gran(struct sched_entity *se) | |||
| 1281 | * + nice tasks. | 1226 | * + nice tasks. |
| 1282 | */ | 1227 | */ |
| 1283 | if (sched_feat(ASYM_GRAN)) | 1228 | if (sched_feat(ASYM_GRAN)) |
| 1284 | gran = calc_delta_asym(sysctl_sched_wakeup_granularity, se); | 1229 | gran = calc_delta_mine(gran, NICE_0_LOAD, &se->load); |
| 1285 | else | ||
| 1286 | gran = calc_delta_fair(sysctl_sched_wakeup_granularity, se); | ||
| 1287 | 1230 | ||
| 1288 | return gran; | 1231 | return gran; |
| 1289 | } | 1232 | } |
| 1290 | 1233 | ||
| 1291 | /* | 1234 | /* |
| 1292 | * Should 'se' preempt 'curr'. | ||
| 1293 | * | ||
| 1294 | * |s1 | ||
| 1295 | * |s2 | ||
| 1296 | * |s3 | ||
| 1297 | * g | ||
| 1298 | * |<--->|c | ||
| 1299 | * | ||
| 1300 | * w(c, s1) = -1 | ||
| 1301 | * w(c, s2) = 0 | ||
| 1302 | * w(c, s3) = 1 | ||
| 1303 | * | ||
| 1304 | */ | ||
| 1305 | static int | ||
| 1306 | wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se) | ||
| 1307 | { | ||
| 1308 | s64 gran, vdiff = curr->vruntime - se->vruntime; | ||
| 1309 | |||
| 1310 | if (vdiff < 0) | ||
| 1311 | return -1; | ||
| 1312 | |||
| 1313 | gran = wakeup_gran(curr); | ||
| 1314 | if (vdiff > gran) | ||
| 1315 | return 1; | ||
| 1316 | |||
| 1317 | return 0; | ||
| 1318 | } | ||
| 1319 | |||
| 1320 | /* return depth at which a sched entity is present in the hierarchy */ | ||
| 1321 | static inline int depth_se(struct sched_entity *se) | ||
| 1322 | { | ||
| 1323 | int depth = 0; | ||
| 1324 | |||
| 1325 | for_each_sched_entity(se) | ||
| 1326 | depth++; | ||
| 1327 | |||
| 1328 | return depth; | ||
| 1329 | } | ||
| 1330 | |||
| 1331 | /* | ||
| 1332 | * Preempt the current task with a newly woken task if needed: | 1235 | * Preempt the current task with a newly woken task if needed: |
| 1333 | */ | 1236 | */ |
| 1334 | static void check_preempt_wakeup(struct rq *rq, struct task_struct *p) | 1237 | static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync) |
| 1335 | { | 1238 | { |
| 1336 | struct task_struct *curr = rq->curr; | 1239 | struct task_struct *curr = rq->curr; |
| 1337 | struct cfs_rq *cfs_rq = task_cfs_rq(curr); | 1240 | struct cfs_rq *cfs_rq = task_cfs_rq(curr); |
| 1338 | struct sched_entity *se = &curr->se, *pse = &p->se; | 1241 | struct sched_entity *se = &curr->se, *pse = &p->se; |
| 1339 | int se_depth, pse_depth; | 1242 | s64 delta_exec; |
| 1340 | 1243 | ||
| 1341 | if (unlikely(rt_prio(p->prio))) { | 1244 | if (unlikely(rt_prio(p->prio))) { |
| 1342 | update_rq_clock(rq); | 1245 | update_rq_clock(rq); |
| @@ -1351,6 +1254,13 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p) | |||
| 1351 | cfs_rq_of(pse)->next = pse; | 1254 | cfs_rq_of(pse)->next = pse; |
| 1352 | 1255 | ||
| 1353 | /* | 1256 | /* |
| 1257 | * We can come here with TIF_NEED_RESCHED already set from new task | ||
| 1258 | * wake up path. | ||
| 1259 | */ | ||
| 1260 | if (test_tsk_need_resched(curr)) | ||
| 1261 | return; | ||
| 1262 | |||
| 1263 | /* | ||
| 1354 | * Batch tasks do not preempt (their preemption is driven by | 1264 | * Batch tasks do not preempt (their preemption is driven by |
| 1355 | * the tick): | 1265 | * the tick): |
| 1356 | */ | 1266 | */ |
| @@ -1360,33 +1270,15 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p) | |||
| 1360 | if (!sched_feat(WAKEUP_PREEMPT)) | 1270 | if (!sched_feat(WAKEUP_PREEMPT)) |
| 1361 | return; | 1271 | return; |
| 1362 | 1272 | ||
| 1363 | /* | 1273 | if (sched_feat(WAKEUP_OVERLAP) && (sync || |
| 1364 | * preemption test can be made between sibling entities who are in the | 1274 | (se->avg_overlap < sysctl_sched_migration_cost && |
| 1365 | * same cfs_rq i.e who have a common parent. Walk up the hierarchy of | 1275 | pse->avg_overlap < sysctl_sched_migration_cost))) { |
| 1366 | * both tasks until we find their ancestors who are siblings of common | 1276 | resched_task(curr); |
| 1367 | * parent. | 1277 | return; |
| 1368 | */ | ||
| 1369 | |||
| 1370 | /* First walk up until both entities are at same depth */ | ||
| 1371 | se_depth = depth_se(se); | ||
| 1372 | pse_depth = depth_se(pse); | ||
| 1373 | |||
| 1374 | while (se_depth > pse_depth) { | ||
| 1375 | se_depth--; | ||
| 1376 | se = parent_entity(se); | ||
| 1377 | } | ||
| 1378 | |||
| 1379 | while (pse_depth > se_depth) { | ||
| 1380 | pse_depth--; | ||
| 1381 | pse = parent_entity(pse); | ||
| 1382 | } | ||
| 1383 | |||
| 1384 | while (!is_same_group(se, pse)) { | ||
| 1385 | se = parent_entity(se); | ||
| 1386 | pse = parent_entity(pse); | ||
| 1387 | } | 1278 | } |
| 1388 | 1279 | ||
| 1389 | if (wakeup_preempt_entity(se, pse) == 1) | 1280 | delta_exec = se->sum_exec_runtime - se->prev_sum_exec_runtime; |
| 1281 | if (delta_exec > wakeup_gran(pse)) | ||
| 1390 | resched_task(curr); | 1282 | resched_task(curr); |
| 1391 | } | 1283 | } |
| 1392 | 1284 | ||
| @@ -1445,19 +1337,9 @@ __load_balance_iterator(struct cfs_rq *cfs_rq, struct list_head *next) | |||
| 1445 | if (next == &cfs_rq->tasks) | 1337 | if (next == &cfs_rq->tasks) |
| 1446 | return NULL; | 1338 | return NULL; |
| 1447 | 1339 | ||
| 1448 | /* Skip over entities that are not tasks */ | 1340 | se = list_entry(next, struct sched_entity, group_node); |
| 1449 | do { | 1341 | p = task_of(se); |
| 1450 | se = list_entry(next, struct sched_entity, group_node); | 1342 | cfs_rq->balance_iterator = next->next; |
| 1451 | next = next->next; | ||
| 1452 | } while (next != &cfs_rq->tasks && !entity_is_task(se)); | ||
| 1453 | |||
| 1454 | if (next == &cfs_rq->tasks) | ||
| 1455 | return NULL; | ||
| 1456 | |||
| 1457 | cfs_rq->balance_iterator = next; | ||
| 1458 | |||
| 1459 | if (entity_is_task(se)) | ||
| 1460 | p = task_of(se); | ||
| 1461 | 1343 | ||
| 1462 | return p; | 1344 | return p; |
| 1463 | } | 1345 | } |
| @@ -1507,7 +1389,7 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, | |||
| 1507 | rcu_read_lock(); | 1389 | rcu_read_lock(); |
| 1508 | update_h_load(busiest_cpu); | 1390 | update_h_load(busiest_cpu); |
| 1509 | 1391 | ||
| 1510 | list_for_each_entry(tg, &task_groups, list) { | 1392 | list_for_each_entry_rcu(tg, &task_groups, list) { |
| 1511 | struct cfs_rq *busiest_cfs_rq = tg->cfs_rq[busiest_cpu]; | 1393 | struct cfs_rq *busiest_cfs_rq = tg->cfs_rq[busiest_cpu]; |
| 1512 | unsigned long busiest_h_load = busiest_cfs_rq->h_load; | 1394 | unsigned long busiest_h_load = busiest_cfs_rq->h_load; |
| 1513 | unsigned long busiest_weight = busiest_cfs_rq->load.weight; | 1395 | unsigned long busiest_weight = busiest_cfs_rq->load.weight; |
| @@ -1620,10 +1502,10 @@ static void task_new_fair(struct rq *rq, struct task_struct *p) | |||
| 1620 | * 'current' within the tree based on its new key value. | 1502 | * 'current' within the tree based on its new key value. |
| 1621 | */ | 1503 | */ |
| 1622 | swap(curr->vruntime, se->vruntime); | 1504 | swap(curr->vruntime, se->vruntime); |
| 1505 | resched_task(rq->curr); | ||
| 1623 | } | 1506 | } |
| 1624 | 1507 | ||
| 1625 | enqueue_task_fair(rq, p, 0); | 1508 | enqueue_task_fair(rq, p, 0); |
| 1626 | resched_task(rq->curr); | ||
| 1627 | } | 1509 | } |
| 1628 | 1510 | ||
| 1629 | /* | 1511 | /* |
| @@ -1642,7 +1524,7 @@ static void prio_changed_fair(struct rq *rq, struct task_struct *p, | |||
| 1642 | if (p->prio > oldprio) | 1524 | if (p->prio > oldprio) |
| 1643 | resched_task(rq->curr); | 1525 | resched_task(rq->curr); |
| 1644 | } else | 1526 | } else |
| 1645 | check_preempt_curr(rq, p); | 1527 | check_preempt_curr(rq, p, 0); |
| 1646 | } | 1528 | } |
| 1647 | 1529 | ||
| 1648 | /* | 1530 | /* |
| @@ -1659,7 +1541,7 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p, | |||
| 1659 | if (running) | 1541 | if (running) |
| 1660 | resched_task(rq->curr); | 1542 | resched_task(rq->curr); |
| 1661 | else | 1543 | else |
| 1662 | check_preempt_curr(rq, p); | 1544 | check_preempt_curr(rq, p, 0); |
| 1663 | } | 1545 | } |
| 1664 | 1546 | ||
| 1665 | /* Account for a task changing its policy or group. | 1547 | /* Account for a task changing its policy or group. |
diff --git a/kernel/sched_features.h b/kernel/sched_features.h index 9353ca78154e..7c9e8f4a049f 100644 --- a/kernel/sched_features.h +++ b/kernel/sched_features.h | |||
| @@ -11,3 +11,4 @@ SCHED_FEAT(ASYM_GRAN, 1) | |||
| 11 | SCHED_FEAT(LB_BIAS, 1) | 11 | SCHED_FEAT(LB_BIAS, 1) |
| 12 | SCHED_FEAT(LB_WAKEUP_UPDATE, 1) | 12 | SCHED_FEAT(LB_WAKEUP_UPDATE, 1) |
| 13 | SCHED_FEAT(ASYM_EFF_LOAD, 1) | 13 | SCHED_FEAT(ASYM_EFF_LOAD, 1) |
| 14 | SCHED_FEAT(WAKEUP_OVERLAP, 0) | ||
diff --git a/kernel/sched_idletask.c b/kernel/sched_idletask.c index 3a4f92dbbe66..dec4ccabe2f5 100644 --- a/kernel/sched_idletask.c +++ b/kernel/sched_idletask.c | |||
| @@ -14,7 +14,7 @@ static int select_task_rq_idle(struct task_struct *p, int sync) | |||
| 14 | /* | 14 | /* |
| 15 | * Idle tasks are unconditionally rescheduled: | 15 | * Idle tasks are unconditionally rescheduled: |
| 16 | */ | 16 | */ |
| 17 | static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p) | 17 | static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int sync) |
| 18 | { | 18 | { |
| 19 | resched_task(rq->idle); | 19 | resched_task(rq->idle); |
| 20 | } | 20 | } |
| @@ -76,7 +76,7 @@ static void switched_to_idle(struct rq *rq, struct task_struct *p, | |||
| 76 | if (running) | 76 | if (running) |
| 77 | resched_task(rq->curr); | 77 | resched_task(rq->curr); |
| 78 | else | 78 | else |
| 79 | check_preempt_curr(rq, p); | 79 | check_preempt_curr(rq, p, 0); |
| 80 | } | 80 | } |
| 81 | 81 | ||
| 82 | static void prio_changed_idle(struct rq *rq, struct task_struct *p, | 82 | static void prio_changed_idle(struct rq *rq, struct task_struct *p, |
| @@ -93,7 +93,7 @@ static void prio_changed_idle(struct rq *rq, struct task_struct *p, | |||
| 93 | if (p->prio > oldprio) | 93 | if (p->prio > oldprio) |
| 94 | resched_task(rq->curr); | 94 | resched_task(rq->curr); |
| 95 | } else | 95 | } else |
| 96 | check_preempt_curr(rq, p); | 96 | check_preempt_curr(rq, p, 0); |
| 97 | } | 97 | } |
| 98 | 98 | ||
| 99 | /* | 99 | /* |
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 552310798dad..cdf5740ab03e 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c | |||
| @@ -102,12 +102,12 @@ static void dequeue_rt_entity(struct sched_rt_entity *rt_se); | |||
| 102 | 102 | ||
| 103 | static void sched_rt_rq_enqueue(struct rt_rq *rt_rq) | 103 | static void sched_rt_rq_enqueue(struct rt_rq *rt_rq) |
| 104 | { | 104 | { |
| 105 | struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr; | ||
| 105 | struct sched_rt_entity *rt_se = rt_rq->rt_se; | 106 | struct sched_rt_entity *rt_se = rt_rq->rt_se; |
| 106 | 107 | ||
| 107 | if (rt_se && !on_rt_rq(rt_se) && rt_rq->rt_nr_running) { | 108 | if (rt_rq->rt_nr_running) { |
| 108 | struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr; | 109 | if (rt_se && !on_rt_rq(rt_se)) |
| 109 | 110 | enqueue_rt_entity(rt_se); | |
| 110 | enqueue_rt_entity(rt_se); | ||
| 111 | if (rt_rq->highest_prio < curr->prio) | 111 | if (rt_rq->highest_prio < curr->prio) |
| 112 | resched_task(curr); | 112 | resched_task(curr); |
| 113 | } | 113 | } |
| @@ -231,6 +231,9 @@ static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq) | |||
| 231 | #endif /* CONFIG_RT_GROUP_SCHED */ | 231 | #endif /* CONFIG_RT_GROUP_SCHED */ |
| 232 | 232 | ||
| 233 | #ifdef CONFIG_SMP | 233 | #ifdef CONFIG_SMP |
| 234 | /* | ||
| 235 | * We ran out of runtime, see if we can borrow some from our neighbours. | ||
| 236 | */ | ||
| 234 | static int do_balance_runtime(struct rt_rq *rt_rq) | 237 | static int do_balance_runtime(struct rt_rq *rt_rq) |
| 235 | { | 238 | { |
| 236 | struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); | 239 | struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); |
| @@ -250,9 +253,18 @@ static int do_balance_runtime(struct rt_rq *rt_rq) | |||
| 250 | continue; | 253 | continue; |
| 251 | 254 | ||
| 252 | spin_lock(&iter->rt_runtime_lock); | 255 | spin_lock(&iter->rt_runtime_lock); |
| 256 | /* | ||
| 257 | * Either all rqs have inf runtime and there's nothing to steal | ||
| 258 | * or __disable_runtime() below sets a specific rq to inf to | ||
| 259 | * indicate its been disabled and disalow stealing. | ||
| 260 | */ | ||
| 253 | if (iter->rt_runtime == RUNTIME_INF) | 261 | if (iter->rt_runtime == RUNTIME_INF) |
| 254 | goto next; | 262 | goto next; |
| 255 | 263 | ||
| 264 | /* | ||
| 265 | * From runqueues with spare time, take 1/n part of their | ||
| 266 | * spare time, but no more than our period. | ||
| 267 | */ | ||
| 256 | diff = iter->rt_runtime - iter->rt_time; | 268 | diff = iter->rt_runtime - iter->rt_time; |
| 257 | if (diff > 0) { | 269 | if (diff > 0) { |
| 258 | diff = div_u64((u64)diff, weight); | 270 | diff = div_u64((u64)diff, weight); |
| @@ -274,6 +286,9 @@ next: | |||
| 274 | return more; | 286 | return more; |
| 275 | } | 287 | } |
| 276 | 288 | ||
| 289 | /* | ||
| 290 | * Ensure this RQ takes back all the runtime it lend to its neighbours. | ||
| 291 | */ | ||
| 277 | static void __disable_runtime(struct rq *rq) | 292 | static void __disable_runtime(struct rq *rq) |
| 278 | { | 293 | { |
| 279 | struct root_domain *rd = rq->rd; | 294 | struct root_domain *rd = rq->rd; |
| @@ -289,17 +304,33 @@ static void __disable_runtime(struct rq *rq) | |||
| 289 | 304 | ||
| 290 | spin_lock(&rt_b->rt_runtime_lock); | 305 | spin_lock(&rt_b->rt_runtime_lock); |
| 291 | spin_lock(&rt_rq->rt_runtime_lock); | 306 | spin_lock(&rt_rq->rt_runtime_lock); |
| 307 | /* | ||
| 308 | * Either we're all inf and nobody needs to borrow, or we're | ||
| 309 | * already disabled and thus have nothing to do, or we have | ||
| 310 | * exactly the right amount of runtime to take out. | ||
| 311 | */ | ||
| 292 | if (rt_rq->rt_runtime == RUNTIME_INF || | 312 | if (rt_rq->rt_runtime == RUNTIME_INF || |
| 293 | rt_rq->rt_runtime == rt_b->rt_runtime) | 313 | rt_rq->rt_runtime == rt_b->rt_runtime) |
| 294 | goto balanced; | 314 | goto balanced; |
| 295 | spin_unlock(&rt_rq->rt_runtime_lock); | 315 | spin_unlock(&rt_rq->rt_runtime_lock); |
| 296 | 316 | ||
| 317 | /* | ||
| 318 | * Calculate the difference between what we started out with | ||
| 319 | * and what we current have, that's the amount of runtime | ||
| 320 | * we lend and now have to reclaim. | ||
| 321 | */ | ||
| 297 | want = rt_b->rt_runtime - rt_rq->rt_runtime; | 322 | want = rt_b->rt_runtime - rt_rq->rt_runtime; |
| 298 | 323 | ||
| 324 | /* | ||
| 325 | * Greedy reclaim, take back as much as we can. | ||
| 326 | */ | ||
| 299 | for_each_cpu_mask(i, rd->span) { | 327 | for_each_cpu_mask(i, rd->span) { |
| 300 | struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); | 328 | struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); |
| 301 | s64 diff; | 329 | s64 diff; |
| 302 | 330 | ||
| 331 | /* | ||
| 332 | * Can't reclaim from ourselves or disabled runqueues. | ||
| 333 | */ | ||
| 303 | if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF) | 334 | if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF) |
| 304 | continue; | 335 | continue; |
| 305 | 336 | ||
| @@ -319,8 +350,16 @@ static void __disable_runtime(struct rq *rq) | |||
| 319 | } | 350 | } |
| 320 | 351 | ||
| 321 | spin_lock(&rt_rq->rt_runtime_lock); | 352 | spin_lock(&rt_rq->rt_runtime_lock); |
| 353 | /* | ||
| 354 | * We cannot be left wanting - that would mean some runtime | ||
| 355 | * leaked out of the system. | ||
| 356 | */ | ||
| 322 | BUG_ON(want); | 357 | BUG_ON(want); |
| 323 | balanced: | 358 | balanced: |
| 359 | /* | ||
| 360 | * Disable all the borrow logic by pretending we have inf | ||
| 361 | * runtime - in which case borrowing doesn't make sense. | ||
| 362 | */ | ||
| 324 | rt_rq->rt_runtime = RUNTIME_INF; | 363 | rt_rq->rt_runtime = RUNTIME_INF; |
| 325 | spin_unlock(&rt_rq->rt_runtime_lock); | 364 | spin_unlock(&rt_rq->rt_runtime_lock); |
| 326 | spin_unlock(&rt_b->rt_runtime_lock); | 365 | spin_unlock(&rt_b->rt_runtime_lock); |
| @@ -343,6 +382,9 @@ static void __enable_runtime(struct rq *rq) | |||
| 343 | if (unlikely(!scheduler_running)) | 382 | if (unlikely(!scheduler_running)) |
| 344 | return; | 383 | return; |
| 345 | 384 | ||
| 385 | /* | ||
| 386 | * Reset each runqueue's bandwidth settings | ||
| 387 | */ | ||
| 346 | for_each_leaf_rt_rq(rt_rq, rq) { | 388 | for_each_leaf_rt_rq(rt_rq, rq) { |
| 347 | struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); | 389 | struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); |
| 348 | 390 | ||
| @@ -350,6 +392,7 @@ static void __enable_runtime(struct rq *rq) | |||
| 350 | spin_lock(&rt_rq->rt_runtime_lock); | 392 | spin_lock(&rt_rq->rt_runtime_lock); |
| 351 | rt_rq->rt_runtime = rt_b->rt_runtime; | 393 | rt_rq->rt_runtime = rt_b->rt_runtime; |
| 352 | rt_rq->rt_time = 0; | 394 | rt_rq->rt_time = 0; |
| 395 | rt_rq->rt_throttled = 0; | ||
| 353 | spin_unlock(&rt_rq->rt_runtime_lock); | 396 | spin_unlock(&rt_rq->rt_runtime_lock); |
| 354 | spin_unlock(&rt_b->rt_runtime_lock); | 397 | spin_unlock(&rt_b->rt_runtime_lock); |
| 355 | } | 398 | } |
| @@ -388,7 +431,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) | |||
| 388 | int i, idle = 1; | 431 | int i, idle = 1; |
| 389 | cpumask_t span; | 432 | cpumask_t span; |
| 390 | 433 | ||
| 391 | if (rt_b->rt_runtime == RUNTIME_INF) | 434 | if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF) |
| 392 | return 1; | 435 | return 1; |
| 393 | 436 | ||
| 394 | span = sched_rt_period_mask(); | 437 | span = sched_rt_period_mask(); |
| @@ -486,6 +529,9 @@ static void update_curr_rt(struct rq *rq) | |||
| 486 | curr->se.exec_start = rq->clock; | 529 | curr->se.exec_start = rq->clock; |
| 487 | cpuacct_charge(curr, delta_exec); | 530 | cpuacct_charge(curr, delta_exec); |
| 488 | 531 | ||
| 532 | if (!rt_bandwidth_enabled()) | ||
| 533 | return; | ||
| 534 | |||
| 489 | for_each_sched_rt_entity(rt_se) { | 535 | for_each_sched_rt_entity(rt_se) { |
| 490 | rt_rq = rt_rq_of_se(rt_se); | 536 | rt_rq = rt_rq_of_se(rt_se); |
| 491 | 537 | ||
| @@ -783,7 +829,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) | |||
| 783 | /* | 829 | /* |
| 784 | * Preempt the current task with a newly woken task if needed: | 830 | * Preempt the current task with a newly woken task if needed: |
| 785 | */ | 831 | */ |
| 786 | static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p) | 832 | static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int sync) |
| 787 | { | 833 | { |
| 788 | if (p->prio < rq->curr->prio) { | 834 | if (p->prio < rq->curr->prio) { |
| 789 | resched_task(rq->curr); | 835 | resched_task(rq->curr); |
diff --git a/kernel/softirq.c b/kernel/softirq.c index c506f266a6b9..37d67aa2d56f 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c | |||
| @@ -46,7 +46,7 @@ irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned; | |||
| 46 | EXPORT_SYMBOL(irq_stat); | 46 | EXPORT_SYMBOL(irq_stat); |
| 47 | #endif | 47 | #endif |
| 48 | 48 | ||
| 49 | static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp; | 49 | static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp; |
| 50 | 50 | ||
| 51 | static DEFINE_PER_CPU(struct task_struct *, ksoftirqd); | 51 | static DEFINE_PER_CPU(struct task_struct *, ksoftirqd); |
| 52 | 52 | ||
| @@ -205,7 +205,18 @@ restart: | |||
| 205 | 205 | ||
| 206 | do { | 206 | do { |
| 207 | if (pending & 1) { | 207 | if (pending & 1) { |
| 208 | int prev_count = preempt_count(); | ||
| 209 | |||
| 208 | h->action(h); | 210 | h->action(h); |
| 211 | |||
| 212 | if (unlikely(prev_count != preempt_count())) { | ||
| 213 | printk(KERN_ERR "huh, entered softirq %td %p" | ||
| 214 | "with preempt_count %08x," | ||
| 215 | " exited with %08x?\n", h - softirq_vec, | ||
| 216 | h->action, prev_count, preempt_count()); | ||
| 217 | preempt_count() = prev_count; | ||
| 218 | } | ||
| 219 | |||
| 209 | rcu_bh_qsctr_inc(cpu); | 220 | rcu_bh_qsctr_inc(cpu); |
| 210 | } | 221 | } |
| 211 | h++; | 222 | h++; |
diff --git a/kernel/softlockup.c b/kernel/softlockup.c index cb838ee93a82..3953e4aed733 100644 --- a/kernel/softlockup.c +++ b/kernel/softlockup.c | |||
| @@ -226,7 +226,7 @@ static void check_hung_uninterruptible_tasks(int this_cpu) | |||
| 226 | * If the system crashed already then all bets are off, | 226 | * If the system crashed already then all bets are off, |
| 227 | * do not report extra hung tasks: | 227 | * do not report extra hung tasks: |
| 228 | */ | 228 | */ |
| 229 | if ((tainted & TAINT_DIE) || did_panic) | 229 | if (test_taint(TAINT_DIE) || did_panic) |
| 230 | return; | 230 | return; |
| 231 | 231 | ||
| 232 | read_lock(&tasklist_lock); | 232 | read_lock(&tasklist_lock); |
diff --git a/kernel/sys.c b/kernel/sys.c index 1b96401a0576..fc71f99fb469 100644 --- a/kernel/sys.c +++ b/kernel/sys.c | |||
| @@ -1060,9 +1060,7 @@ asmlinkage long sys_setsid(void) | |||
| 1060 | group_leader->signal->leader = 1; | 1060 | group_leader->signal->leader = 1; |
| 1061 | __set_special_pids(sid); | 1061 | __set_special_pids(sid); |
| 1062 | 1062 | ||
| 1063 | spin_lock(&group_leader->sighand->siglock); | 1063 | proc_clear_tty(group_leader); |
| 1064 | group_leader->signal->tty = NULL; | ||
| 1065 | spin_unlock(&group_leader->sighand->siglock); | ||
| 1066 | 1064 | ||
| 1067 | err = session; | 1065 | err = session; |
| 1068 | out: | 1066 | out: |
| @@ -1351,8 +1349,10 @@ asmlinkage long sys_sethostname(char __user *name, int len) | |||
| 1351 | down_write(&uts_sem); | 1349 | down_write(&uts_sem); |
| 1352 | errno = -EFAULT; | 1350 | errno = -EFAULT; |
| 1353 | if (!copy_from_user(tmp, name, len)) { | 1351 | if (!copy_from_user(tmp, name, len)) { |
| 1354 | memcpy(utsname()->nodename, tmp, len); | 1352 | struct new_utsname *u = utsname(); |
| 1355 | utsname()->nodename[len] = 0; | 1353 | |
| 1354 | memcpy(u->nodename, tmp, len); | ||
| 1355 | memset(u->nodename + len, 0, sizeof(u->nodename) - len); | ||
| 1356 | errno = 0; | 1356 | errno = 0; |
| 1357 | } | 1357 | } |
| 1358 | up_write(&uts_sem); | 1358 | up_write(&uts_sem); |
| @@ -1364,15 +1364,17 @@ asmlinkage long sys_sethostname(char __user *name, int len) | |||
| 1364 | asmlinkage long sys_gethostname(char __user *name, int len) | 1364 | asmlinkage long sys_gethostname(char __user *name, int len) |
| 1365 | { | 1365 | { |
| 1366 | int i, errno; | 1366 | int i, errno; |
| 1367 | struct new_utsname *u; | ||
| 1367 | 1368 | ||
| 1368 | if (len < 0) | 1369 | if (len < 0) |
| 1369 | return -EINVAL; | 1370 | return -EINVAL; |
| 1370 | down_read(&uts_sem); | 1371 | down_read(&uts_sem); |
| 1371 | i = 1 + strlen(utsname()->nodename); | 1372 | u = utsname(); |
| 1373 | i = 1 + strlen(u->nodename); | ||
| 1372 | if (i > len) | 1374 | if (i > len) |
| 1373 | i = len; | 1375 | i = len; |
| 1374 | errno = 0; | 1376 | errno = 0; |
| 1375 | if (copy_to_user(name, utsname()->nodename, i)) | 1377 | if (copy_to_user(name, u->nodename, i)) |
| 1376 | errno = -EFAULT; | 1378 | errno = -EFAULT; |
| 1377 | up_read(&uts_sem); | 1379 | up_read(&uts_sem); |
| 1378 | return errno; | 1380 | return errno; |
| @@ -1397,8 +1399,10 @@ asmlinkage long sys_setdomainname(char __user *name, int len) | |||
| 1397 | down_write(&uts_sem); | 1399 | down_write(&uts_sem); |
| 1398 | errno = -EFAULT; | 1400 | errno = -EFAULT; |
| 1399 | if (!copy_from_user(tmp, name, len)) { | 1401 | if (!copy_from_user(tmp, name, len)) { |
| 1400 | memcpy(utsname()->domainname, tmp, len); | 1402 | struct new_utsname *u = utsname(); |
| 1401 | utsname()->domainname[len] = 0; | 1403 | |
| 1404 | memcpy(u->domainname, tmp, len); | ||
| 1405 | memset(u->domainname + len, 0, sizeof(u->domainname) - len); | ||
| 1402 | errno = 0; | 1406 | errno = 0; |
| 1403 | } | 1407 | } |
| 1404 | up_write(&uts_sem); | 1408 | up_write(&uts_sem); |
| @@ -1452,14 +1456,22 @@ asmlinkage long sys_setrlimit(unsigned int resource, struct rlimit __user *rlim) | |||
| 1452 | return -EINVAL; | 1456 | return -EINVAL; |
| 1453 | if (copy_from_user(&new_rlim, rlim, sizeof(*rlim))) | 1457 | if (copy_from_user(&new_rlim, rlim, sizeof(*rlim))) |
| 1454 | return -EFAULT; | 1458 | return -EFAULT; |
| 1455 | if (new_rlim.rlim_cur > new_rlim.rlim_max) | ||
| 1456 | return -EINVAL; | ||
| 1457 | old_rlim = current->signal->rlim + resource; | 1459 | old_rlim = current->signal->rlim + resource; |
| 1458 | if ((new_rlim.rlim_max > old_rlim->rlim_max) && | 1460 | if ((new_rlim.rlim_max > old_rlim->rlim_max) && |
| 1459 | !capable(CAP_SYS_RESOURCE)) | 1461 | !capable(CAP_SYS_RESOURCE)) |
| 1460 | return -EPERM; | 1462 | return -EPERM; |
| 1461 | if (resource == RLIMIT_NOFILE && new_rlim.rlim_max > sysctl_nr_open) | 1463 | |
| 1462 | return -EPERM; | 1464 | if (resource == RLIMIT_NOFILE) { |
| 1465 | if (new_rlim.rlim_max == RLIM_INFINITY) | ||
| 1466 | new_rlim.rlim_max = sysctl_nr_open; | ||
| 1467 | if (new_rlim.rlim_cur == RLIM_INFINITY) | ||
| 1468 | new_rlim.rlim_cur = sysctl_nr_open; | ||
| 1469 | if (new_rlim.rlim_max > sysctl_nr_open) | ||
| 1470 | return -EPERM; | ||
| 1471 | } | ||
| 1472 | |||
| 1473 | if (new_rlim.rlim_cur > new_rlim.rlim_max) | ||
| 1474 | return -EINVAL; | ||
| 1463 | 1475 | ||
| 1464 | retval = security_task_setrlimit(resource, &new_rlim); | 1476 | retval = security_task_setrlimit(resource, &new_rlim); |
| 1465 | if (retval) | 1477 | if (retval) |
diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index 08d6e1bb99ac..a77b27b11b04 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c | |||
| @@ -125,6 +125,12 @@ cond_syscall(sys_vm86old); | |||
| 125 | cond_syscall(sys_vm86); | 125 | cond_syscall(sys_vm86); |
| 126 | cond_syscall(compat_sys_ipc); | 126 | cond_syscall(compat_sys_ipc); |
| 127 | cond_syscall(compat_sys_sysctl); | 127 | cond_syscall(compat_sys_sysctl); |
| 128 | cond_syscall(sys_flock); | ||
| 129 | cond_syscall(sys_io_setup); | ||
| 130 | cond_syscall(sys_io_destroy); | ||
| 131 | cond_syscall(sys_io_submit); | ||
| 132 | cond_syscall(sys_io_cancel); | ||
| 133 | cond_syscall(sys_io_getevents); | ||
| 128 | 134 | ||
| 129 | /* arch-specific weak syscall entries */ | 135 | /* arch-specific weak syscall entries */ |
| 130 | cond_syscall(sys_pciconfig_read); | 136 | cond_syscall(sys_pciconfig_read); |
diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 50ec0886fa3d..617d41e4d6a0 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c | |||
| @@ -80,7 +80,6 @@ extern int pid_max_min, pid_max_max; | |||
| 80 | extern int sysctl_drop_caches; | 80 | extern int sysctl_drop_caches; |
| 81 | extern int percpu_pagelist_fraction; | 81 | extern int percpu_pagelist_fraction; |
| 82 | extern int compat_log; | 82 | extern int compat_log; |
| 83 | extern int maps_protect; | ||
| 84 | extern int latencytop_enabled; | 83 | extern int latencytop_enabled; |
| 85 | extern int sysctl_nr_open_min, sysctl_nr_open_max; | 84 | extern int sysctl_nr_open_min, sysctl_nr_open_max; |
| 86 | #ifdef CONFIG_RCU_TORTURE_TEST | 85 | #ifdef CONFIG_RCU_TORTURE_TEST |
| @@ -97,7 +96,7 @@ static int sixty = 60; | |||
| 97 | static int neg_one = -1; | 96 | static int neg_one = -1; |
| 98 | #endif | 97 | #endif |
| 99 | 98 | ||
| 100 | #ifdef CONFIG_MMU | 99 | #if defined(CONFIG_MMU) && defined(CONFIG_FILE_LOCKING) |
| 101 | static int two = 2; | 100 | static int two = 2; |
| 102 | #endif | 101 | #endif |
| 103 | 102 | ||
| @@ -118,10 +117,8 @@ extern char modprobe_path[]; | |||
| 118 | extern int sg_big_buff; | 117 | extern int sg_big_buff; |
| 119 | #endif | 118 | #endif |
| 120 | 119 | ||
| 121 | #ifdef __sparc__ | 120 | #ifdef CONFIG_SPARC |
| 122 | extern char reboot_command []; | 121 | #include <asm/system.h> |
| 123 | extern int stop_a_enabled; | ||
| 124 | extern int scons_pwroff; | ||
| 125 | #endif | 122 | #endif |
| 126 | 123 | ||
| 127 | #ifdef __hppa__ | 124 | #ifdef __hppa__ |
| @@ -152,7 +149,7 @@ extern int max_lock_depth; | |||
| 152 | #ifdef CONFIG_PROC_SYSCTL | 149 | #ifdef CONFIG_PROC_SYSCTL |
| 153 | static int proc_do_cad_pid(struct ctl_table *table, int write, struct file *filp, | 150 | static int proc_do_cad_pid(struct ctl_table *table, int write, struct file *filp, |
| 154 | void __user *buffer, size_t *lenp, loff_t *ppos); | 151 | void __user *buffer, size_t *lenp, loff_t *ppos); |
| 155 | static int proc_dointvec_taint(struct ctl_table *table, int write, struct file *filp, | 152 | static int proc_taint(struct ctl_table *table, int write, struct file *filp, |
| 156 | void __user *buffer, size_t *lenp, loff_t *ppos); | 153 | void __user *buffer, size_t *lenp, loff_t *ppos); |
| 157 | #endif | 154 | #endif |
| 158 | 155 | ||
| @@ -382,10 +379,9 @@ static struct ctl_table kern_table[] = { | |||
| 382 | #ifdef CONFIG_PROC_SYSCTL | 379 | #ifdef CONFIG_PROC_SYSCTL |
| 383 | { | 380 | { |
| 384 | .procname = "tainted", | 381 | .procname = "tainted", |
| 385 | .data = &tainted, | 382 | .maxlen = sizeof(long), |
| 386 | .maxlen = sizeof(int), | ||
| 387 | .mode = 0644, | 383 | .mode = 0644, |
| 388 | .proc_handler = &proc_dointvec_taint, | 384 | .proc_handler = &proc_taint, |
| 389 | }, | 385 | }, |
| 390 | #endif | 386 | #endif |
| 391 | #ifdef CONFIG_LATENCYTOP | 387 | #ifdef CONFIG_LATENCYTOP |
| @@ -415,7 +411,7 @@ static struct ctl_table kern_table[] = { | |||
| 415 | .mode = 0644, | 411 | .mode = 0644, |
| 416 | .proc_handler = &proc_dointvec, | 412 | .proc_handler = &proc_dointvec, |
| 417 | }, | 413 | }, |
| 418 | #ifdef __sparc__ | 414 | #ifdef CONFIG_SPARC |
| 419 | { | 415 | { |
| 420 | .ctl_name = KERN_SPARC_REBOOT, | 416 | .ctl_name = KERN_SPARC_REBOOT, |
| 421 | .procname = "reboot-cmd", | 417 | .procname = "reboot-cmd", |
| @@ -810,16 +806,6 @@ static struct ctl_table kern_table[] = { | |||
| 810 | .proc_handler = &proc_dointvec, | 806 | .proc_handler = &proc_dointvec, |
| 811 | }, | 807 | }, |
| 812 | #endif | 808 | #endif |
| 813 | #ifdef CONFIG_PROC_FS | ||
| 814 | { | ||
| 815 | .ctl_name = CTL_UNNUMBERED, | ||
| 816 | .procname = "maps_protect", | ||
| 817 | .data = &maps_protect, | ||
| 818 | .maxlen = sizeof(int), | ||
| 819 | .mode = 0644, | ||
| 820 | .proc_handler = &proc_dointvec, | ||
| 821 | }, | ||
| 822 | #endif | ||
| 823 | { | 809 | { |
| 824 | .ctl_name = CTL_UNNUMBERED, | 810 | .ctl_name = CTL_UNNUMBERED, |
| 825 | .procname = "poweroff_cmd", | 811 | .procname = "poweroff_cmd", |
| @@ -1261,6 +1247,7 @@ static struct ctl_table fs_table[] = { | |||
| 1261 | .extra1 = &minolduid, | 1247 | .extra1 = &minolduid, |
| 1262 | .extra2 = &maxolduid, | 1248 | .extra2 = &maxolduid, |
| 1263 | }, | 1249 | }, |
| 1250 | #ifdef CONFIG_FILE_LOCKING | ||
| 1264 | { | 1251 | { |
| 1265 | .ctl_name = FS_LEASES, | 1252 | .ctl_name = FS_LEASES, |
| 1266 | .procname = "leases-enable", | 1253 | .procname = "leases-enable", |
| @@ -1269,6 +1256,7 @@ static struct ctl_table fs_table[] = { | |||
| 1269 | .mode = 0644, | 1256 | .mode = 0644, |
| 1270 | .proc_handler = &proc_dointvec, | 1257 | .proc_handler = &proc_dointvec, |
| 1271 | }, | 1258 | }, |
| 1259 | #endif | ||
| 1272 | #ifdef CONFIG_DNOTIFY | 1260 | #ifdef CONFIG_DNOTIFY |
| 1273 | { | 1261 | { |
| 1274 | .ctl_name = FS_DIR_NOTIFY, | 1262 | .ctl_name = FS_DIR_NOTIFY, |
| @@ -1280,6 +1268,7 @@ static struct ctl_table fs_table[] = { | |||
| 1280 | }, | 1268 | }, |
| 1281 | #endif | 1269 | #endif |
| 1282 | #ifdef CONFIG_MMU | 1270 | #ifdef CONFIG_MMU |
| 1271 | #ifdef CONFIG_FILE_LOCKING | ||
| 1283 | { | 1272 | { |
| 1284 | .ctl_name = FS_LEASE_TIME, | 1273 | .ctl_name = FS_LEASE_TIME, |
| 1285 | .procname = "lease-break-time", | 1274 | .procname = "lease-break-time", |
| @@ -1291,6 +1280,8 @@ static struct ctl_table fs_table[] = { | |||
| 1291 | .extra1 = &zero, | 1280 | .extra1 = &zero, |
| 1292 | .extra2 = &two, | 1281 | .extra2 = &two, |
| 1293 | }, | 1282 | }, |
| 1283 | #endif | ||
| 1284 | #ifdef CONFIG_AIO | ||
| 1294 | { | 1285 | { |
| 1295 | .procname = "aio-nr", | 1286 | .procname = "aio-nr", |
| 1296 | .data = &aio_nr, | 1287 | .data = &aio_nr, |
| @@ -1305,6 +1296,7 @@ static struct ctl_table fs_table[] = { | |||
| 1305 | .mode = 0644, | 1296 | .mode = 0644, |
| 1306 | .proc_handler = &proc_doulongvec_minmax, | 1297 | .proc_handler = &proc_doulongvec_minmax, |
| 1307 | }, | 1298 | }, |
| 1299 | #endif /* CONFIG_AIO */ | ||
| 1308 | #ifdef CONFIG_INOTIFY_USER | 1300 | #ifdef CONFIG_INOTIFY_USER |
| 1309 | { | 1301 | { |
| 1310 | .ctl_name = FS_INOTIFY, | 1302 | .ctl_name = FS_INOTIFY, |
| @@ -1510,7 +1502,6 @@ void register_sysctl_root(struct ctl_table_root *root) | |||
| 1510 | /* Perform the actual read/write of a sysctl table entry. */ | 1502 | /* Perform the actual read/write of a sysctl table entry. */ |
| 1511 | static int do_sysctl_strategy(struct ctl_table_root *root, | 1503 | static int do_sysctl_strategy(struct ctl_table_root *root, |
| 1512 | struct ctl_table *table, | 1504 | struct ctl_table *table, |
| 1513 | int __user *name, int nlen, | ||
| 1514 | void __user *oldval, size_t __user *oldlenp, | 1505 | void __user *oldval, size_t __user *oldlenp, |
| 1515 | void __user *newval, size_t newlen) | 1506 | void __user *newval, size_t newlen) |
| 1516 | { | 1507 | { |
| @@ -1524,8 +1515,7 @@ static int do_sysctl_strategy(struct ctl_table_root *root, | |||
| 1524 | return -EPERM; | 1515 | return -EPERM; |
| 1525 | 1516 | ||
| 1526 | if (table->strategy) { | 1517 | if (table->strategy) { |
| 1527 | rc = table->strategy(table, name, nlen, oldval, oldlenp, | 1518 | rc = table->strategy(table, oldval, oldlenp, newval, newlen); |
| 1528 | newval, newlen); | ||
| 1529 | if (rc < 0) | 1519 | if (rc < 0) |
| 1530 | return rc; | 1520 | return rc; |
| 1531 | if (rc > 0) | 1521 | if (rc > 0) |
| @@ -1535,8 +1525,7 @@ static int do_sysctl_strategy(struct ctl_table_root *root, | |||
| 1535 | /* If there is no strategy routine, or if the strategy returns | 1525 | /* If there is no strategy routine, or if the strategy returns |
| 1536 | * zero, proceed with automatic r/w */ | 1526 | * zero, proceed with automatic r/w */ |
| 1537 | if (table->data && table->maxlen) { | 1527 | if (table->data && table->maxlen) { |
| 1538 | rc = sysctl_data(table, name, nlen, oldval, oldlenp, | 1528 | rc = sysctl_data(table, oldval, oldlenp, newval, newlen); |
| 1539 | newval, newlen); | ||
| 1540 | if (rc < 0) | 1529 | if (rc < 0) |
| 1541 | return rc; | 1530 | return rc; |
| 1542 | } | 1531 | } |
| @@ -1568,7 +1557,7 @@ repeat: | |||
| 1568 | table = table->child; | 1557 | table = table->child; |
| 1569 | goto repeat; | 1558 | goto repeat; |
| 1570 | } | 1559 | } |
| 1571 | error = do_sysctl_strategy(root, table, name, nlen, | 1560 | error = do_sysctl_strategy(root, table, |
| 1572 | oldval, oldlenp, | 1561 | oldval, oldlenp, |
| 1573 | newval, newlen); | 1562 | newval, newlen); |
| 1574 | return error; | 1563 | return error; |
| @@ -2237,49 +2226,39 @@ int proc_dointvec(struct ctl_table *table, int write, struct file *filp, | |||
| 2237 | NULL,NULL); | 2226 | NULL,NULL); |
| 2238 | } | 2227 | } |
| 2239 | 2228 | ||
| 2240 | #define OP_SET 0 | ||
| 2241 | #define OP_AND 1 | ||
| 2242 | #define OP_OR 2 | ||
| 2243 | |||
| 2244 | static int do_proc_dointvec_bset_conv(int *negp, unsigned long *lvalp, | ||
| 2245 | int *valp, | ||
| 2246 | int write, void *data) | ||
| 2247 | { | ||
| 2248 | int op = *(int *)data; | ||
| 2249 | if (write) { | ||
| 2250 | int val = *negp ? -*lvalp : *lvalp; | ||
| 2251 | switch(op) { | ||
| 2252 | case OP_SET: *valp = val; break; | ||
| 2253 | case OP_AND: *valp &= val; break; | ||
| 2254 | case OP_OR: *valp |= val; break; | ||
| 2255 | } | ||
| 2256 | } else { | ||
| 2257 | int val = *valp; | ||
| 2258 | if (val < 0) { | ||
| 2259 | *negp = -1; | ||
| 2260 | *lvalp = (unsigned long)-val; | ||
| 2261 | } else { | ||
| 2262 | *negp = 0; | ||
| 2263 | *lvalp = (unsigned long)val; | ||
| 2264 | } | ||
| 2265 | } | ||
| 2266 | return 0; | ||
| 2267 | } | ||
| 2268 | |||
| 2269 | /* | 2229 | /* |
| 2270 | * Taint values can only be increased | 2230 | * Taint values can only be increased |
| 2231 | * This means we can safely use a temporary. | ||
| 2271 | */ | 2232 | */ |
| 2272 | static int proc_dointvec_taint(struct ctl_table *table, int write, struct file *filp, | 2233 | static int proc_taint(struct ctl_table *table, int write, struct file *filp, |
| 2273 | void __user *buffer, size_t *lenp, loff_t *ppos) | 2234 | void __user *buffer, size_t *lenp, loff_t *ppos) |
| 2274 | { | 2235 | { |
| 2275 | int op; | 2236 | struct ctl_table t; |
| 2237 | unsigned long tmptaint = get_taint(); | ||
| 2238 | int err; | ||
| 2276 | 2239 | ||
| 2277 | if (write && !capable(CAP_SYS_ADMIN)) | 2240 | if (write && !capable(CAP_SYS_ADMIN)) |
| 2278 | return -EPERM; | 2241 | return -EPERM; |
| 2279 | 2242 | ||
| 2280 | op = OP_OR; | 2243 | t = *table; |
| 2281 | return do_proc_dointvec(table,write,filp,buffer,lenp,ppos, | 2244 | t.data = &tmptaint; |
| 2282 | do_proc_dointvec_bset_conv,&op); | 2245 | err = proc_doulongvec_minmax(&t, write, filp, buffer, lenp, ppos); |
| 2246 | if (err < 0) | ||
| 2247 | return err; | ||
| 2248 | |||
| 2249 | if (write) { | ||
| 2250 | /* | ||
| 2251 | * Poor man's atomic or. Not worth adding a primitive | ||
| 2252 | * to everyone's atomic.h for this | ||
| 2253 | */ | ||
| 2254 | int i; | ||
| 2255 | for (i = 0; i < BITS_PER_LONG && tmptaint >> i; i++) { | ||
| 2256 | if ((tmptaint >> i) & 1) | ||
| 2257 | add_taint(i); | ||
| 2258 | } | ||
| 2259 | } | ||
| 2260 | |||
| 2261 | return err; | ||
| 2283 | } | 2262 | } |
| 2284 | 2263 | ||
| 2285 | struct do_proc_dointvec_minmax_conv_param { | 2264 | struct do_proc_dointvec_minmax_conv_param { |
| @@ -2727,7 +2706,7 @@ int proc_doulongvec_ms_jiffies_minmax(struct ctl_table *table, int write, | |||
| 2727 | */ | 2706 | */ |
| 2728 | 2707 | ||
| 2729 | /* The generic sysctl data routine (used if no strategy routine supplied) */ | 2708 | /* The generic sysctl data routine (used if no strategy routine supplied) */ |
| 2730 | int sysctl_data(struct ctl_table *table, int __user *name, int nlen, | 2709 | int sysctl_data(struct ctl_table *table, |
| 2731 | void __user *oldval, size_t __user *oldlenp, | 2710 | void __user *oldval, size_t __user *oldlenp, |
| 2732 | void __user *newval, size_t newlen) | 2711 | void __user *newval, size_t newlen) |
| 2733 | { | 2712 | { |
| @@ -2761,7 +2740,7 @@ int sysctl_data(struct ctl_table *table, int __user *name, int nlen, | |||
| 2761 | } | 2740 | } |
| 2762 | 2741 | ||
| 2763 | /* The generic string strategy routine: */ | 2742 | /* The generic string strategy routine: */ |
| 2764 | int sysctl_string(struct ctl_table *table, int __user *name, int nlen, | 2743 | int sysctl_string(struct ctl_table *table, |
| 2765 | void __user *oldval, size_t __user *oldlenp, | 2744 | void __user *oldval, size_t __user *oldlenp, |
| 2766 | void __user *newval, size_t newlen) | 2745 | void __user *newval, size_t newlen) |
| 2767 | { | 2746 | { |
| @@ -2807,7 +2786,7 @@ int sysctl_string(struct ctl_table *table, int __user *name, int nlen, | |||
| 2807 | * are between the minimum and maximum values given in the arrays | 2786 | * are between the minimum and maximum values given in the arrays |
| 2808 | * table->extra1 and table->extra2, respectively. | 2787 | * table->extra1 and table->extra2, respectively. |
| 2809 | */ | 2788 | */ |
| 2810 | int sysctl_intvec(struct ctl_table *table, int __user *name, int nlen, | 2789 | int sysctl_intvec(struct ctl_table *table, |
| 2811 | void __user *oldval, size_t __user *oldlenp, | 2790 | void __user *oldval, size_t __user *oldlenp, |
| 2812 | void __user *newval, size_t newlen) | 2791 | void __user *newval, size_t newlen) |
| 2813 | { | 2792 | { |
| @@ -2843,7 +2822,7 @@ int sysctl_intvec(struct ctl_table *table, int __user *name, int nlen, | |||
| 2843 | } | 2822 | } |
| 2844 | 2823 | ||
| 2845 | /* Strategy function to convert jiffies to seconds */ | 2824 | /* Strategy function to convert jiffies to seconds */ |
| 2846 | int sysctl_jiffies(struct ctl_table *table, int __user *name, int nlen, | 2825 | int sysctl_jiffies(struct ctl_table *table, |
| 2847 | void __user *oldval, size_t __user *oldlenp, | 2826 | void __user *oldval, size_t __user *oldlenp, |
| 2848 | void __user *newval, size_t newlen) | 2827 | void __user *newval, size_t newlen) |
| 2849 | { | 2828 | { |
| @@ -2877,7 +2856,7 @@ int sysctl_jiffies(struct ctl_table *table, int __user *name, int nlen, | |||
| 2877 | } | 2856 | } |
| 2878 | 2857 | ||
| 2879 | /* Strategy function to convert jiffies to seconds */ | 2858 | /* Strategy function to convert jiffies to seconds */ |
| 2880 | int sysctl_ms_jiffies(struct ctl_table *table, int __user *name, int nlen, | 2859 | int sysctl_ms_jiffies(struct ctl_table *table, |
| 2881 | void __user *oldval, size_t __user *oldlenp, | 2860 | void __user *oldval, size_t __user *oldlenp, |
| 2882 | void __user *newval, size_t newlen) | 2861 | void __user *newval, size_t newlen) |
| 2883 | { | 2862 | { |
| @@ -2932,35 +2911,35 @@ asmlinkage long sys_sysctl(struct __sysctl_args __user *args) | |||
| 2932 | return error; | 2911 | return error; |
| 2933 | } | 2912 | } |
| 2934 | 2913 | ||
| 2935 | int sysctl_data(struct ctl_table *table, int __user *name, int nlen, | 2914 | int sysctl_data(struct ctl_table *table, |
| 2936 | void __user *oldval, size_t __user *oldlenp, | 2915 | void __user *oldval, size_t __user *oldlenp, |
| 2937 | void __user *newval, size_t newlen) | 2916 | void __user *newval, size_t newlen) |
| 2938 | { | 2917 | { |
| 2939 | return -ENOSYS; | 2918 | return -ENOSYS; |
| 2940 | } | 2919 | } |
| 2941 | 2920 | ||
| 2942 | int sysctl_string(struct ctl_table *table, int __user *name, int nlen, | 2921 | int sysctl_string(struct ctl_table *table, |
| 2943 | void __user *oldval, size_t __user *oldlenp, | 2922 | void __user *oldval, size_t __user *oldlenp, |
| 2944 | void __user *newval, size_t newlen) | 2923 | void __user *newval, size_t newlen) |
| 2945 | { | 2924 | { |
| 2946 | return -ENOSYS; | 2925 | return -ENOSYS; |
| 2947 | } | 2926 | } |
| 2948 | 2927 | ||
| 2949 | int sysctl_intvec(struct ctl_table *table, int __user *name, int nlen, | 2928 | int sysctl_intvec(struct ctl_table *table, |
| 2950 | void __user *oldval, size_t __user *oldlenp, | 2929 | void __user *oldval, size_t __user *oldlenp, |
| 2951 | void __user *newval, size_t newlen) | 2930 | void __user *newval, size_t newlen) |
| 2952 | { | 2931 | { |
| 2953 | return -ENOSYS; | 2932 | return -ENOSYS; |
| 2954 | } | 2933 | } |
| 2955 | 2934 | ||
| 2956 | int sysctl_jiffies(struct ctl_table *table, int __user *name, int nlen, | 2935 | int sysctl_jiffies(struct ctl_table *table, |
| 2957 | void __user *oldval, size_t __user *oldlenp, | 2936 | void __user *oldval, size_t __user *oldlenp, |
| 2958 | void __user *newval, size_t newlen) | 2937 | void __user *newval, size_t newlen) |
| 2959 | { | 2938 | { |
| 2960 | return -ENOSYS; | 2939 | return -ENOSYS; |
| 2961 | } | 2940 | } |
| 2962 | 2941 | ||
| 2963 | int sysctl_ms_jiffies(struct ctl_table *table, int __user *name, int nlen, | 2942 | int sysctl_ms_jiffies(struct ctl_table *table, |
| 2964 | void __user *oldval, size_t __user *oldlenp, | 2943 | void __user *oldval, size_t __user *oldlenp, |
| 2965 | void __user *newval, size_t newlen) | 2944 | void __user *newval, size_t newlen) |
| 2966 | { | 2945 | { |
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig index 8d53106a0a92..95ed42951e0a 100644 --- a/kernel/time/Kconfig +++ b/kernel/time/Kconfig | |||
| @@ -3,7 +3,6 @@ | |||
| 3 | # | 3 | # |
| 4 | config TICK_ONESHOT | 4 | config TICK_ONESHOT |
| 5 | bool | 5 | bool |
| 6 | default n | ||
| 7 | 6 | ||
| 8 | config NO_HZ | 7 | config NO_HZ |
| 9 | bool "Tickless System (Dynamic Ticks)" | 8 | bool "Tickless System (Dynamic Ticks)" |
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 3d1e3e1a1971..f8d968063cea 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c | |||
| @@ -72,6 +72,16 @@ void clockevents_set_mode(struct clock_event_device *dev, | |||
| 72 | } | 72 | } |
| 73 | 73 | ||
| 74 | /** | 74 | /** |
| 75 | * clockevents_shutdown - shutdown the device and clear next_event | ||
| 76 | * @dev: device to shutdown | ||
| 77 | */ | ||
| 78 | void clockevents_shutdown(struct clock_event_device *dev) | ||
| 79 | { | ||
| 80 | clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN); | ||
| 81 | dev->next_event.tv64 = KTIME_MAX; | ||
| 82 | } | ||
| 83 | |||
| 84 | /** | ||
| 75 | * clockevents_program_event - Reprogram the clock event device. | 85 | * clockevents_program_event - Reprogram the clock event device. |
| 76 | * @expires: absolute expiry time (monotonic clock) | 86 | * @expires: absolute expiry time (monotonic clock) |
| 77 | * | 87 | * |
| @@ -177,7 +187,7 @@ void clockevents_register_device(struct clock_event_device *dev) | |||
| 177 | /* | 187 | /* |
| 178 | * Noop handler when we shut down an event device | 188 | * Noop handler when we shut down an event device |
| 179 | */ | 189 | */ |
| 180 | static void clockevents_handle_noop(struct clock_event_device *dev) | 190 | void clockevents_handle_noop(struct clock_event_device *dev) |
| 181 | { | 191 | { |
| 182 | } | 192 | } |
| 183 | 193 | ||
| @@ -199,7 +209,6 @@ void clockevents_exchange_device(struct clock_event_device *old, | |||
| 199 | * released list and do a notify add later. | 209 | * released list and do a notify add later. |
| 200 | */ | 210 | */ |
| 201 | if (old) { | 211 | if (old) { |
| 202 | old->event_handler = clockevents_handle_noop; | ||
| 203 | clockevents_set_mode(old, CLOCK_EVT_MODE_UNUSED); | 212 | clockevents_set_mode(old, CLOCK_EVT_MODE_UNUSED); |
| 204 | list_del(&old->list); | 213 | list_del(&old->list); |
| 205 | list_add(&old->list, &clockevents_released); | 214 | list_add(&old->list, &clockevents_released); |
| @@ -207,7 +216,7 @@ void clockevents_exchange_device(struct clock_event_device *old, | |||
| 207 | 216 | ||
| 208 | if (new) { | 217 | if (new) { |
| 209 | BUG_ON(new->mode != CLOCK_EVT_MODE_UNUSED); | 218 | BUG_ON(new->mode != CLOCK_EVT_MODE_UNUSED); |
| 210 | clockevents_set_mode(new, CLOCK_EVT_MODE_SHUTDOWN); | 219 | clockevents_shutdown(new); |
| 211 | } | 220 | } |
| 212 | local_irq_restore(flags); | 221 | local_irq_restore(flags); |
| 213 | } | 222 | } |
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index 4c8d85421d24..9c114b726ab3 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c | |||
| @@ -244,7 +244,7 @@ static void sync_cmos_clock(unsigned long dummy) | |||
| 244 | if (abs(now.tv_nsec - (NSEC_PER_SEC / 2)) <= tick_nsec / 2) | 244 | if (abs(now.tv_nsec - (NSEC_PER_SEC / 2)) <= tick_nsec / 2) |
| 245 | fail = update_persistent_clock(now); | 245 | fail = update_persistent_clock(now); |
| 246 | 246 | ||
| 247 | next.tv_nsec = (NSEC_PER_SEC / 2) - now.tv_nsec; | 247 | next.tv_nsec = (NSEC_PER_SEC / 2) - now.tv_nsec - (TICK_NSEC / 2); |
| 248 | if (next.tv_nsec <= 0) | 248 | if (next.tv_nsec <= 0) |
| 249 | next.tv_nsec += NSEC_PER_SEC; | 249 | next.tv_nsec += NSEC_PER_SEC; |
| 250 | 250 | ||
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 31463d370b94..cb01cd8f919b 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c | |||
| @@ -175,6 +175,8 @@ static void tick_do_periodic_broadcast(void) | |||
| 175 | */ | 175 | */ |
| 176 | static void tick_handle_periodic_broadcast(struct clock_event_device *dev) | 176 | static void tick_handle_periodic_broadcast(struct clock_event_device *dev) |
| 177 | { | 177 | { |
| 178 | ktime_t next; | ||
| 179 | |||
| 178 | tick_do_periodic_broadcast(); | 180 | tick_do_periodic_broadcast(); |
| 179 | 181 | ||
| 180 | /* | 182 | /* |
| @@ -185,10 +187,13 @@ static void tick_handle_periodic_broadcast(struct clock_event_device *dev) | |||
| 185 | 187 | ||
| 186 | /* | 188 | /* |
| 187 | * Setup the next period for devices, which do not have | 189 | * Setup the next period for devices, which do not have |
| 188 | * periodic mode: | 190 | * periodic mode. We read dev->next_event first and add to it |
| 191 | * when the event alrady expired. clockevents_program_event() | ||
| 192 | * sets dev->next_event only when the event is really | ||
| 193 | * programmed to the device. | ||
| 189 | */ | 194 | */ |
| 190 | for (;;) { | 195 | for (next = dev->next_event; ;) { |
| 191 | ktime_t next = ktime_add(dev->next_event, tick_period); | 196 | next = ktime_add(next, tick_period); |
| 192 | 197 | ||
| 193 | if (!clockevents_program_event(dev, next, ktime_get())) | 198 | if (!clockevents_program_event(dev, next, ktime_get())) |
| 194 | return; | 199 | return; |
| @@ -205,7 +210,7 @@ static void tick_do_broadcast_on_off(void *why) | |||
| 205 | struct clock_event_device *bc, *dev; | 210 | struct clock_event_device *bc, *dev; |
| 206 | struct tick_device *td; | 211 | struct tick_device *td; |
| 207 | unsigned long flags, *reason = why; | 212 | unsigned long flags, *reason = why; |
| 208 | int cpu; | 213 | int cpu, bc_stopped; |
| 209 | 214 | ||
| 210 | spin_lock_irqsave(&tick_broadcast_lock, flags); | 215 | spin_lock_irqsave(&tick_broadcast_lock, flags); |
| 211 | 216 | ||
| @@ -223,14 +228,16 @@ static void tick_do_broadcast_on_off(void *why) | |||
| 223 | if (!tick_device_is_functional(dev)) | 228 | if (!tick_device_is_functional(dev)) |
| 224 | goto out; | 229 | goto out; |
| 225 | 230 | ||
| 231 | bc_stopped = cpus_empty(tick_broadcast_mask); | ||
| 232 | |||
| 226 | switch (*reason) { | 233 | switch (*reason) { |
| 227 | case CLOCK_EVT_NOTIFY_BROADCAST_ON: | 234 | case CLOCK_EVT_NOTIFY_BROADCAST_ON: |
| 228 | case CLOCK_EVT_NOTIFY_BROADCAST_FORCE: | 235 | case CLOCK_EVT_NOTIFY_BROADCAST_FORCE: |
| 229 | if (!cpu_isset(cpu, tick_broadcast_mask)) { | 236 | if (!cpu_isset(cpu, tick_broadcast_mask)) { |
| 230 | cpu_set(cpu, tick_broadcast_mask); | 237 | cpu_set(cpu, tick_broadcast_mask); |
| 231 | if (td->mode == TICKDEV_MODE_PERIODIC) | 238 | if (tick_broadcast_device.mode == |
| 232 | clockevents_set_mode(dev, | 239 | TICKDEV_MODE_PERIODIC) |
| 233 | CLOCK_EVT_MODE_SHUTDOWN); | 240 | clockevents_shutdown(dev); |
| 234 | } | 241 | } |
| 235 | if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE) | 242 | if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE) |
| 236 | tick_broadcast_force = 1; | 243 | tick_broadcast_force = 1; |
| @@ -239,15 +246,17 @@ static void tick_do_broadcast_on_off(void *why) | |||
| 239 | if (!tick_broadcast_force && | 246 | if (!tick_broadcast_force && |
| 240 | cpu_isset(cpu, tick_broadcast_mask)) { | 247 | cpu_isset(cpu, tick_broadcast_mask)) { |
| 241 | cpu_clear(cpu, tick_broadcast_mask); | 248 | cpu_clear(cpu, tick_broadcast_mask); |
| 242 | if (td->mode == TICKDEV_MODE_PERIODIC) | 249 | if (tick_broadcast_device.mode == |
| 250 | TICKDEV_MODE_PERIODIC) | ||
| 243 | tick_setup_periodic(dev, 0); | 251 | tick_setup_periodic(dev, 0); |
| 244 | } | 252 | } |
| 245 | break; | 253 | break; |
| 246 | } | 254 | } |
| 247 | 255 | ||
| 248 | if (cpus_empty(tick_broadcast_mask)) | 256 | if (cpus_empty(tick_broadcast_mask)) { |
| 249 | clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN); | 257 | if (!bc_stopped) |
| 250 | else { | 258 | clockevents_shutdown(bc); |
| 259 | } else if (bc_stopped) { | ||
| 251 | if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) | 260 | if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) |
| 252 | tick_broadcast_start_periodic(bc); | 261 | tick_broadcast_start_periodic(bc); |
| 253 | else | 262 | else |
| @@ -298,7 +307,7 @@ void tick_shutdown_broadcast(unsigned int *cpup) | |||
| 298 | 307 | ||
| 299 | if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) { | 308 | if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) { |
| 300 | if (bc && cpus_empty(tick_broadcast_mask)) | 309 | if (bc && cpus_empty(tick_broadcast_mask)) |
| 301 | clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN); | 310 | clockevents_shutdown(bc); |
| 302 | } | 311 | } |
| 303 | 312 | ||
| 304 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); | 313 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); |
| @@ -313,7 +322,7 @@ void tick_suspend_broadcast(void) | |||
| 313 | 322 | ||
| 314 | bc = tick_broadcast_device.evtdev; | 323 | bc = tick_broadcast_device.evtdev; |
| 315 | if (bc) | 324 | if (bc) |
| 316 | clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN); | 325 | clockevents_shutdown(bc); |
| 317 | 326 | ||
| 318 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); | 327 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); |
| 319 | } | 328 | } |
| @@ -364,16 +373,8 @@ cpumask_t *tick_get_broadcast_oneshot_mask(void) | |||
| 364 | static int tick_broadcast_set_event(ktime_t expires, int force) | 373 | static int tick_broadcast_set_event(ktime_t expires, int force) |
| 365 | { | 374 | { |
| 366 | struct clock_event_device *bc = tick_broadcast_device.evtdev; | 375 | struct clock_event_device *bc = tick_broadcast_device.evtdev; |
| 367 | ktime_t now = ktime_get(); | 376 | |
| 368 | int res; | 377 | return tick_dev_program_event(bc, expires, force); |
| 369 | |||
| 370 | for(;;) { | ||
| 371 | res = clockevents_program_event(bc, expires, now); | ||
| 372 | if (!res || !force) | ||
| 373 | return res; | ||
| 374 | now = ktime_get(); | ||
| 375 | expires = ktime_add(now, ktime_set(0, bc->min_delta_ns)); | ||
| 376 | } | ||
| 377 | } | 378 | } |
| 378 | 379 | ||
| 379 | int tick_resume_broadcast_oneshot(struct clock_event_device *bc) | 380 | int tick_resume_broadcast_oneshot(struct clock_event_device *bc) |
| @@ -491,14 +492,52 @@ static void tick_broadcast_clear_oneshot(int cpu) | |||
| 491 | cpu_clear(cpu, tick_broadcast_oneshot_mask); | 492 | cpu_clear(cpu, tick_broadcast_oneshot_mask); |
| 492 | } | 493 | } |
| 493 | 494 | ||
| 495 | static void tick_broadcast_init_next_event(cpumask_t *mask, ktime_t expires) | ||
| 496 | { | ||
| 497 | struct tick_device *td; | ||
| 498 | int cpu; | ||
| 499 | |||
| 500 | for_each_cpu_mask_nr(cpu, *mask) { | ||
| 501 | td = &per_cpu(tick_cpu_device, cpu); | ||
| 502 | if (td->evtdev) | ||
| 503 | td->evtdev->next_event = expires; | ||
| 504 | } | ||
| 505 | } | ||
| 506 | |||
| 494 | /** | 507 | /** |
| 495 | * tick_broadcast_setup_oneshot - setup the broadcast device | 508 | * tick_broadcast_setup_oneshot - setup the broadcast device |
| 496 | */ | 509 | */ |
| 497 | void tick_broadcast_setup_oneshot(struct clock_event_device *bc) | 510 | void tick_broadcast_setup_oneshot(struct clock_event_device *bc) |
| 498 | { | 511 | { |
| 499 | bc->event_handler = tick_handle_oneshot_broadcast; | 512 | /* Set it up only once ! */ |
| 500 | clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); | 513 | if (bc->event_handler != tick_handle_oneshot_broadcast) { |
| 501 | bc->next_event.tv64 = KTIME_MAX; | 514 | int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC; |
| 515 | int cpu = smp_processor_id(); | ||
| 516 | cpumask_t mask; | ||
| 517 | |||
| 518 | bc->event_handler = tick_handle_oneshot_broadcast; | ||
| 519 | clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); | ||
| 520 | |||
| 521 | /* Take the do_timer update */ | ||
| 522 | tick_do_timer_cpu = cpu; | ||
| 523 | |||
| 524 | /* | ||
| 525 | * We must be careful here. There might be other CPUs | ||
| 526 | * waiting for periodic broadcast. We need to set the | ||
| 527 | * oneshot_mask bits for those and program the | ||
| 528 | * broadcast device to fire. | ||
| 529 | */ | ||
| 530 | mask = tick_broadcast_mask; | ||
| 531 | cpu_clear(cpu, mask); | ||
| 532 | cpus_or(tick_broadcast_oneshot_mask, | ||
| 533 | tick_broadcast_oneshot_mask, mask); | ||
| 534 | |||
| 535 | if (was_periodic && !cpus_empty(mask)) { | ||
| 536 | tick_broadcast_init_next_event(&mask, tick_next_period); | ||
| 537 | tick_broadcast_set_event(tick_next_period, 1); | ||
| 538 | } else | ||
| 539 | bc->next_event.tv64 = KTIME_MAX; | ||
| 540 | } | ||
| 502 | } | 541 | } |
| 503 | 542 | ||
| 504 | /* | 543 | /* |
| @@ -538,4 +577,12 @@ void tick_shutdown_broadcast_oneshot(unsigned int *cpup) | |||
| 538 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); | 577 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); |
| 539 | } | 578 | } |
| 540 | 579 | ||
| 580 | /* | ||
| 581 | * Check, whether the broadcast device is in one shot mode | ||
| 582 | */ | ||
| 583 | int tick_broadcast_oneshot_active(void) | ||
| 584 | { | ||
| 585 | return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT; | ||
| 586 | } | ||
| 587 | |||
| 541 | #endif | 588 | #endif |
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c index 80c4336f4188..df12434b43ca 100644 --- a/kernel/time/tick-common.c +++ b/kernel/time/tick-common.c | |||
| @@ -33,7 +33,7 @@ DEFINE_PER_CPU(struct tick_device, tick_cpu_device); | |||
| 33 | */ | 33 | */ |
| 34 | ktime_t tick_next_period; | 34 | ktime_t tick_next_period; |
| 35 | ktime_t tick_period; | 35 | ktime_t tick_period; |
| 36 | int tick_do_timer_cpu __read_mostly = -1; | 36 | int tick_do_timer_cpu __read_mostly = TICK_DO_TIMER_BOOT; |
| 37 | DEFINE_SPINLOCK(tick_device_lock); | 37 | DEFINE_SPINLOCK(tick_device_lock); |
| 38 | 38 | ||
| 39 | /* | 39 | /* |
| @@ -109,7 +109,8 @@ void tick_setup_periodic(struct clock_event_device *dev, int broadcast) | |||
| 109 | if (!tick_device_is_functional(dev)) | 109 | if (!tick_device_is_functional(dev)) |
| 110 | return; | 110 | return; |
| 111 | 111 | ||
| 112 | if (dev->features & CLOCK_EVT_FEAT_PERIODIC) { | 112 | if ((dev->features & CLOCK_EVT_FEAT_PERIODIC) && |
| 113 | !tick_broadcast_oneshot_active()) { | ||
| 113 | clockevents_set_mode(dev, CLOCK_EVT_MODE_PERIODIC); | 114 | clockevents_set_mode(dev, CLOCK_EVT_MODE_PERIODIC); |
| 114 | } else { | 115 | } else { |
| 115 | unsigned long seq; | 116 | unsigned long seq; |
| @@ -148,7 +149,7 @@ static void tick_setup_device(struct tick_device *td, | |||
| 148 | * If no cpu took the do_timer update, assign it to | 149 | * If no cpu took the do_timer update, assign it to |
| 149 | * this cpu: | 150 | * this cpu: |
| 150 | */ | 151 | */ |
| 151 | if (tick_do_timer_cpu == -1) { | 152 | if (tick_do_timer_cpu == TICK_DO_TIMER_BOOT) { |
| 152 | tick_do_timer_cpu = cpu; | 153 | tick_do_timer_cpu = cpu; |
| 153 | tick_next_period = ktime_get(); | 154 | tick_next_period = ktime_get(); |
| 154 | tick_period = ktime_set(0, NSEC_PER_SEC / HZ); | 155 | tick_period = ktime_set(0, NSEC_PER_SEC / HZ); |
| @@ -161,6 +162,7 @@ static void tick_setup_device(struct tick_device *td, | |||
| 161 | } else { | 162 | } else { |
| 162 | handler = td->evtdev->event_handler; | 163 | handler = td->evtdev->event_handler; |
| 163 | next_event = td->evtdev->next_event; | 164 | next_event = td->evtdev->next_event; |
| 165 | td->evtdev->event_handler = clockevents_handle_noop; | ||
| 164 | } | 166 | } |
| 165 | 167 | ||
| 166 | td->evtdev = newdev; | 168 | td->evtdev = newdev; |
| @@ -248,7 +250,7 @@ static int tick_check_new_device(struct clock_event_device *newdev) | |||
| 248 | * not give it back to the clockevents layer ! | 250 | * not give it back to the clockevents layer ! |
| 249 | */ | 251 | */ |
| 250 | if (tick_is_broadcast_device(curdev)) { | 252 | if (tick_is_broadcast_device(curdev)) { |
| 251 | clockevents_set_mode(curdev, CLOCK_EVT_MODE_SHUTDOWN); | 253 | clockevents_shutdown(curdev); |
| 252 | curdev = NULL; | 254 | curdev = NULL; |
| 253 | } | 255 | } |
| 254 | clockevents_exchange_device(curdev, newdev); | 256 | clockevents_exchange_device(curdev, newdev); |
| @@ -299,7 +301,8 @@ static void tick_shutdown(unsigned int *cpup) | |||
| 299 | if (*cpup == tick_do_timer_cpu) { | 301 | if (*cpup == tick_do_timer_cpu) { |
| 300 | int cpu = first_cpu(cpu_online_map); | 302 | int cpu = first_cpu(cpu_online_map); |
| 301 | 303 | ||
| 302 | tick_do_timer_cpu = (cpu != NR_CPUS) ? cpu : -1; | 304 | tick_do_timer_cpu = (cpu != NR_CPUS) ? cpu : |
| 305 | TICK_DO_TIMER_NONE; | ||
| 303 | } | 306 | } |
| 304 | spin_unlock_irqrestore(&tick_device_lock, flags); | 307 | spin_unlock_irqrestore(&tick_device_lock, flags); |
| 305 | } | 308 | } |
| @@ -310,7 +313,7 @@ static void tick_suspend(void) | |||
| 310 | unsigned long flags; | 313 | unsigned long flags; |
| 311 | 314 | ||
| 312 | spin_lock_irqsave(&tick_device_lock, flags); | 315 | spin_lock_irqsave(&tick_device_lock, flags); |
| 313 | clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_SHUTDOWN); | 316 | clockevents_shutdown(td->evtdev); |
| 314 | spin_unlock_irqrestore(&tick_device_lock, flags); | 317 | spin_unlock_irqrestore(&tick_device_lock, flags); |
| 315 | } | 318 | } |
| 316 | 319 | ||
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index f13f2b7f4fd4..469248782c23 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h | |||
| @@ -1,6 +1,10 @@ | |||
| 1 | /* | 1 | /* |
| 2 | * tick internal variable and functions used by low/high res code | 2 | * tick internal variable and functions used by low/high res code |
| 3 | */ | 3 | */ |
| 4 | |||
| 5 | #define TICK_DO_TIMER_NONE -1 | ||
| 6 | #define TICK_DO_TIMER_BOOT -2 | ||
| 7 | |||
| 4 | DECLARE_PER_CPU(struct tick_device, tick_cpu_device); | 8 | DECLARE_PER_CPU(struct tick_device, tick_cpu_device); |
| 5 | extern spinlock_t tick_device_lock; | 9 | extern spinlock_t tick_device_lock; |
| 6 | extern ktime_t tick_next_period; | 10 | extern ktime_t tick_next_period; |
| @@ -10,6 +14,8 @@ extern int tick_do_timer_cpu __read_mostly; | |||
| 10 | extern void tick_setup_periodic(struct clock_event_device *dev, int broadcast); | 14 | extern void tick_setup_periodic(struct clock_event_device *dev, int broadcast); |
| 11 | extern void tick_handle_periodic(struct clock_event_device *dev); | 15 | extern void tick_handle_periodic(struct clock_event_device *dev); |
| 12 | 16 | ||
| 17 | extern void clockevents_shutdown(struct clock_event_device *dev); | ||
| 18 | |||
| 13 | /* | 19 | /* |
| 14 | * NO_HZ / high resolution timer shared code | 20 | * NO_HZ / high resolution timer shared code |
| 15 | */ | 21 | */ |
| @@ -17,6 +23,8 @@ extern void tick_handle_periodic(struct clock_event_device *dev); | |||
| 17 | extern void tick_setup_oneshot(struct clock_event_device *newdev, | 23 | extern void tick_setup_oneshot(struct clock_event_device *newdev, |
| 18 | void (*handler)(struct clock_event_device *), | 24 | void (*handler)(struct clock_event_device *), |
| 19 | ktime_t nextevt); | 25 | ktime_t nextevt); |
| 26 | extern int tick_dev_program_event(struct clock_event_device *dev, | ||
| 27 | ktime_t expires, int force); | ||
| 20 | extern int tick_program_event(ktime_t expires, int force); | 28 | extern int tick_program_event(ktime_t expires, int force); |
| 21 | extern void tick_oneshot_notify(void); | 29 | extern void tick_oneshot_notify(void); |
| 22 | extern int tick_switch_to_oneshot(void (*handler)(struct clock_event_device *)); | 30 | extern int tick_switch_to_oneshot(void (*handler)(struct clock_event_device *)); |
| @@ -27,6 +35,7 @@ extern void tick_broadcast_oneshot_control(unsigned long reason); | |||
| 27 | extern void tick_broadcast_switch_to_oneshot(void); | 35 | extern void tick_broadcast_switch_to_oneshot(void); |
| 28 | extern void tick_shutdown_broadcast_oneshot(unsigned int *cpup); | 36 | extern void tick_shutdown_broadcast_oneshot(unsigned int *cpup); |
| 29 | extern int tick_resume_broadcast_oneshot(struct clock_event_device *bc); | 37 | extern int tick_resume_broadcast_oneshot(struct clock_event_device *bc); |
| 38 | extern int tick_broadcast_oneshot_active(void); | ||
| 30 | # else /* BROADCAST */ | 39 | # else /* BROADCAST */ |
| 31 | static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc) | 40 | static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc) |
| 32 | { | 41 | { |
| @@ -35,6 +44,7 @@ static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc) | |||
| 35 | static inline void tick_broadcast_oneshot_control(unsigned long reason) { } | 44 | static inline void tick_broadcast_oneshot_control(unsigned long reason) { } |
| 36 | static inline void tick_broadcast_switch_to_oneshot(void) { } | 45 | static inline void tick_broadcast_switch_to_oneshot(void) { } |
| 37 | static inline void tick_shutdown_broadcast_oneshot(unsigned int *cpup) { } | 46 | static inline void tick_shutdown_broadcast_oneshot(unsigned int *cpup) { } |
| 47 | static inline int tick_broadcast_oneshot_active(void) { return 0; } | ||
| 38 | # endif /* !BROADCAST */ | 48 | # endif /* !BROADCAST */ |
| 39 | 49 | ||
| 40 | #else /* !ONESHOT */ | 50 | #else /* !ONESHOT */ |
| @@ -64,6 +74,7 @@ static inline int tick_resume_broadcast_oneshot(struct clock_event_device *bc) | |||
| 64 | { | 74 | { |
| 65 | return 0; | 75 | return 0; |
| 66 | } | 76 | } |
| 77 | static inline int tick_broadcast_oneshot_active(void) { return 0; } | ||
| 67 | #endif /* !TICK_ONESHOT */ | 78 | #endif /* !TICK_ONESHOT */ |
| 68 | 79 | ||
| 69 | /* | 80 | /* |
diff --git a/kernel/time/tick-oneshot.c b/kernel/time/tick-oneshot.c index 450c04935b66..2e8de678e767 100644 --- a/kernel/time/tick-oneshot.c +++ b/kernel/time/tick-oneshot.c | |||
| @@ -23,24 +23,56 @@ | |||
| 23 | #include "tick-internal.h" | 23 | #include "tick-internal.h" |
| 24 | 24 | ||
| 25 | /** | 25 | /** |
| 26 | * tick_program_event | 26 | * tick_program_event internal worker function |
| 27 | */ | 27 | */ |
| 28 | int tick_program_event(ktime_t expires, int force) | 28 | int tick_dev_program_event(struct clock_event_device *dev, ktime_t expires, |
| 29 | int force) | ||
| 29 | { | 30 | { |
| 30 | struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev; | ||
| 31 | ktime_t now = ktime_get(); | 31 | ktime_t now = ktime_get(); |
| 32 | int i; | ||
| 32 | 33 | ||
| 33 | while (1) { | 34 | for (i = 0;;) { |
| 34 | int ret = clockevents_program_event(dev, expires, now); | 35 | int ret = clockevents_program_event(dev, expires, now); |
| 35 | 36 | ||
| 36 | if (!ret || !force) | 37 | if (!ret || !force) |
| 37 | return ret; | 38 | return ret; |
| 39 | |||
| 40 | /* | ||
| 41 | * We tried 2 times to program the device with the given | ||
| 42 | * min_delta_ns. If that's not working then we double it | ||
| 43 | * and emit a warning. | ||
| 44 | */ | ||
| 45 | if (++i > 2) { | ||
| 46 | /* Increase the min. delta and try again */ | ||
| 47 | if (!dev->min_delta_ns) | ||
| 48 | dev->min_delta_ns = 5000; | ||
| 49 | else | ||
| 50 | dev->min_delta_ns += dev->min_delta_ns >> 1; | ||
| 51 | |||
| 52 | printk(KERN_WARNING | ||
| 53 | "CE: %s increasing min_delta_ns to %lu nsec\n", | ||
| 54 | dev->name ? dev->name : "?", | ||
| 55 | dev->min_delta_ns << 1); | ||
| 56 | |||
| 57 | i = 0; | ||
| 58 | } | ||
| 59 | |||
| 38 | now = ktime_get(); | 60 | now = ktime_get(); |
| 39 | expires = ktime_add(now, ktime_set(0, dev->min_delta_ns)); | 61 | expires = ktime_add_ns(now, dev->min_delta_ns); |
| 40 | } | 62 | } |
| 41 | } | 63 | } |
| 42 | 64 | ||
| 43 | /** | 65 | /** |
| 66 | * tick_program_event | ||
| 67 | */ | ||
| 68 | int tick_program_event(ktime_t expires, int force) | ||
| 69 | { | ||
| 70 | struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev; | ||
| 71 | |||
| 72 | return tick_dev_program_event(dev, expires, force); | ||
| 73 | } | ||
| 74 | |||
| 75 | /** | ||
| 44 | * tick_resume_onshot - resume oneshot mode | 76 | * tick_resume_onshot - resume oneshot mode |
| 45 | */ | 77 | */ |
| 46 | void tick_resume_oneshot(void) | 78 | void tick_resume_oneshot(void) |
| @@ -61,7 +93,7 @@ void tick_setup_oneshot(struct clock_event_device *newdev, | |||
| 61 | { | 93 | { |
| 62 | newdev->event_handler = handler; | 94 | newdev->event_handler = handler; |
| 63 | clockevents_set_mode(newdev, CLOCK_EVT_MODE_ONESHOT); | 95 | clockevents_set_mode(newdev, CLOCK_EVT_MODE_ONESHOT); |
| 64 | clockevents_program_event(newdev, next_event, ktime_get()); | 96 | tick_dev_program_event(newdev, next_event, 1); |
| 65 | } | 97 | } |
| 66 | 98 | ||
| 67 | /** | 99 | /** |
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index b33be61c0f6b..a547be11cf97 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c | |||
| @@ -20,6 +20,7 @@ | |||
| 20 | #include <linux/profile.h> | 20 | #include <linux/profile.h> |
| 21 | #include <linux/sched.h> | 21 | #include <linux/sched.h> |
| 22 | #include <linux/tick.h> | 22 | #include <linux/tick.h> |
| 23 | #include <linux/module.h> | ||
| 23 | 24 | ||
| 24 | #include <asm/irq_regs.h> | 25 | #include <asm/irq_regs.h> |
| 25 | 26 | ||
| @@ -75,6 +76,9 @@ static void tick_do_update_jiffies64(ktime_t now) | |||
| 75 | incr * ticks); | 76 | incr * ticks); |
| 76 | } | 77 | } |
| 77 | do_timer(++ticks); | 78 | do_timer(++ticks); |
| 79 | |||
| 80 | /* Keep the tick_next_period variable up to date */ | ||
| 81 | tick_next_period = ktime_add(last_jiffies_update, tick_period); | ||
| 78 | } | 82 | } |
| 79 | write_sequnlock(&xtime_lock); | 83 | write_sequnlock(&xtime_lock); |
| 80 | } | 84 | } |
| @@ -187,9 +191,17 @@ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) | |||
| 187 | { | 191 | { |
| 188 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); | 192 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); |
| 189 | 193 | ||
| 190 | *last_update_time = ktime_to_us(ts->idle_lastupdate); | 194 | if (!tick_nohz_enabled) |
| 195 | return -1; | ||
| 196 | |||
| 197 | if (ts->idle_active) | ||
| 198 | *last_update_time = ktime_to_us(ts->idle_lastupdate); | ||
| 199 | else | ||
| 200 | *last_update_time = ktime_to_us(ktime_get()); | ||
| 201 | |||
| 191 | return ktime_to_us(ts->idle_sleeptime); | 202 | return ktime_to_us(ts->idle_sleeptime); |
| 192 | } | 203 | } |
| 204 | EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); | ||
| 193 | 205 | ||
| 194 | /** | 206 | /** |
| 195 | * tick_nohz_stop_sched_tick - stop the idle tick from the idle task | 207 | * tick_nohz_stop_sched_tick - stop the idle tick from the idle task |
| @@ -221,7 +233,7 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
| 221 | */ | 233 | */ |
| 222 | if (unlikely(!cpu_online(cpu))) { | 234 | if (unlikely(!cpu_online(cpu))) { |
| 223 | if (cpu == tick_do_timer_cpu) | 235 | if (cpu == tick_do_timer_cpu) |
| 224 | tick_do_timer_cpu = -1; | 236 | tick_do_timer_cpu = TICK_DO_TIMER_NONE; |
| 225 | } | 237 | } |
| 226 | 238 | ||
| 227 | if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) | 239 | if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) |
| @@ -258,7 +270,7 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
| 258 | next_jiffies = get_next_timer_interrupt(last_jiffies); | 270 | next_jiffies = get_next_timer_interrupt(last_jiffies); |
| 259 | delta_jiffies = next_jiffies - last_jiffies; | 271 | delta_jiffies = next_jiffies - last_jiffies; |
| 260 | 272 | ||
| 261 | if (rcu_needs_cpu(cpu)) | 273 | if (rcu_needs_cpu(cpu) || printk_needs_cpu(cpu)) |
| 262 | delta_jiffies = 1; | 274 | delta_jiffies = 1; |
| 263 | /* | 275 | /* |
| 264 | * Do not stop the tick, if we are only one off | 276 | * Do not stop the tick, if we are only one off |
| @@ -303,7 +315,7 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
| 303 | * invoked. | 315 | * invoked. |
| 304 | */ | 316 | */ |
| 305 | if (cpu == tick_do_timer_cpu) | 317 | if (cpu == tick_do_timer_cpu) |
| 306 | tick_do_timer_cpu = -1; | 318 | tick_do_timer_cpu = TICK_DO_TIMER_NONE; |
| 307 | 319 | ||
| 308 | ts->idle_sleeps++; | 320 | ts->idle_sleeps++; |
| 309 | 321 | ||
| @@ -468,7 +480,7 @@ static void tick_nohz_handler(struct clock_event_device *dev) | |||
| 468 | * this duty, then the jiffies update is still serialized by | 480 | * this duty, then the jiffies update is still serialized by |
| 469 | * xtime_lock. | 481 | * xtime_lock. |
| 470 | */ | 482 | */ |
| 471 | if (unlikely(tick_do_timer_cpu == -1)) | 483 | if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) |
| 472 | tick_do_timer_cpu = cpu; | 484 | tick_do_timer_cpu = cpu; |
| 473 | 485 | ||
| 474 | /* Check, if the jiffies need an update */ | 486 | /* Check, if the jiffies need an update */ |
| @@ -570,7 +582,7 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) | |||
| 570 | * this duty, then the jiffies update is still serialized by | 582 | * this duty, then the jiffies update is still serialized by |
| 571 | * xtime_lock. | 583 | * xtime_lock. |
| 572 | */ | 584 | */ |
| 573 | if (unlikely(tick_do_timer_cpu == -1)) | 585 | if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) |
| 574 | tick_do_timer_cpu = cpu; | 586 | tick_do_timer_cpu = cpu; |
| 575 | #endif | 587 | #endif |
| 576 | 588 | ||
| @@ -622,7 +634,7 @@ void tick_setup_sched_timer(void) | |||
| 622 | */ | 634 | */ |
| 623 | hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); | 635 | hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); |
| 624 | ts->sched_timer.function = tick_sched_timer; | 636 | ts->sched_timer.function = tick_sched_timer; |
| 625 | ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ; | 637 | ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU; |
| 626 | 638 | ||
| 627 | /* Get the next period (per cpu) */ | 639 | /* Get the next period (per cpu) */ |
| 628 | hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); | 640 | hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); |
diff --git a/kernel/timer.c b/kernel/timer.c index 03bc7f1f1593..510fe69351ca 100644 --- a/kernel/timer.c +++ b/kernel/timer.c | |||
| @@ -978,6 +978,7 @@ void update_process_times(int user_tick) | |||
| 978 | run_local_timers(); | 978 | run_local_timers(); |
| 979 | if (rcu_pending(cpu)) | 979 | if (rcu_pending(cpu)) |
| 980 | rcu_check_callbacks(cpu, user_tick); | 980 | rcu_check_callbacks(cpu, user_tick); |
| 981 | printk_tick(); | ||
| 981 | scheduler_tick(); | 982 | scheduler_tick(); |
| 982 | run_posix_cpu_timers(p); | 983 | run_posix_cpu_timers(p); |
| 983 | } | 984 | } |
diff --git a/kernel/trace/trace_sysprof.c b/kernel/trace/trace_sysprof.c index bb948e52ce20..db58fb66a135 100644 --- a/kernel/trace/trace_sysprof.c +++ b/kernel/trace/trace_sysprof.c | |||
| @@ -202,7 +202,7 @@ static void start_stack_timer(int cpu) | |||
| 202 | 202 | ||
| 203 | hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | 203 | hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| 204 | hrtimer->function = stack_trace_timer_fn; | 204 | hrtimer->function = stack_trace_timer_fn; |
| 205 | hrtimer->cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ; | 205 | hrtimer->cb_mode = HRTIMER_CB_IRQSAFE_PERCPU; |
| 206 | 206 | ||
| 207 | hrtimer_start(hrtimer, ns_to_ktime(sample_period), HRTIMER_MODE_REL); | 207 | hrtimer_start(hrtimer, ns_to_ktime(sample_period), HRTIMER_MODE_REL); |
| 208 | } | 208 | } |
diff --git a/kernel/user.c b/kernel/user.c index 865ecf57a096..39d6159fae43 100644 --- a/kernel/user.c +++ b/kernel/user.c | |||
| @@ -169,7 +169,7 @@ static ssize_t cpu_rt_runtime_show(struct kobject *kobj, | |||
| 169 | { | 169 | { |
| 170 | struct user_struct *up = container_of(kobj, struct user_struct, kobj); | 170 | struct user_struct *up = container_of(kobj, struct user_struct, kobj); |
| 171 | 171 | ||
| 172 | return sprintf(buf, "%lu\n", sched_group_rt_runtime(up->tg)); | 172 | return sprintf(buf, "%ld\n", sched_group_rt_runtime(up->tg)); |
| 173 | } | 173 | } |
| 174 | 174 | ||
| 175 | static ssize_t cpu_rt_runtime_store(struct kobject *kobj, | 175 | static ssize_t cpu_rt_runtime_store(struct kobject *kobj, |
| @@ -180,7 +180,7 @@ static ssize_t cpu_rt_runtime_store(struct kobject *kobj, | |||
| 180 | unsigned long rt_runtime; | 180 | unsigned long rt_runtime; |
| 181 | int rc; | 181 | int rc; |
| 182 | 182 | ||
| 183 | sscanf(buf, "%lu", &rt_runtime); | 183 | sscanf(buf, "%ld", &rt_runtime); |
| 184 | 184 | ||
| 185 | rc = sched_group_set_rt_runtime(up->tg, rt_runtime); | 185 | rc = sched_group_set_rt_runtime(up->tg, rt_runtime); |
| 186 | 186 | ||
diff --git a/kernel/utsname_sysctl.c b/kernel/utsname_sysctl.c index 4ab9659d269e..3b34b3545936 100644 --- a/kernel/utsname_sysctl.c +++ b/kernel/utsname_sysctl.c | |||
| @@ -60,7 +60,7 @@ static int proc_do_uts_string(ctl_table *table, int write, struct file *filp, | |||
| 60 | 60 | ||
| 61 | #ifdef CONFIG_SYSCTL_SYSCALL | 61 | #ifdef CONFIG_SYSCTL_SYSCALL |
| 62 | /* The generic string strategy routine: */ | 62 | /* The generic string strategy routine: */ |
| 63 | static int sysctl_uts_string(ctl_table *table, int __user *name, int nlen, | 63 | static int sysctl_uts_string(ctl_table *table, |
| 64 | void __user *oldval, size_t __user *oldlenp, | 64 | void __user *oldval, size_t __user *oldlenp, |
| 65 | void __user *newval, size_t newlen) | 65 | void __user *newval, size_t newlen) |
| 66 | { | 66 | { |
| @@ -69,8 +69,7 @@ static int sysctl_uts_string(ctl_table *table, int __user *name, int nlen, | |||
| 69 | write = newval && newlen; | 69 | write = newval && newlen; |
| 70 | memcpy(&uts_table, table, sizeof(uts_table)); | 70 | memcpy(&uts_table, table, sizeof(uts_table)); |
| 71 | uts_table.data = get_uts(table, write); | 71 | uts_table.data = get_uts(table, write); |
| 72 | r = sysctl_string(&uts_table, name, nlen, | 72 | r = sysctl_string(&uts_table, oldval, oldlenp, newval, newlen); |
| 73 | oldval, oldlenp, newval, newlen); | ||
| 74 | put_uts(table, write, uts_table.data); | 73 | put_uts(table, write, uts_table.data); |
| 75 | return r; | 74 | return r; |
| 76 | } | 75 | } |
diff --git a/kernel/wait.c b/kernel/wait.c index c275c56cf2d3..cd87131f2fc2 100644 --- a/kernel/wait.c +++ b/kernel/wait.c | |||
| @@ -72,12 +72,7 @@ prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state) | |||
| 72 | spin_lock_irqsave(&q->lock, flags); | 72 | spin_lock_irqsave(&q->lock, flags); |
| 73 | if (list_empty(&wait->task_list)) | 73 | if (list_empty(&wait->task_list)) |
| 74 | __add_wait_queue(q, wait); | 74 | __add_wait_queue(q, wait); |
| 75 | /* | 75 | set_current_state(state); |
| 76 | * don't alter the task state if this is just going to | ||
| 77 | * queue an async wait queue callback | ||
| 78 | */ | ||
| 79 | if (is_sync_wait(wait)) | ||
| 80 | set_current_state(state); | ||
| 81 | spin_unlock_irqrestore(&q->lock, flags); | 76 | spin_unlock_irqrestore(&q->lock, flags); |
| 82 | } | 77 | } |
| 83 | EXPORT_SYMBOL(prepare_to_wait); | 78 | EXPORT_SYMBOL(prepare_to_wait); |
| @@ -91,12 +86,7 @@ prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state) | |||
| 91 | spin_lock_irqsave(&q->lock, flags); | 86 | spin_lock_irqsave(&q->lock, flags); |
| 92 | if (list_empty(&wait->task_list)) | 87 | if (list_empty(&wait->task_list)) |
| 93 | __add_wait_queue_tail(q, wait); | 88 | __add_wait_queue_tail(q, wait); |
| 94 | /* | 89 | set_current_state(state); |
| 95 | * don't alter the task state if this is just going to | ||
| 96 | * queue an async wait queue callback | ||
| 97 | */ | ||
| 98 | if (is_sync_wait(wait)) | ||
| 99 | set_current_state(state); | ||
| 100 | spin_unlock_irqrestore(&q->lock, flags); | 90 | spin_unlock_irqrestore(&q->lock, flags); |
| 101 | } | 91 | } |
| 102 | EXPORT_SYMBOL(prepare_to_wait_exclusive); | 92 | EXPORT_SYMBOL(prepare_to_wait_exclusive); |
diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 4048e92aa04f..714afad46539 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c | |||
| @@ -9,7 +9,7 @@ | |||
| 9 | * Derived from the taskqueue/keventd code by: | 9 | * Derived from the taskqueue/keventd code by: |
| 10 | * | 10 | * |
| 11 | * David Woodhouse <dwmw2@infradead.org> | 11 | * David Woodhouse <dwmw2@infradead.org> |
| 12 | * Andrew Morton <andrewm@uow.edu.au> | 12 | * Andrew Morton |
| 13 | * Kai Petzke <wpp@marie.physik.tu-berlin.de> | 13 | * Kai Petzke <wpp@marie.physik.tu-berlin.de> |
| 14 | * Theodore Ts'o <tytso@mit.edu> | 14 | * Theodore Ts'o <tytso@mit.edu> |
| 15 | * | 15 | * |
