diff options
Diffstat (limited to 'kernel')
-rw-r--r-- | kernel/cgroup.c | 8 | ||||
-rw-r--r-- | kernel/exit.c | 4 | ||||
-rw-r--r-- | kernel/fork.c | 2 | ||||
-rw-r--r-- | kernel/irq/handle.c | 1 | ||||
-rw-r--r-- | kernel/irq/spurious.c | 2 | ||||
-rw-r--r-- | kernel/kthread.c | 23 | ||||
-rw-r--r-- | kernel/mutex-debug.c | 1 | ||||
-rw-r--r-- | kernel/params.c | 17 | ||||
-rw-r--r-- | kernel/perf_event.c | 68 | ||||
-rw-r--r-- | kernel/power/hibernate.c | 11 | ||||
-rw-r--r-- | kernel/power/suspend_test.c | 5 | ||||
-rw-r--r-- | kernel/power/swap.c | 43 | ||||
-rw-r--r-- | kernel/sched.c | 78 | ||||
-rw-r--r-- | kernel/sched_fair.c | 74 | ||||
-rw-r--r-- | kernel/sys.c | 25 | ||||
-rw-r--r-- | kernel/sysctl_check.c | 2 | ||||
-rw-r--r-- | kernel/time/timekeeping.c | 1 | ||||
-rw-r--r-- | kernel/trace/ftrace.c | 8 | ||||
-rw-r--r-- | kernel/trace/ring_buffer.c | 14 | ||||
-rw-r--r-- | kernel/trace/trace.c | 10 | ||||
-rw-r--r-- | kernel/trace/trace_events_filter.c | 3 | ||||
-rw-r--r-- | kernel/trace/trace_output.c | 5 | ||||
-rw-r--r-- | kernel/workqueue.c | 39 |
23 files changed, 279 insertions, 165 deletions
diff --git a/kernel/cgroup.c b/kernel/cgroup.c index ca83b73fba1..0249f4be9b5 100644 --- a/kernel/cgroup.c +++ b/kernel/cgroup.c | |||
@@ -1710,14 +1710,13 @@ static ssize_t cgroup_write_X64(struct cgroup *cgrp, struct cftype *cft, | |||
1710 | return -EFAULT; | 1710 | return -EFAULT; |
1711 | 1711 | ||
1712 | buffer[nbytes] = 0; /* nul-terminate */ | 1712 | buffer[nbytes] = 0; /* nul-terminate */ |
1713 | strstrip(buffer); | ||
1714 | if (cft->write_u64) { | 1713 | if (cft->write_u64) { |
1715 | u64 val = simple_strtoull(buffer, &end, 0); | 1714 | u64 val = simple_strtoull(strstrip(buffer), &end, 0); |
1716 | if (*end) | 1715 | if (*end) |
1717 | return -EINVAL; | 1716 | return -EINVAL; |
1718 | retval = cft->write_u64(cgrp, cft, val); | 1717 | retval = cft->write_u64(cgrp, cft, val); |
1719 | } else { | 1718 | } else { |
1720 | s64 val = simple_strtoll(buffer, &end, 0); | 1719 | s64 val = simple_strtoll(strstrip(buffer), &end, 0); |
1721 | if (*end) | 1720 | if (*end) |
1722 | return -EINVAL; | 1721 | return -EINVAL; |
1723 | retval = cft->write_s64(cgrp, cft, val); | 1722 | retval = cft->write_s64(cgrp, cft, val); |
@@ -1753,8 +1752,7 @@ static ssize_t cgroup_write_string(struct cgroup *cgrp, struct cftype *cft, | |||
1753 | } | 1752 | } |
1754 | 1753 | ||
1755 | buffer[nbytes] = 0; /* nul-terminate */ | 1754 | buffer[nbytes] = 0; /* nul-terminate */ |
1756 | strstrip(buffer); | 1755 | retval = cft->write_string(cgrp, cft, strstrip(buffer)); |
1757 | retval = cft->write_string(cgrp, cft, buffer); | ||
1758 | if (!retval) | 1756 | if (!retval) |
1759 | retval = nbytes; | 1757 | retval = nbytes; |
1760 | out: | 1758 | out: |
diff --git a/kernel/exit.c b/kernel/exit.c index e61891f8012..f7864ac2ecc 100644 --- a/kernel/exit.c +++ b/kernel/exit.c | |||
@@ -359,10 +359,8 @@ void __set_special_pids(struct pid *pid) | |||
359 | { | 359 | { |
360 | struct task_struct *curr = current->group_leader; | 360 | struct task_struct *curr = current->group_leader; |
361 | 361 | ||
362 | if (task_session(curr) != pid) { | 362 | if (task_session(curr) != pid) |
363 | change_pid(curr, PIDTYPE_SID, pid); | 363 | change_pid(curr, PIDTYPE_SID, pid); |
364 | proc_sid_connector(curr); | ||
365 | } | ||
366 | 364 | ||
367 | if (task_pgrp(curr) != pid) | 365 | if (task_pgrp(curr) != pid) |
368 | change_pid(curr, PIDTYPE_PGID, pid); | 366 | change_pid(curr, PIDTYPE_PGID, pid); |
diff --git a/kernel/fork.c b/kernel/fork.c index 4c20fff8c13..166b8c49257 100644 --- a/kernel/fork.c +++ b/kernel/fork.c | |||
@@ -91,7 +91,7 @@ int nr_processes(void) | |||
91 | int cpu; | 91 | int cpu; |
92 | int total = 0; | 92 | int total = 0; |
93 | 93 | ||
94 | for_each_online_cpu(cpu) | 94 | for_each_possible_cpu(cpu) |
95 | total += per_cpu(process_counts, cpu); | 95 | total += per_cpu(process_counts, cpu); |
96 | 96 | ||
97 | return total; | 97 | return total; |
diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c index a81cf80554d..17c71bb565c 100644 --- a/kernel/irq/handle.c +++ b/kernel/irq/handle.c | |||
@@ -11,6 +11,7 @@ | |||
11 | */ | 11 | */ |
12 | 12 | ||
13 | #include <linux/irq.h> | 13 | #include <linux/irq.h> |
14 | #include <linux/sched.h> | ||
14 | #include <linux/slab.h> | 15 | #include <linux/slab.h> |
15 | #include <linux/module.h> | 16 | #include <linux/module.h> |
16 | #include <linux/random.h> | 17 | #include <linux/random.h> |
diff --git a/kernel/irq/spurious.c b/kernel/irq/spurious.c index 114e704760f..bd7273e6282 100644 --- a/kernel/irq/spurious.c +++ b/kernel/irq/spurious.c | |||
@@ -121,7 +121,9 @@ static void poll_all_shared_irqs(void) | |||
121 | if (!(status & IRQ_SPURIOUS_DISABLED)) | 121 | if (!(status & IRQ_SPURIOUS_DISABLED)) |
122 | continue; | 122 | continue; |
123 | 123 | ||
124 | local_irq_disable(); | ||
124 | try_one_irq(i, desc); | 125 | try_one_irq(i, desc); |
126 | local_irq_enable(); | ||
125 | } | 127 | } |
126 | } | 128 | } |
127 | 129 | ||
diff --git a/kernel/kthread.c b/kernel/kthread.c index 5fe709982ca..ab7ae57773e 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c | |||
@@ -150,29 +150,6 @@ struct task_struct *kthread_create(int (*threadfn)(void *data), | |||
150 | EXPORT_SYMBOL(kthread_create); | 150 | EXPORT_SYMBOL(kthread_create); |
151 | 151 | ||
152 | /** | 152 | /** |
153 | * kthread_bind - bind a just-created kthread to a cpu. | ||
154 | * @k: thread created by kthread_create(). | ||
155 | * @cpu: cpu (might not be online, must be possible) for @k to run on. | ||
156 | * | ||
157 | * Description: This function is equivalent to set_cpus_allowed(), | ||
158 | * except that @cpu doesn't need to be online, and the thread must be | ||
159 | * stopped (i.e., just returned from kthread_create()). | ||
160 | */ | ||
161 | void kthread_bind(struct task_struct *k, unsigned int cpu) | ||
162 | { | ||
163 | /* Must have done schedule() in kthread() before we set_task_cpu */ | ||
164 | if (!wait_task_inactive(k, TASK_UNINTERRUPTIBLE)) { | ||
165 | WARN_ON(1); | ||
166 | return; | ||
167 | } | ||
168 | set_task_cpu(k, cpu); | ||
169 | k->cpus_allowed = cpumask_of_cpu(cpu); | ||
170 | k->rt.nr_cpus_allowed = 1; | ||
171 | k->flags |= PF_THREAD_BOUND; | ||
172 | } | ||
173 | EXPORT_SYMBOL(kthread_bind); | ||
174 | |||
175 | /** | ||
176 | * kthread_stop - stop a thread created by kthread_create(). | 153 | * kthread_stop - stop a thread created by kthread_create(). |
177 | * @k: thread created by kthread_create(). | 154 | * @k: thread created by kthread_create(). |
178 | * | 155 | * |
diff --git a/kernel/mutex-debug.c b/kernel/mutex-debug.c index 50d022e5a56..ec815a960b5 100644 --- a/kernel/mutex-debug.c +++ b/kernel/mutex-debug.c | |||
@@ -16,6 +16,7 @@ | |||
16 | #include <linux/delay.h> | 16 | #include <linux/delay.h> |
17 | #include <linux/module.h> | 17 | #include <linux/module.h> |
18 | #include <linux/poison.h> | 18 | #include <linux/poison.h> |
19 | #include <linux/sched.h> | ||
19 | #include <linux/spinlock.h> | 20 | #include <linux/spinlock.h> |
20 | #include <linux/kallsyms.h> | 21 | #include <linux/kallsyms.h> |
21 | #include <linux/interrupt.h> | 22 | #include <linux/interrupt.h> |
diff --git a/kernel/params.c b/kernel/params.c index 9da58eabdcb..d656c276508 100644 --- a/kernel/params.c +++ b/kernel/params.c | |||
@@ -218,15 +218,11 @@ int param_set_charp(const char *val, struct kernel_param *kp) | |||
218 | return -ENOSPC; | 218 | return -ENOSPC; |
219 | } | 219 | } |
220 | 220 | ||
221 | if (kp->flags & KPARAM_KMALLOCED) | ||
222 | kfree(*(char **)kp->arg); | ||
223 | |||
224 | /* This is a hack. We can't need to strdup in early boot, and we | 221 | /* This is a hack. We can't need to strdup in early boot, and we |
225 | * don't need to; this mangled commandline is preserved. */ | 222 | * don't need to; this mangled commandline is preserved. */ |
226 | if (slab_is_available()) { | 223 | if (slab_is_available()) { |
227 | kp->flags |= KPARAM_KMALLOCED; | ||
228 | *(char **)kp->arg = kstrdup(val, GFP_KERNEL); | 224 | *(char **)kp->arg = kstrdup(val, GFP_KERNEL); |
229 | if (!kp->arg) | 225 | if (!*(char **)kp->arg) |
230 | return -ENOMEM; | 226 | return -ENOMEM; |
231 | } else | 227 | } else |
232 | *(const char **)kp->arg = val; | 228 | *(const char **)kp->arg = val; |
@@ -304,6 +300,7 @@ static int param_array(const char *name, | |||
304 | unsigned int min, unsigned int max, | 300 | unsigned int min, unsigned int max, |
305 | void *elem, int elemsize, | 301 | void *elem, int elemsize, |
306 | int (*set)(const char *, struct kernel_param *kp), | 302 | int (*set)(const char *, struct kernel_param *kp), |
303 | u16 flags, | ||
307 | unsigned int *num) | 304 | unsigned int *num) |
308 | { | 305 | { |
309 | int ret; | 306 | int ret; |
@@ -313,6 +310,7 @@ static int param_array(const char *name, | |||
313 | /* Get the name right for errors. */ | 310 | /* Get the name right for errors. */ |
314 | kp.name = name; | 311 | kp.name = name; |
315 | kp.arg = elem; | 312 | kp.arg = elem; |
313 | kp.flags = flags; | ||
316 | 314 | ||
317 | /* No equals sign? */ | 315 | /* No equals sign? */ |
318 | if (!val) { | 316 | if (!val) { |
@@ -358,7 +356,8 @@ int param_array_set(const char *val, struct kernel_param *kp) | |||
358 | unsigned int temp_num; | 356 | unsigned int temp_num; |
359 | 357 | ||
360 | return param_array(kp->name, val, 1, arr->max, arr->elem, | 358 | return param_array(kp->name, val, 1, arr->max, arr->elem, |
361 | arr->elemsize, arr->set, arr->num ?: &temp_num); | 359 | arr->elemsize, arr->set, kp->flags, |
360 | arr->num ?: &temp_num); | ||
362 | } | 361 | } |
363 | 362 | ||
364 | int param_array_get(char *buffer, struct kernel_param *kp) | 363 | int param_array_get(char *buffer, struct kernel_param *kp) |
@@ -605,11 +604,7 @@ void module_param_sysfs_remove(struct module *mod) | |||
605 | 604 | ||
606 | void destroy_params(const struct kernel_param *params, unsigned num) | 605 | void destroy_params(const struct kernel_param *params, unsigned num) |
607 | { | 606 | { |
608 | unsigned int i; | 607 | /* FIXME: This should free kmalloced charp parameters. It doesn't. */ |
609 | |||
610 | for (i = 0; i < num; i++) | ||
611 | if (params[i].flags & KPARAM_KMALLOCED) | ||
612 | kfree(*(char **)params[i].arg); | ||
613 | } | 608 | } |
614 | 609 | ||
615 | static void __init kernel_add_sysfs_param(const char *name, | 610 | static void __init kernel_add_sysfs_param(const char *name, |
diff --git a/kernel/perf_event.c b/kernel/perf_event.c index 9d0b5c66588..7f29643c898 100644 --- a/kernel/perf_event.c +++ b/kernel/perf_event.c | |||
@@ -1355,7 +1355,7 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx) | |||
1355 | u64 interrupts, freq; | 1355 | u64 interrupts, freq; |
1356 | 1356 | ||
1357 | spin_lock(&ctx->lock); | 1357 | spin_lock(&ctx->lock); |
1358 | list_for_each_entry(event, &ctx->group_list, group_entry) { | 1358 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { |
1359 | if (event->state != PERF_EVENT_STATE_ACTIVE) | 1359 | if (event->state != PERF_EVENT_STATE_ACTIVE) |
1360 | continue; | 1360 | continue; |
1361 | 1361 | ||
@@ -3959,8 +3959,9 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) | |||
3959 | regs = task_pt_regs(current); | 3959 | regs = task_pt_regs(current); |
3960 | 3960 | ||
3961 | if (regs) { | 3961 | if (regs) { |
3962 | if (perf_event_overflow(event, 0, &data, regs)) | 3962 | if (!(event->attr.exclude_idle && current->pid == 0)) |
3963 | ret = HRTIMER_NORESTART; | 3963 | if (perf_event_overflow(event, 0, &data, regs)) |
3964 | ret = HRTIMER_NORESTART; | ||
3964 | } | 3965 | } |
3965 | 3966 | ||
3966 | period = max_t(u64, 10000, event->hw.sample_period); | 3967 | period = max_t(u64, 10000, event->hw.sample_period); |
@@ -3969,6 +3970,42 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) | |||
3969 | return ret; | 3970 | return ret; |
3970 | } | 3971 | } |
3971 | 3972 | ||
3973 | static void perf_swevent_start_hrtimer(struct perf_event *event) | ||
3974 | { | ||
3975 | struct hw_perf_event *hwc = &event->hw; | ||
3976 | |||
3977 | hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | ||
3978 | hwc->hrtimer.function = perf_swevent_hrtimer; | ||
3979 | if (hwc->sample_period) { | ||
3980 | u64 period; | ||
3981 | |||
3982 | if (hwc->remaining) { | ||
3983 | if (hwc->remaining < 0) | ||
3984 | period = 10000; | ||
3985 | else | ||
3986 | period = hwc->remaining; | ||
3987 | hwc->remaining = 0; | ||
3988 | } else { | ||
3989 | period = max_t(u64, 10000, hwc->sample_period); | ||
3990 | } | ||
3991 | __hrtimer_start_range_ns(&hwc->hrtimer, | ||
3992 | ns_to_ktime(period), 0, | ||
3993 | HRTIMER_MODE_REL, 0); | ||
3994 | } | ||
3995 | } | ||
3996 | |||
3997 | static void perf_swevent_cancel_hrtimer(struct perf_event *event) | ||
3998 | { | ||
3999 | struct hw_perf_event *hwc = &event->hw; | ||
4000 | |||
4001 | if (hwc->sample_period) { | ||
4002 | ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer); | ||
4003 | hwc->remaining = ktime_to_ns(remaining); | ||
4004 | |||
4005 | hrtimer_cancel(&hwc->hrtimer); | ||
4006 | } | ||
4007 | } | ||
4008 | |||
3972 | /* | 4009 | /* |
3973 | * Software event: cpu wall time clock | 4010 | * Software event: cpu wall time clock |
3974 | */ | 4011 | */ |
@@ -3991,22 +4028,14 @@ static int cpu_clock_perf_event_enable(struct perf_event *event) | |||
3991 | int cpu = raw_smp_processor_id(); | 4028 | int cpu = raw_smp_processor_id(); |
3992 | 4029 | ||
3993 | atomic64_set(&hwc->prev_count, cpu_clock(cpu)); | 4030 | atomic64_set(&hwc->prev_count, cpu_clock(cpu)); |
3994 | hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | 4031 | perf_swevent_start_hrtimer(event); |
3995 | hwc->hrtimer.function = perf_swevent_hrtimer; | ||
3996 | if (hwc->sample_period) { | ||
3997 | u64 period = max_t(u64, 10000, hwc->sample_period); | ||
3998 | __hrtimer_start_range_ns(&hwc->hrtimer, | ||
3999 | ns_to_ktime(period), 0, | ||
4000 | HRTIMER_MODE_REL, 0); | ||
4001 | } | ||
4002 | 4032 | ||
4003 | return 0; | 4033 | return 0; |
4004 | } | 4034 | } |
4005 | 4035 | ||
4006 | static void cpu_clock_perf_event_disable(struct perf_event *event) | 4036 | static void cpu_clock_perf_event_disable(struct perf_event *event) |
4007 | { | 4037 | { |
4008 | if (event->hw.sample_period) | 4038 | perf_swevent_cancel_hrtimer(event); |
4009 | hrtimer_cancel(&event->hw.hrtimer); | ||
4010 | cpu_clock_perf_event_update(event); | 4039 | cpu_clock_perf_event_update(event); |
4011 | } | 4040 | } |
4012 | 4041 | ||
@@ -4043,22 +4072,15 @@ static int task_clock_perf_event_enable(struct perf_event *event) | |||
4043 | now = event->ctx->time; | 4072 | now = event->ctx->time; |
4044 | 4073 | ||
4045 | atomic64_set(&hwc->prev_count, now); | 4074 | atomic64_set(&hwc->prev_count, now); |
4046 | hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | 4075 | |
4047 | hwc->hrtimer.function = perf_swevent_hrtimer; | 4076 | perf_swevent_start_hrtimer(event); |
4048 | if (hwc->sample_period) { | ||
4049 | u64 period = max_t(u64, 10000, hwc->sample_period); | ||
4050 | __hrtimer_start_range_ns(&hwc->hrtimer, | ||
4051 | ns_to_ktime(period), 0, | ||
4052 | HRTIMER_MODE_REL, 0); | ||
4053 | } | ||
4054 | 4077 | ||
4055 | return 0; | 4078 | return 0; |
4056 | } | 4079 | } |
4057 | 4080 | ||
4058 | static void task_clock_perf_event_disable(struct perf_event *event) | 4081 | static void task_clock_perf_event_disable(struct perf_event *event) |
4059 | { | 4082 | { |
4060 | if (event->hw.sample_period) | 4083 | perf_swevent_cancel_hrtimer(event); |
4061 | hrtimer_cancel(&event->hw.hrtimer); | ||
4062 | task_clock_perf_event_update(event, event->ctx->time); | 4084 | task_clock_perf_event_update(event, event->ctx->time); |
4063 | 4085 | ||
4064 | } | 4086 | } |
diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index 04b3a83d686..04a9e90d248 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c | |||
@@ -693,21 +693,22 @@ static int software_resume(void) | |||
693 | /* The snapshot device should not be opened while we're running */ | 693 | /* The snapshot device should not be opened while we're running */ |
694 | if (!atomic_add_unless(&snapshot_device_available, -1, 0)) { | 694 | if (!atomic_add_unless(&snapshot_device_available, -1, 0)) { |
695 | error = -EBUSY; | 695 | error = -EBUSY; |
696 | swsusp_close(FMODE_READ); | ||
696 | goto Unlock; | 697 | goto Unlock; |
697 | } | 698 | } |
698 | 699 | ||
699 | pm_prepare_console(); | 700 | pm_prepare_console(); |
700 | error = pm_notifier_call_chain(PM_RESTORE_PREPARE); | 701 | error = pm_notifier_call_chain(PM_RESTORE_PREPARE); |
701 | if (error) | 702 | if (error) |
702 | goto Finish; | 703 | goto close_finish; |
703 | 704 | ||
704 | error = usermodehelper_disable(); | 705 | error = usermodehelper_disable(); |
705 | if (error) | 706 | if (error) |
706 | goto Finish; | 707 | goto close_finish; |
707 | 708 | ||
708 | error = create_basic_memory_bitmaps(); | 709 | error = create_basic_memory_bitmaps(); |
709 | if (error) | 710 | if (error) |
710 | goto Finish; | 711 | goto close_finish; |
711 | 712 | ||
712 | pr_debug("PM: Preparing processes for restore.\n"); | 713 | pr_debug("PM: Preparing processes for restore.\n"); |
713 | error = prepare_processes(); | 714 | error = prepare_processes(); |
@@ -719,6 +720,7 @@ static int software_resume(void) | |||
719 | pr_debug("PM: Reading hibernation image.\n"); | 720 | pr_debug("PM: Reading hibernation image.\n"); |
720 | 721 | ||
721 | error = swsusp_read(&flags); | 722 | error = swsusp_read(&flags); |
723 | swsusp_close(FMODE_READ); | ||
722 | if (!error) | 724 | if (!error) |
723 | hibernation_restore(flags & SF_PLATFORM_MODE); | 725 | hibernation_restore(flags & SF_PLATFORM_MODE); |
724 | 726 | ||
@@ -737,6 +739,9 @@ static int software_resume(void) | |||
737 | mutex_unlock(&pm_mutex); | 739 | mutex_unlock(&pm_mutex); |
738 | pr_debug("PM: Resume from disk failed.\n"); | 740 | pr_debug("PM: Resume from disk failed.\n"); |
739 | return error; | 741 | return error; |
742 | close_finish: | ||
743 | swsusp_close(FMODE_READ); | ||
744 | goto Finish; | ||
740 | } | 745 | } |
741 | 746 | ||
742 | late_initcall(software_resume); | 747 | late_initcall(software_resume); |
diff --git a/kernel/power/suspend_test.c b/kernel/power/suspend_test.c index 17d8bb1acf9..25596e450ac 100644 --- a/kernel/power/suspend_test.c +++ b/kernel/power/suspend_test.c | |||
@@ -19,7 +19,7 @@ | |||
19 | * The time it takes is system-specific though, so when we test this | 19 | * The time it takes is system-specific though, so when we test this |
20 | * during system bootup we allow a LOT of time. | 20 | * during system bootup we allow a LOT of time. |
21 | */ | 21 | */ |
22 | #define TEST_SUSPEND_SECONDS 5 | 22 | #define TEST_SUSPEND_SECONDS 10 |
23 | 23 | ||
24 | static unsigned long suspend_test_start_time; | 24 | static unsigned long suspend_test_start_time; |
25 | 25 | ||
@@ -49,7 +49,8 @@ void suspend_test_finish(const char *label) | |||
49 | * has some performance issues. The stack dump of a WARN_ON | 49 | * has some performance issues. The stack dump of a WARN_ON |
50 | * is more likely to get the right attention than a printk... | 50 | * is more likely to get the right attention than a printk... |
51 | */ | 51 | */ |
52 | WARN(msec > (TEST_SUSPEND_SECONDS * 1000), "Component: %s\n", label); | 52 | WARN(msec > (TEST_SUSPEND_SECONDS * 1000), |
53 | "Component: %s, time: %u\n", label, msec); | ||
53 | } | 54 | } |
54 | 55 | ||
55 | /* | 56 | /* |
diff --git a/kernel/power/swap.c b/kernel/power/swap.c index b101cdc4df3..890f6b11b1d 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c | |||
@@ -314,7 +314,6 @@ static int save_image(struct swap_map_handle *handle, | |||
314 | { | 314 | { |
315 | unsigned int m; | 315 | unsigned int m; |
316 | int ret; | 316 | int ret; |
317 | int error = 0; | ||
318 | int nr_pages; | 317 | int nr_pages; |
319 | int err2; | 318 | int err2; |
320 | struct bio *bio; | 319 | struct bio *bio; |
@@ -329,26 +328,27 @@ static int save_image(struct swap_map_handle *handle, | |||
329 | nr_pages = 0; | 328 | nr_pages = 0; |
330 | bio = NULL; | 329 | bio = NULL; |
331 | do_gettimeofday(&start); | 330 | do_gettimeofday(&start); |
332 | do { | 331 | while (1) { |
333 | ret = snapshot_read_next(snapshot, PAGE_SIZE); | 332 | ret = snapshot_read_next(snapshot, PAGE_SIZE); |
334 | if (ret > 0) { | 333 | if (ret <= 0) |
335 | error = swap_write_page(handle, data_of(*snapshot), | 334 | break; |
336 | &bio); | 335 | ret = swap_write_page(handle, data_of(*snapshot), &bio); |
337 | if (error) | 336 | if (ret) |
338 | break; | 337 | break; |
339 | if (!(nr_pages % m)) | 338 | if (!(nr_pages % m)) |
340 | printk("\b\b\b\b%3d%%", nr_pages / m); | 339 | printk("\b\b\b\b%3d%%", nr_pages / m); |
341 | nr_pages++; | 340 | nr_pages++; |
342 | } | 341 | } |
343 | } while (ret > 0); | ||
344 | err2 = wait_on_bio_chain(&bio); | 342 | err2 = wait_on_bio_chain(&bio); |
345 | do_gettimeofday(&stop); | 343 | do_gettimeofday(&stop); |
346 | if (!error) | 344 | if (!ret) |
347 | error = err2; | 345 | ret = err2; |
348 | if (!error) | 346 | if (!ret) |
349 | printk("\b\b\b\bdone\n"); | 347 | printk("\b\b\b\bdone\n"); |
348 | else | ||
349 | printk("\n"); | ||
350 | swsusp_show_speed(&start, &stop, nr_to_write, "Wrote"); | 350 | swsusp_show_speed(&start, &stop, nr_to_write, "Wrote"); |
351 | return error; | 351 | return ret; |
352 | } | 352 | } |
353 | 353 | ||
354 | /** | 354 | /** |
@@ -536,7 +536,8 @@ static int load_image(struct swap_map_handle *handle, | |||
536 | snapshot_write_finalize(snapshot); | 536 | snapshot_write_finalize(snapshot); |
537 | if (!snapshot_image_loaded(snapshot)) | 537 | if (!snapshot_image_loaded(snapshot)) |
538 | error = -ENODATA; | 538 | error = -ENODATA; |
539 | } | 539 | } else |
540 | printk("\n"); | ||
540 | swsusp_show_speed(&start, &stop, nr_to_read, "Read"); | 541 | swsusp_show_speed(&start, &stop, nr_to_read, "Read"); |
541 | return error; | 542 | return error; |
542 | } | 543 | } |
@@ -572,8 +573,6 @@ int swsusp_read(unsigned int *flags_p) | |||
572 | error = load_image(&handle, &snapshot, header->pages - 1); | 573 | error = load_image(&handle, &snapshot, header->pages - 1); |
573 | release_swap_reader(&handle); | 574 | release_swap_reader(&handle); |
574 | 575 | ||
575 | blkdev_put(resume_bdev, FMODE_READ); | ||
576 | |||
577 | if (!error) | 576 | if (!error) |
578 | pr_debug("PM: Image successfully loaded\n"); | 577 | pr_debug("PM: Image successfully loaded\n"); |
579 | else | 578 | else |
@@ -596,7 +595,7 @@ int swsusp_check(void) | |||
596 | error = bio_read_page(swsusp_resume_block, | 595 | error = bio_read_page(swsusp_resume_block, |
597 | swsusp_header, NULL); | 596 | swsusp_header, NULL); |
598 | if (error) | 597 | if (error) |
599 | return error; | 598 | goto put; |
600 | 599 | ||
601 | if (!memcmp(SWSUSP_SIG, swsusp_header->sig, 10)) { | 600 | if (!memcmp(SWSUSP_SIG, swsusp_header->sig, 10)) { |
602 | memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10); | 601 | memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10); |
@@ -604,8 +603,10 @@ int swsusp_check(void) | |||
604 | error = bio_write_page(swsusp_resume_block, | 603 | error = bio_write_page(swsusp_resume_block, |
605 | swsusp_header, NULL); | 604 | swsusp_header, NULL); |
606 | } else { | 605 | } else { |
607 | return -EINVAL; | 606 | error = -EINVAL; |
608 | } | 607 | } |
608 | |||
609 | put: | ||
609 | if (error) | 610 | if (error) |
610 | blkdev_put(resume_bdev, FMODE_READ); | 611 | blkdev_put(resume_bdev, FMODE_READ); |
611 | else | 612 | else |
diff --git a/kernel/sched.c b/kernel/sched.c index 76c0e9691fc..3c11ae0a948 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
@@ -309,6 +309,8 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rt_rq, init_rt_rq); | |||
309 | */ | 309 | */ |
310 | static DEFINE_SPINLOCK(task_group_lock); | 310 | static DEFINE_SPINLOCK(task_group_lock); |
311 | 311 | ||
312 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
313 | |||
312 | #ifdef CONFIG_SMP | 314 | #ifdef CONFIG_SMP |
313 | static int root_task_group_empty(void) | 315 | static int root_task_group_empty(void) |
314 | { | 316 | { |
@@ -316,7 +318,6 @@ static int root_task_group_empty(void) | |||
316 | } | 318 | } |
317 | #endif | 319 | #endif |
318 | 320 | ||
319 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
320 | #ifdef CONFIG_USER_SCHED | 321 | #ifdef CONFIG_USER_SCHED |
321 | # define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD) | 322 | # define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD) |
322 | #else /* !CONFIG_USER_SCHED */ | 323 | #else /* !CONFIG_USER_SCHED */ |
@@ -676,6 +677,7 @@ inline void update_rq_clock(struct rq *rq) | |||
676 | 677 | ||
677 | /** | 678 | /** |
678 | * runqueue_is_locked | 679 | * runqueue_is_locked |
680 | * @cpu: the processor in question. | ||
679 | * | 681 | * |
680 | * Returns true if the current cpu runqueue is locked. | 682 | * Returns true if the current cpu runqueue is locked. |
681 | * This interface allows printk to be called with the runqueue lock | 683 | * This interface allows printk to be called with the runqueue lock |
@@ -1563,11 +1565,7 @@ static unsigned long cpu_avg_load_per_task(int cpu) | |||
1563 | 1565 | ||
1564 | #ifdef CONFIG_FAIR_GROUP_SCHED | 1566 | #ifdef CONFIG_FAIR_GROUP_SCHED |
1565 | 1567 | ||
1566 | struct update_shares_data { | 1568 | static __read_mostly unsigned long *update_shares_data; |
1567 | unsigned long rq_weight[NR_CPUS]; | ||
1568 | }; | ||
1569 | |||
1570 | static DEFINE_PER_CPU(struct update_shares_data, update_shares_data); | ||
1571 | 1569 | ||
1572 | static void __set_se_shares(struct sched_entity *se, unsigned long shares); | 1570 | static void __set_se_shares(struct sched_entity *se, unsigned long shares); |
1573 | 1571 | ||
@@ -1577,12 +1575,12 @@ static void __set_se_shares(struct sched_entity *se, unsigned long shares); | |||
1577 | static void update_group_shares_cpu(struct task_group *tg, int cpu, | 1575 | static void update_group_shares_cpu(struct task_group *tg, int cpu, |
1578 | unsigned long sd_shares, | 1576 | unsigned long sd_shares, |
1579 | unsigned long sd_rq_weight, | 1577 | unsigned long sd_rq_weight, |
1580 | struct update_shares_data *usd) | 1578 | unsigned long *usd_rq_weight) |
1581 | { | 1579 | { |
1582 | unsigned long shares, rq_weight; | 1580 | unsigned long shares, rq_weight; |
1583 | int boost = 0; | 1581 | int boost = 0; |
1584 | 1582 | ||
1585 | rq_weight = usd->rq_weight[cpu]; | 1583 | rq_weight = usd_rq_weight[cpu]; |
1586 | if (!rq_weight) { | 1584 | if (!rq_weight) { |
1587 | boost = 1; | 1585 | boost = 1; |
1588 | rq_weight = NICE_0_LOAD; | 1586 | rq_weight = NICE_0_LOAD; |
@@ -1617,7 +1615,7 @@ static void update_group_shares_cpu(struct task_group *tg, int cpu, | |||
1617 | static int tg_shares_up(struct task_group *tg, void *data) | 1615 | static int tg_shares_up(struct task_group *tg, void *data) |
1618 | { | 1616 | { |
1619 | unsigned long weight, rq_weight = 0, shares = 0; | 1617 | unsigned long weight, rq_weight = 0, shares = 0; |
1620 | struct update_shares_data *usd; | 1618 | unsigned long *usd_rq_weight; |
1621 | struct sched_domain *sd = data; | 1619 | struct sched_domain *sd = data; |
1622 | unsigned long flags; | 1620 | unsigned long flags; |
1623 | int i; | 1621 | int i; |
@@ -1626,11 +1624,11 @@ static int tg_shares_up(struct task_group *tg, void *data) | |||
1626 | return 0; | 1624 | return 0; |
1627 | 1625 | ||
1628 | local_irq_save(flags); | 1626 | local_irq_save(flags); |
1629 | usd = &__get_cpu_var(update_shares_data); | 1627 | usd_rq_weight = per_cpu_ptr(update_shares_data, smp_processor_id()); |
1630 | 1628 | ||
1631 | for_each_cpu(i, sched_domain_span(sd)) { | 1629 | for_each_cpu(i, sched_domain_span(sd)) { |
1632 | weight = tg->cfs_rq[i]->load.weight; | 1630 | weight = tg->cfs_rq[i]->load.weight; |
1633 | usd->rq_weight[i] = weight; | 1631 | usd_rq_weight[i] = weight; |
1634 | 1632 | ||
1635 | /* | 1633 | /* |
1636 | * If there are currently no tasks on the cpu pretend there | 1634 | * If there are currently no tasks on the cpu pretend there |
@@ -1651,7 +1649,7 @@ static int tg_shares_up(struct task_group *tg, void *data) | |||
1651 | shares = tg->shares; | 1649 | shares = tg->shares; |
1652 | 1650 | ||
1653 | for_each_cpu(i, sched_domain_span(sd)) | 1651 | for_each_cpu(i, sched_domain_span(sd)) |
1654 | update_group_shares_cpu(tg, i, shares, rq_weight, usd); | 1652 | update_group_shares_cpu(tg, i, shares, rq_weight, usd_rq_weight); |
1655 | 1653 | ||
1656 | local_irq_restore(flags); | 1654 | local_irq_restore(flags); |
1657 | 1655 | ||
@@ -1995,6 +1993,38 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p, | |||
1995 | p->sched_class->prio_changed(rq, p, oldprio, running); | 1993 | p->sched_class->prio_changed(rq, p, oldprio, running); |
1996 | } | 1994 | } |
1997 | 1995 | ||
1996 | /** | ||
1997 | * kthread_bind - bind a just-created kthread to a cpu. | ||
1998 | * @p: thread created by kthread_create(). | ||
1999 | * @cpu: cpu (might not be online, must be possible) for @k to run on. | ||
2000 | * | ||
2001 | * Description: This function is equivalent to set_cpus_allowed(), | ||
2002 | * except that @cpu doesn't need to be online, and the thread must be | ||
2003 | * stopped (i.e., just returned from kthread_create()). | ||
2004 | * | ||
2005 | * Function lives here instead of kthread.c because it messes with | ||
2006 | * scheduler internals which require locking. | ||
2007 | */ | ||
2008 | void kthread_bind(struct task_struct *p, unsigned int cpu) | ||
2009 | { | ||
2010 | struct rq *rq = cpu_rq(cpu); | ||
2011 | unsigned long flags; | ||
2012 | |||
2013 | /* Must have done schedule() in kthread() before we set_task_cpu */ | ||
2014 | if (!wait_task_inactive(p, TASK_UNINTERRUPTIBLE)) { | ||
2015 | WARN_ON(1); | ||
2016 | return; | ||
2017 | } | ||
2018 | |||
2019 | spin_lock_irqsave(&rq->lock, flags); | ||
2020 | set_task_cpu(p, cpu); | ||
2021 | p->cpus_allowed = cpumask_of_cpu(cpu); | ||
2022 | p->rt.nr_cpus_allowed = 1; | ||
2023 | p->flags |= PF_THREAD_BOUND; | ||
2024 | spin_unlock_irqrestore(&rq->lock, flags); | ||
2025 | } | ||
2026 | EXPORT_SYMBOL(kthread_bind); | ||
2027 | |||
1998 | #ifdef CONFIG_SMP | 2028 | #ifdef CONFIG_SMP |
1999 | /* | 2029 | /* |
2000 | * Is this task likely cache-hot: | 2030 | * Is this task likely cache-hot: |
@@ -2007,7 +2037,7 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) | |||
2007 | /* | 2037 | /* |
2008 | * Buddy candidates are cache hot: | 2038 | * Buddy candidates are cache hot: |
2009 | */ | 2039 | */ |
2010 | if (sched_feat(CACHE_HOT_BUDDY) && | 2040 | if (sched_feat(CACHE_HOT_BUDDY) && this_rq()->nr_running && |
2011 | (&p->se == cfs_rq_of(&p->se)->next || | 2041 | (&p->se == cfs_rq_of(&p->se)->next || |
2012 | &p->se == cfs_rq_of(&p->se)->last)) | 2042 | &p->se == cfs_rq_of(&p->se)->last)) |
2013 | return 1; | 2043 | return 1; |
@@ -2311,7 +2341,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, | |||
2311 | { | 2341 | { |
2312 | int cpu, orig_cpu, this_cpu, success = 0; | 2342 | int cpu, orig_cpu, this_cpu, success = 0; |
2313 | unsigned long flags; | 2343 | unsigned long flags; |
2314 | struct rq *rq; | 2344 | struct rq *rq, *orig_rq; |
2315 | 2345 | ||
2316 | if (!sched_feat(SYNC_WAKEUPS)) | 2346 | if (!sched_feat(SYNC_WAKEUPS)) |
2317 | wake_flags &= ~WF_SYNC; | 2347 | wake_flags &= ~WF_SYNC; |
@@ -2319,7 +2349,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, | |||
2319 | this_cpu = get_cpu(); | 2349 | this_cpu = get_cpu(); |
2320 | 2350 | ||
2321 | smp_wmb(); | 2351 | smp_wmb(); |
2322 | rq = task_rq_lock(p, &flags); | 2352 | rq = orig_rq = task_rq_lock(p, &flags); |
2323 | update_rq_clock(rq); | 2353 | update_rq_clock(rq); |
2324 | if (!(p->state & state)) | 2354 | if (!(p->state & state)) |
2325 | goto out; | 2355 | goto out; |
@@ -2350,6 +2380,10 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, | |||
2350 | set_task_cpu(p, cpu); | 2380 | set_task_cpu(p, cpu); |
2351 | 2381 | ||
2352 | rq = task_rq_lock(p, &flags); | 2382 | rq = task_rq_lock(p, &flags); |
2383 | |||
2384 | if (rq != orig_rq) | ||
2385 | update_rq_clock(rq); | ||
2386 | |||
2353 | WARN_ON(p->state != TASK_WAKING); | 2387 | WARN_ON(p->state != TASK_WAKING); |
2354 | cpu = task_cpu(p); | 2388 | cpu = task_cpu(p); |
2355 | 2389 | ||
@@ -3656,6 +3690,7 @@ static void update_group_power(struct sched_domain *sd, int cpu) | |||
3656 | 3690 | ||
3657 | /** | 3691 | /** |
3658 | * update_sg_lb_stats - Update sched_group's statistics for load balancing. | 3692 | * update_sg_lb_stats - Update sched_group's statistics for load balancing. |
3693 | * @sd: The sched_domain whose statistics are to be updated. | ||
3659 | * @group: sched_group whose statistics are to be updated. | 3694 | * @group: sched_group whose statistics are to be updated. |
3660 | * @this_cpu: Cpu for which load balance is currently performed. | 3695 | * @this_cpu: Cpu for which load balance is currently performed. |
3661 | * @idle: Idle status of this_cpu | 3696 | * @idle: Idle status of this_cpu |
@@ -6718,9 +6753,6 @@ EXPORT_SYMBOL(yield); | |||
6718 | /* | 6753 | /* |
6719 | * This task is about to go to sleep on IO. Increment rq->nr_iowait so | 6754 | * This task is about to go to sleep on IO. Increment rq->nr_iowait so |
6720 | * that process accounting knows that this is a task in IO wait state. | 6755 | * that process accounting knows that this is a task in IO wait state. |
6721 | * | ||
6722 | * But don't do that if it is a deliberate, throttling IO wait (this task | ||
6723 | * has set its backing_dev_info: the queue against which it should throttle) | ||
6724 | */ | 6756 | */ |
6725 | void __sched io_schedule(void) | 6757 | void __sched io_schedule(void) |
6726 | { | 6758 | { |
@@ -9404,6 +9436,10 @@ void __init sched_init(void) | |||
9404 | #endif /* CONFIG_USER_SCHED */ | 9436 | #endif /* CONFIG_USER_SCHED */ |
9405 | #endif /* CONFIG_GROUP_SCHED */ | 9437 | #endif /* CONFIG_GROUP_SCHED */ |
9406 | 9438 | ||
9439 | #if defined CONFIG_FAIR_GROUP_SCHED && defined CONFIG_SMP | ||
9440 | update_shares_data = __alloc_percpu(nr_cpu_ids * sizeof(unsigned long), | ||
9441 | __alignof__(unsigned long)); | ||
9442 | #endif | ||
9407 | for_each_possible_cpu(i) { | 9443 | for_each_possible_cpu(i) { |
9408 | struct rq *rq; | 9444 | struct rq *rq; |
9409 | 9445 | ||
@@ -9529,13 +9565,13 @@ void __init sched_init(void) | |||
9529 | current->sched_class = &fair_sched_class; | 9565 | current->sched_class = &fair_sched_class; |
9530 | 9566 | ||
9531 | /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */ | 9567 | /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */ |
9532 | alloc_cpumask_var(&nohz_cpu_mask, GFP_NOWAIT); | 9568 | zalloc_cpumask_var(&nohz_cpu_mask, GFP_NOWAIT); |
9533 | #ifdef CONFIG_SMP | 9569 | #ifdef CONFIG_SMP |
9534 | #ifdef CONFIG_NO_HZ | 9570 | #ifdef CONFIG_NO_HZ |
9535 | alloc_cpumask_var(&nohz.cpu_mask, GFP_NOWAIT); | 9571 | zalloc_cpumask_var(&nohz.cpu_mask, GFP_NOWAIT); |
9536 | alloc_cpumask_var(&nohz.ilb_grp_nohz_mask, GFP_NOWAIT); | 9572 | alloc_cpumask_var(&nohz.ilb_grp_nohz_mask, GFP_NOWAIT); |
9537 | #endif | 9573 | #endif |
9538 | alloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT); | 9574 | zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT); |
9539 | #endif /* SMP */ | 9575 | #endif /* SMP */ |
9540 | 9576 | ||
9541 | perf_event_init(); | 9577 | perf_event_init(); |
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 4e777b47eed..37087a7fac2 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c | |||
@@ -822,6 +822,26 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) | |||
822 | * re-elected due to buddy favours. | 822 | * re-elected due to buddy favours. |
823 | */ | 823 | */ |
824 | clear_buddies(cfs_rq, curr); | 824 | clear_buddies(cfs_rq, curr); |
825 | return; | ||
826 | } | ||
827 | |||
828 | /* | ||
829 | * Ensure that a task that missed wakeup preemption by a | ||
830 | * narrow margin doesn't have to wait for a full slice. | ||
831 | * This also mitigates buddy induced latencies under load. | ||
832 | */ | ||
833 | if (!sched_feat(WAKEUP_PREEMPT)) | ||
834 | return; | ||
835 | |||
836 | if (delta_exec < sysctl_sched_min_granularity) | ||
837 | return; | ||
838 | |||
839 | if (cfs_rq->nr_running > 1) { | ||
840 | struct sched_entity *se = __pick_next_entity(cfs_rq); | ||
841 | s64 delta = curr->vruntime - se->vruntime; | ||
842 | |||
843 | if (delta > ideal_runtime) | ||
844 | resched_task(rq_of(cfs_rq)->curr); | ||
825 | } | 845 | } |
826 | } | 846 | } |
827 | 847 | ||
@@ -861,12 +881,18 @@ wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se); | |||
861 | static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) | 881 | static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) |
862 | { | 882 | { |
863 | struct sched_entity *se = __pick_next_entity(cfs_rq); | 883 | struct sched_entity *se = __pick_next_entity(cfs_rq); |
884 | struct sched_entity *left = se; | ||
864 | 885 | ||
865 | if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, se) < 1) | 886 | if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) |
866 | return cfs_rq->next; | 887 | se = cfs_rq->next; |
867 | 888 | ||
868 | if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, se) < 1) | 889 | /* |
869 | return cfs_rq->last; | 890 | * Prefer last buddy, try to return the CPU to a preempted task. |
891 | */ | ||
892 | if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) | ||
893 | se = cfs_rq->last; | ||
894 | |||
895 | clear_buddies(cfs_rq, se); | ||
870 | 896 | ||
871 | return se; | 897 | return se; |
872 | } | 898 | } |
@@ -1568,6 +1594,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ | |||
1568 | struct sched_entity *se = &curr->se, *pse = &p->se; | 1594 | struct sched_entity *se = &curr->se, *pse = &p->se; |
1569 | struct cfs_rq *cfs_rq = task_cfs_rq(curr); | 1595 | struct cfs_rq *cfs_rq = task_cfs_rq(curr); |
1570 | int sync = wake_flags & WF_SYNC; | 1596 | int sync = wake_flags & WF_SYNC; |
1597 | int scale = cfs_rq->nr_running >= sched_nr_latency; | ||
1571 | 1598 | ||
1572 | update_curr(cfs_rq); | 1599 | update_curr(cfs_rq); |
1573 | 1600 | ||
@@ -1582,18 +1609,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ | |||
1582 | if (unlikely(se == pse)) | 1609 | if (unlikely(se == pse)) |
1583 | return; | 1610 | return; |
1584 | 1611 | ||
1585 | /* | 1612 | if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) |
1586 | * Only set the backward buddy when the current task is still on the | ||
1587 | * rq. This can happen when a wakeup gets interleaved with schedule on | ||
1588 | * the ->pre_schedule() or idle_balance() point, either of which can | ||
1589 | * drop the rq lock. | ||
1590 | * | ||
1591 | * Also, during early boot the idle thread is in the fair class, for | ||
1592 | * obvious reasons its a bad idea to schedule back to the idle thread. | ||
1593 | */ | ||
1594 | if (sched_feat(LAST_BUDDY) && likely(se->on_rq && curr != rq->idle)) | ||
1595 | set_last_buddy(se); | ||
1596 | if (sched_feat(NEXT_BUDDY) && !(wake_flags & WF_FORK)) | ||
1597 | set_next_buddy(pse); | 1613 | set_next_buddy(pse); |
1598 | 1614 | ||
1599 | /* | 1615 | /* |
@@ -1639,8 +1655,22 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ | |||
1639 | 1655 | ||
1640 | BUG_ON(!pse); | 1656 | BUG_ON(!pse); |
1641 | 1657 | ||
1642 | if (wakeup_preempt_entity(se, pse) == 1) | 1658 | if (wakeup_preempt_entity(se, pse) == 1) { |
1643 | resched_task(curr); | 1659 | resched_task(curr); |
1660 | /* | ||
1661 | * Only set the backward buddy when the current task is still | ||
1662 | * on the rq. This can happen when a wakeup gets interleaved | ||
1663 | * with schedule on the ->pre_schedule() or idle_balance() | ||
1664 | * point, either of which can * drop the rq lock. | ||
1665 | * | ||
1666 | * Also, during early boot the idle thread is in the fair class, | ||
1667 | * for obvious reasons its a bad idea to schedule back to it. | ||
1668 | */ | ||
1669 | if (unlikely(!se->on_rq || curr == rq->idle)) | ||
1670 | return; | ||
1671 | if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se)) | ||
1672 | set_last_buddy(se); | ||
1673 | } | ||
1644 | } | 1674 | } |
1645 | 1675 | ||
1646 | static struct task_struct *pick_next_task_fair(struct rq *rq) | 1676 | static struct task_struct *pick_next_task_fair(struct rq *rq) |
@@ -1654,16 +1684,6 @@ static struct task_struct *pick_next_task_fair(struct rq *rq) | |||
1654 | 1684 | ||
1655 | do { | 1685 | do { |
1656 | se = pick_next_entity(cfs_rq); | 1686 | se = pick_next_entity(cfs_rq); |
1657 | /* | ||
1658 | * If se was a buddy, clear it so that it will have to earn | ||
1659 | * the favour again. | ||
1660 | * | ||
1661 | * If se was not a buddy, clear the buddies because neither | ||
1662 | * was elegible to run, let them earn it again. | ||
1663 | * | ||
1664 | * IOW. unconditionally clear buddies. | ||
1665 | */ | ||
1666 | __clear_buddies(cfs_rq, NULL); | ||
1667 | set_next_entity(cfs_rq, se); | 1687 | set_next_entity(cfs_rq, se); |
1668 | cfs_rq = group_cfs_rq(se); | 1688 | cfs_rq = group_cfs_rq(se); |
1669 | } while (cfs_rq); | 1689 | } while (cfs_rq); |
diff --git a/kernel/sys.c b/kernel/sys.c index 255475d163e..ce17760d9c5 100644 --- a/kernel/sys.c +++ b/kernel/sys.c | |||
@@ -1110,6 +1110,8 @@ SYSCALL_DEFINE0(setsid) | |||
1110 | err = session; | 1110 | err = session; |
1111 | out: | 1111 | out: |
1112 | write_unlock_irq(&tasklist_lock); | 1112 | write_unlock_irq(&tasklist_lock); |
1113 | if (err > 0) | ||
1114 | proc_sid_connector(group_leader); | ||
1113 | return err; | 1115 | return err; |
1114 | } | 1116 | } |
1115 | 1117 | ||
@@ -1546,24 +1548,37 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, | |||
1546 | if (arg4 | arg5) | 1548 | if (arg4 | arg5) |
1547 | return -EINVAL; | 1549 | return -EINVAL; |
1548 | switch (arg2) { | 1550 | switch (arg2) { |
1549 | case 0: | 1551 | case PR_MCE_KILL_CLEAR: |
1550 | if (arg3 != 0) | 1552 | if (arg3 != 0) |
1551 | return -EINVAL; | 1553 | return -EINVAL; |
1552 | current->flags &= ~PF_MCE_PROCESS; | 1554 | current->flags &= ~PF_MCE_PROCESS; |
1553 | break; | 1555 | break; |
1554 | case 1: | 1556 | case PR_MCE_KILL_SET: |
1555 | current->flags |= PF_MCE_PROCESS; | 1557 | current->flags |= PF_MCE_PROCESS; |
1556 | if (arg3 != 0) | 1558 | if (arg3 == PR_MCE_KILL_EARLY) |
1557 | current->flags |= PF_MCE_EARLY; | 1559 | current->flags |= PF_MCE_EARLY; |
1558 | else | 1560 | else if (arg3 == PR_MCE_KILL_LATE) |
1559 | current->flags &= ~PF_MCE_EARLY; | 1561 | current->flags &= ~PF_MCE_EARLY; |
1562 | else if (arg3 == PR_MCE_KILL_DEFAULT) | ||
1563 | current->flags &= | ||
1564 | ~(PF_MCE_EARLY|PF_MCE_PROCESS); | ||
1565 | else | ||
1566 | return -EINVAL; | ||
1560 | break; | 1567 | break; |
1561 | default: | 1568 | default: |
1562 | return -EINVAL; | 1569 | return -EINVAL; |
1563 | } | 1570 | } |
1564 | error = 0; | 1571 | error = 0; |
1565 | break; | 1572 | break; |
1566 | 1573 | case PR_MCE_KILL_GET: | |
1574 | if (arg2 | arg3 | arg4 | arg5) | ||
1575 | return -EINVAL; | ||
1576 | if (current->flags & PF_MCE_PROCESS) | ||
1577 | error = (current->flags & PF_MCE_EARLY) ? | ||
1578 | PR_MCE_KILL_EARLY : PR_MCE_KILL_LATE; | ||
1579 | else | ||
1580 | error = PR_MCE_KILL_DEFAULT; | ||
1581 | break; | ||
1567 | default: | 1582 | default: |
1568 | error = -EINVAL; | 1583 | error = -EINVAL; |
1569 | break; | 1584 | break; |
diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c index b38423ca711..b6e7aaea460 100644 --- a/kernel/sysctl_check.c +++ b/kernel/sysctl_check.c | |||
@@ -1521,7 +1521,7 @@ int sysctl_check_table(struct nsproxy *namespaces, struct ctl_table *table) | |||
1521 | if (!table->ctl_name && table->strategy) | 1521 | if (!table->ctl_name && table->strategy) |
1522 | set_fail(&fail, table, "Strategy without ctl_name"); | 1522 | set_fail(&fail, table, "Strategy without ctl_name"); |
1523 | #endif | 1523 | #endif |
1524 | #ifdef CONFIG_PROC_FS | 1524 | #ifdef CONFIG_PROC_SYSCTL |
1525 | if (table->procname && !table->proc_handler) | 1525 | if (table->procname && !table->proc_handler) |
1526 | set_fail(&fail, table, "No proc_handler"); | 1526 | set_fail(&fail, table, "No proc_handler"); |
1527 | #endif | 1527 | #endif |
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index fb0f46fa1ec..c3a4e2907ea 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c | |||
@@ -13,6 +13,7 @@ | |||
13 | #include <linux/percpu.h> | 13 | #include <linux/percpu.h> |
14 | #include <linux/init.h> | 14 | #include <linux/init.h> |
15 | #include <linux/mm.h> | 15 | #include <linux/mm.h> |
16 | #include <linux/sched.h> | ||
16 | #include <linux/sysdev.h> | 17 | #include <linux/sysdev.h> |
17 | #include <linux/clocksource.h> | 18 | #include <linux/clocksource.h> |
18 | #include <linux/jiffies.h> | 19 | #include <linux/jiffies.h> |
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 37ba67e3326..6dc4e5ef7a0 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
@@ -740,7 +740,7 @@ ftrace_profile_write(struct file *filp, const char __user *ubuf, | |||
740 | out: | 740 | out: |
741 | mutex_unlock(&ftrace_profile_lock); | 741 | mutex_unlock(&ftrace_profile_lock); |
742 | 742 | ||
743 | filp->f_pos += cnt; | 743 | *ppos += cnt; |
744 | 744 | ||
745 | return cnt; | 745 | return cnt; |
746 | } | 746 | } |
@@ -2222,15 +2222,15 @@ ftrace_regex_write(struct file *file, const char __user *ubuf, | |||
2222 | ret = ftrace_process_regex(parser->buffer, | 2222 | ret = ftrace_process_regex(parser->buffer, |
2223 | parser->idx, enable); | 2223 | parser->idx, enable); |
2224 | if (ret) | 2224 | if (ret) |
2225 | goto out; | 2225 | goto out_unlock; |
2226 | 2226 | ||
2227 | trace_parser_clear(parser); | 2227 | trace_parser_clear(parser); |
2228 | } | 2228 | } |
2229 | 2229 | ||
2230 | ret = read; | 2230 | ret = read; |
2231 | 2231 | out_unlock: | |
2232 | mutex_unlock(&ftrace_regex_lock); | 2232 | mutex_unlock(&ftrace_regex_lock); |
2233 | out: | 2233 | |
2234 | return ret; | 2234 | return ret; |
2235 | } | 2235 | } |
2236 | 2236 | ||
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index d4ff0197054..5dd017fea6f 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c | |||
@@ -483,7 +483,7 @@ struct ring_buffer_iter { | |||
483 | /* Up this if you want to test the TIME_EXTENTS and normalization */ | 483 | /* Up this if you want to test the TIME_EXTENTS and normalization */ |
484 | #define DEBUG_SHIFT 0 | 484 | #define DEBUG_SHIFT 0 |
485 | 485 | ||
486 | static inline u64 rb_time_stamp(struct ring_buffer *buffer, int cpu) | 486 | static inline u64 rb_time_stamp(struct ring_buffer *buffer) |
487 | { | 487 | { |
488 | /* shift to debug/test normalization and TIME_EXTENTS */ | 488 | /* shift to debug/test normalization and TIME_EXTENTS */ |
489 | return buffer->clock() << DEBUG_SHIFT; | 489 | return buffer->clock() << DEBUG_SHIFT; |
@@ -494,7 +494,7 @@ u64 ring_buffer_time_stamp(struct ring_buffer *buffer, int cpu) | |||
494 | u64 time; | 494 | u64 time; |
495 | 495 | ||
496 | preempt_disable_notrace(); | 496 | preempt_disable_notrace(); |
497 | time = rb_time_stamp(buffer, cpu); | 497 | time = rb_time_stamp(buffer); |
498 | preempt_enable_no_resched_notrace(); | 498 | preempt_enable_no_resched_notrace(); |
499 | 499 | ||
500 | return time; | 500 | return time; |
@@ -599,7 +599,7 @@ static struct list_head *rb_list_head(struct list_head *list) | |||
599 | } | 599 | } |
600 | 600 | ||
601 | /* | 601 | /* |
602 | * rb_is_head_page - test if the give page is the head page | 602 | * rb_is_head_page - test if the given page is the head page |
603 | * | 603 | * |
604 | * Because the reader may move the head_page pointer, we can | 604 | * Because the reader may move the head_page pointer, we can |
605 | * not trust what the head page is (it may be pointing to | 605 | * not trust what the head page is (it may be pointing to |
@@ -1193,6 +1193,7 @@ rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages) | |||
1193 | atomic_inc(&cpu_buffer->record_disabled); | 1193 | atomic_inc(&cpu_buffer->record_disabled); |
1194 | synchronize_sched(); | 1194 | synchronize_sched(); |
1195 | 1195 | ||
1196 | spin_lock_irq(&cpu_buffer->reader_lock); | ||
1196 | rb_head_page_deactivate(cpu_buffer); | 1197 | rb_head_page_deactivate(cpu_buffer); |
1197 | 1198 | ||
1198 | for (i = 0; i < nr_pages; i++) { | 1199 | for (i = 0; i < nr_pages; i++) { |
@@ -1207,6 +1208,7 @@ rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages) | |||
1207 | return; | 1208 | return; |
1208 | 1209 | ||
1209 | rb_reset_cpu(cpu_buffer); | 1210 | rb_reset_cpu(cpu_buffer); |
1211 | spin_unlock_irq(&cpu_buffer->reader_lock); | ||
1210 | 1212 | ||
1211 | rb_check_pages(cpu_buffer); | 1213 | rb_check_pages(cpu_buffer); |
1212 | 1214 | ||
@@ -1868,7 +1870,7 @@ rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer, | |||
1868 | * Nested commits always have zero deltas, so | 1870 | * Nested commits always have zero deltas, so |
1869 | * just reread the time stamp | 1871 | * just reread the time stamp |
1870 | */ | 1872 | */ |
1871 | *ts = rb_time_stamp(buffer, cpu_buffer->cpu); | 1873 | *ts = rb_time_stamp(buffer); |
1872 | next_page->page->time_stamp = *ts; | 1874 | next_page->page->time_stamp = *ts; |
1873 | } | 1875 | } |
1874 | 1876 | ||
@@ -2111,7 +2113,7 @@ rb_reserve_next_event(struct ring_buffer *buffer, | |||
2111 | if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000)) | 2113 | if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000)) |
2112 | goto out_fail; | 2114 | goto out_fail; |
2113 | 2115 | ||
2114 | ts = rb_time_stamp(cpu_buffer->buffer, cpu_buffer->cpu); | 2116 | ts = rb_time_stamp(cpu_buffer->buffer); |
2115 | 2117 | ||
2116 | /* | 2118 | /* |
2117 | * Only the first commit can update the timestamp. | 2119 | * Only the first commit can update the timestamp. |
@@ -2681,7 +2683,7 @@ unsigned long ring_buffer_entries(struct ring_buffer *buffer) | |||
2681 | EXPORT_SYMBOL_GPL(ring_buffer_entries); | 2683 | EXPORT_SYMBOL_GPL(ring_buffer_entries); |
2682 | 2684 | ||
2683 | /** | 2685 | /** |
2684 | * ring_buffer_overrun_cpu - get the number of overruns in buffer | 2686 | * ring_buffer_overruns - get the number of overruns in buffer |
2685 | * @buffer: The ring buffer | 2687 | * @buffer: The ring buffer |
2686 | * | 2688 | * |
2687 | * Returns the total number of overruns in the ring buffer | 2689 | * Returns the total number of overruns in the ring buffer |
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 45068269ebb..b20d3ec75de 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
@@ -1393,7 +1393,7 @@ int trace_array_vprintk(struct trace_array *tr, | |||
1393 | 1393 | ||
1394 | int trace_vprintk(unsigned long ip, const char *fmt, va_list args) | 1394 | int trace_vprintk(unsigned long ip, const char *fmt, va_list args) |
1395 | { | 1395 | { |
1396 | return trace_array_printk(&global_trace, ip, fmt, args); | 1396 | return trace_array_vprintk(&global_trace, ip, fmt, args); |
1397 | } | 1397 | } |
1398 | EXPORT_SYMBOL_GPL(trace_vprintk); | 1398 | EXPORT_SYMBOL_GPL(trace_vprintk); |
1399 | 1399 | ||
@@ -2440,7 +2440,7 @@ tracing_trace_options_write(struct file *filp, const char __user *ubuf, | |||
2440 | return ret; | 2440 | return ret; |
2441 | } | 2441 | } |
2442 | 2442 | ||
2443 | filp->f_pos += cnt; | 2443 | *ppos += cnt; |
2444 | 2444 | ||
2445 | return cnt; | 2445 | return cnt; |
2446 | } | 2446 | } |
@@ -2582,7 +2582,7 @@ tracing_ctrl_write(struct file *filp, const char __user *ubuf, | |||
2582 | } | 2582 | } |
2583 | mutex_unlock(&trace_types_lock); | 2583 | mutex_unlock(&trace_types_lock); |
2584 | 2584 | ||
2585 | filp->f_pos += cnt; | 2585 | *ppos += cnt; |
2586 | 2586 | ||
2587 | return cnt; | 2587 | return cnt; |
2588 | } | 2588 | } |
@@ -2764,7 +2764,7 @@ tracing_set_trace_write(struct file *filp, const char __user *ubuf, | |||
2764 | if (err) | 2764 | if (err) |
2765 | return err; | 2765 | return err; |
2766 | 2766 | ||
2767 | filp->f_pos += ret; | 2767 | *ppos += ret; |
2768 | 2768 | ||
2769 | return ret; | 2769 | return ret; |
2770 | } | 2770 | } |
@@ -3299,7 +3299,7 @@ tracing_entries_write(struct file *filp, const char __user *ubuf, | |||
3299 | } | 3299 | } |
3300 | } | 3300 | } |
3301 | 3301 | ||
3302 | filp->f_pos += cnt; | 3302 | *ppos += cnt; |
3303 | 3303 | ||
3304 | /* If check pages failed, return ENOMEM */ | 3304 | /* If check pages failed, return ENOMEM */ |
3305 | if (tracing_disabled) | 3305 | if (tracing_disabled) |
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 23245785927..98a6cc5c64e 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c | |||
@@ -933,8 +933,9 @@ static void postfix_clear(struct filter_parse_state *ps) | |||
933 | 933 | ||
934 | while (!list_empty(&ps->postfix)) { | 934 | while (!list_empty(&ps->postfix)) { |
935 | elt = list_first_entry(&ps->postfix, struct postfix_elt, list); | 935 | elt = list_first_entry(&ps->postfix, struct postfix_elt, list); |
936 | kfree(elt->operand); | ||
937 | list_del(&elt->list); | 936 | list_del(&elt->list); |
937 | kfree(elt->operand); | ||
938 | kfree(elt); | ||
938 | } | 939 | } |
939 | } | 940 | } |
940 | 941 | ||
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c index ed17565826b..b6c12c6a1bc 100644 --- a/kernel/trace/trace_output.c +++ b/kernel/trace/trace_output.c | |||
@@ -69,6 +69,9 @@ enum print_line_t trace_print_printk_msg_only(struct trace_iterator *iter) | |||
69 | * @s: trace sequence descriptor | 69 | * @s: trace sequence descriptor |
70 | * @fmt: printf format string | 70 | * @fmt: printf format string |
71 | * | 71 | * |
72 | * It returns 0 if the trace oversizes the buffer's free | ||
73 | * space, 1 otherwise. | ||
74 | * | ||
72 | * The tracer may use either sequence operations or its own | 75 | * The tracer may use either sequence operations or its own |
73 | * copy to user routines. To simplify formating of a trace | 76 | * copy to user routines. To simplify formating of a trace |
74 | * trace_seq_printf is used to store strings into a special | 77 | * trace_seq_printf is used to store strings into a special |
@@ -95,7 +98,7 @@ trace_seq_printf(struct trace_seq *s, const char *fmt, ...) | |||
95 | 98 | ||
96 | s->len += ret; | 99 | s->len += ret; |
97 | 100 | ||
98 | return len; | 101 | return 1; |
99 | } | 102 | } |
100 | EXPORT_SYMBOL_GPL(trace_seq_printf); | 103 | EXPORT_SYMBOL_GPL(trace_seq_printf); |
101 | 104 | ||
diff --git a/kernel/workqueue.c b/kernel/workqueue.c index addfe2df93b..12328147132 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c | |||
@@ -640,6 +640,24 @@ int schedule_delayed_work(struct delayed_work *dwork, | |||
640 | EXPORT_SYMBOL(schedule_delayed_work); | 640 | EXPORT_SYMBOL(schedule_delayed_work); |
641 | 641 | ||
642 | /** | 642 | /** |
643 | * flush_delayed_work - block until a dwork_struct's callback has terminated | ||
644 | * @dwork: the delayed work which is to be flushed | ||
645 | * | ||
646 | * Any timeout is cancelled, and any pending work is run immediately. | ||
647 | */ | ||
648 | void flush_delayed_work(struct delayed_work *dwork) | ||
649 | { | ||
650 | if (del_timer_sync(&dwork->timer)) { | ||
651 | struct cpu_workqueue_struct *cwq; | ||
652 | cwq = wq_per_cpu(keventd_wq, get_cpu()); | ||
653 | __queue_work(cwq, &dwork->work); | ||
654 | put_cpu(); | ||
655 | } | ||
656 | flush_work(&dwork->work); | ||
657 | } | ||
658 | EXPORT_SYMBOL(flush_delayed_work); | ||
659 | |||
660 | /** | ||
643 | * schedule_delayed_work_on - queue work in global workqueue on CPU after delay | 661 | * schedule_delayed_work_on - queue work in global workqueue on CPU after delay |
644 | * @cpu: cpu to use | 662 | * @cpu: cpu to use |
645 | * @dwork: job to be done | 663 | * @dwork: job to be done |
@@ -667,21 +685,38 @@ EXPORT_SYMBOL(schedule_delayed_work_on); | |||
667 | int schedule_on_each_cpu(work_func_t func) | 685 | int schedule_on_each_cpu(work_func_t func) |
668 | { | 686 | { |
669 | int cpu; | 687 | int cpu; |
688 | int orig = -1; | ||
670 | struct work_struct *works; | 689 | struct work_struct *works; |
671 | 690 | ||
672 | works = alloc_percpu(struct work_struct); | 691 | works = alloc_percpu(struct work_struct); |
673 | if (!works) | 692 | if (!works) |
674 | return -ENOMEM; | 693 | return -ENOMEM; |
675 | 694 | ||
695 | /* | ||
696 | * when running in keventd don't schedule a work item on itself. | ||
697 | * Can just call directly because the work queue is already bound. | ||
698 | * This also is faster. | ||
699 | * Make this a generic parameter for other workqueues? | ||
700 | */ | ||
701 | if (current_is_keventd()) { | ||
702 | orig = raw_smp_processor_id(); | ||
703 | INIT_WORK(per_cpu_ptr(works, orig), func); | ||
704 | func(per_cpu_ptr(works, orig)); | ||
705 | } | ||
706 | |||
676 | get_online_cpus(); | 707 | get_online_cpus(); |
677 | for_each_online_cpu(cpu) { | 708 | for_each_online_cpu(cpu) { |
678 | struct work_struct *work = per_cpu_ptr(works, cpu); | 709 | struct work_struct *work = per_cpu_ptr(works, cpu); |
679 | 710 | ||
711 | if (cpu == orig) | ||
712 | continue; | ||
680 | INIT_WORK(work, func); | 713 | INIT_WORK(work, func); |
681 | schedule_work_on(cpu, work); | 714 | schedule_work_on(cpu, work); |
682 | } | 715 | } |
683 | for_each_online_cpu(cpu) | 716 | for_each_online_cpu(cpu) { |
684 | flush_work(per_cpu_ptr(works, cpu)); | 717 | if (cpu != orig) |
718 | flush_work(per_cpu_ptr(works, cpu)); | ||
719 | } | ||
685 | put_online_cpus(); | 720 | put_online_cpus(); |
686 | free_percpu(works); | 721 | free_percpu(works); |
687 | return 0; | 722 | return 0; |