diff options
Diffstat (limited to 'kernel')
37 files changed, 685 insertions, 210 deletions
diff --git a/kernel/async.c b/kernel/async.c index 608b32b42812..f565891f2c9b 100644 --- a/kernel/async.c +++ b/kernel/async.c | |||
| @@ -54,6 +54,7 @@ asynchronous and synchronous parts of the kernel. | |||
| 54 | #include <linux/sched.h> | 54 | #include <linux/sched.h> |
| 55 | #include <linux/init.h> | 55 | #include <linux/init.h> |
| 56 | #include <linux/kthread.h> | 56 | #include <linux/kthread.h> |
| 57 | #include <linux/delay.h> | ||
| 57 | #include <asm/atomic.h> | 58 | #include <asm/atomic.h> |
| 58 | 59 | ||
| 59 | static async_cookie_t next_cookie = 1; | 60 | static async_cookie_t next_cookie = 1; |
| @@ -132,21 +133,23 @@ static void run_one_entry(void) | |||
| 132 | entry = list_first_entry(&async_pending, struct async_entry, list); | 133 | entry = list_first_entry(&async_pending, struct async_entry, list); |
| 133 | 134 | ||
| 134 | /* 2) move it to the running queue */ | 135 | /* 2) move it to the running queue */ |
| 135 | list_del(&entry->list); | 136 | list_move_tail(&entry->list, entry->running); |
| 136 | list_add_tail(&entry->list, &async_running); | ||
| 137 | spin_unlock_irqrestore(&async_lock, flags); | 137 | spin_unlock_irqrestore(&async_lock, flags); |
| 138 | 138 | ||
| 139 | /* 3) run it (and print duration)*/ | 139 | /* 3) run it (and print duration)*/ |
| 140 | if (initcall_debug && system_state == SYSTEM_BOOTING) { | 140 | if (initcall_debug && system_state == SYSTEM_BOOTING) { |
| 141 | printk("calling %lli_%pF @ %i\n", entry->cookie, entry->func, task_pid_nr(current)); | 141 | printk("calling %lli_%pF @ %i\n", (long long)entry->cookie, |
| 142 | entry->func, task_pid_nr(current)); | ||
| 142 | calltime = ktime_get(); | 143 | calltime = ktime_get(); |
| 143 | } | 144 | } |
| 144 | entry->func(entry->data, entry->cookie); | 145 | entry->func(entry->data, entry->cookie); |
| 145 | if (initcall_debug && system_state == SYSTEM_BOOTING) { | 146 | if (initcall_debug && system_state == SYSTEM_BOOTING) { |
| 146 | rettime = ktime_get(); | 147 | rettime = ktime_get(); |
| 147 | delta = ktime_sub(rettime, calltime); | 148 | delta = ktime_sub(rettime, calltime); |
| 148 | printk("initcall %lli_%pF returned 0 after %lld usecs\n", entry->cookie, | 149 | printk("initcall %lli_%pF returned 0 after %lld usecs\n", |
| 149 | entry->func, ktime_to_ns(delta) >> 10); | 150 | (long long)entry->cookie, |
| 151 | entry->func, | ||
| 152 | (long long)ktime_to_ns(delta) >> 10); | ||
| 150 | } | 153 | } |
| 151 | 154 | ||
| 152 | /* 4) remove it from the running queue */ | 155 | /* 4) remove it from the running queue */ |
| @@ -205,18 +208,44 @@ static async_cookie_t __async_schedule(async_func_ptr *ptr, void *data, struct l | |||
| 205 | return newcookie; | 208 | return newcookie; |
| 206 | } | 209 | } |
| 207 | 210 | ||
| 211 | /** | ||
| 212 | * async_schedule - schedule a function for asynchronous execution | ||
| 213 | * @ptr: function to execute asynchronously | ||
| 214 | * @data: data pointer to pass to the function | ||
| 215 | * | ||
| 216 | * Returns an async_cookie_t that may be used for checkpointing later. | ||
| 217 | * Note: This function may be called from atomic or non-atomic contexts. | ||
| 218 | */ | ||
| 208 | async_cookie_t async_schedule(async_func_ptr *ptr, void *data) | 219 | async_cookie_t async_schedule(async_func_ptr *ptr, void *data) |
| 209 | { | 220 | { |
| 210 | return __async_schedule(ptr, data, &async_pending); | 221 | return __async_schedule(ptr, data, &async_running); |
| 211 | } | 222 | } |
| 212 | EXPORT_SYMBOL_GPL(async_schedule); | 223 | EXPORT_SYMBOL_GPL(async_schedule); |
| 213 | 224 | ||
| 214 | async_cookie_t async_schedule_special(async_func_ptr *ptr, void *data, struct list_head *running) | 225 | /** |
| 226 | * async_schedule_domain - schedule a function for asynchronous execution within a certain domain | ||
| 227 | * @ptr: function to execute asynchronously | ||
| 228 | * @data: data pointer to pass to the function | ||
| 229 | * @running: running list for the domain | ||
| 230 | * | ||
| 231 | * Returns an async_cookie_t that may be used for checkpointing later. | ||
| 232 | * @running may be used in the async_synchronize_*_domain() functions | ||
| 233 | * to wait within a certain synchronization domain rather than globally. | ||
| 234 | * A synchronization domain is specified via the running queue @running to use. | ||
| 235 | * Note: This function may be called from atomic or non-atomic contexts. | ||
| 236 | */ | ||
| 237 | async_cookie_t async_schedule_domain(async_func_ptr *ptr, void *data, | ||
| 238 | struct list_head *running) | ||
| 215 | { | 239 | { |
| 216 | return __async_schedule(ptr, data, running); | 240 | return __async_schedule(ptr, data, running); |
| 217 | } | 241 | } |
| 218 | EXPORT_SYMBOL_GPL(async_schedule_special); | 242 | EXPORT_SYMBOL_GPL(async_schedule_domain); |
| 219 | 243 | ||
| 244 | /** | ||
| 245 | * async_synchronize_full - synchronize all asynchronous function calls | ||
| 246 | * | ||
| 247 | * This function waits until all asynchronous function calls have been done. | ||
| 248 | */ | ||
| 220 | void async_synchronize_full(void) | 249 | void async_synchronize_full(void) |
| 221 | { | 250 | { |
| 222 | do { | 251 | do { |
| @@ -225,13 +254,30 @@ void async_synchronize_full(void) | |||
| 225 | } | 254 | } |
| 226 | EXPORT_SYMBOL_GPL(async_synchronize_full); | 255 | EXPORT_SYMBOL_GPL(async_synchronize_full); |
| 227 | 256 | ||
| 228 | void async_synchronize_full_special(struct list_head *list) | 257 | /** |
| 258 | * async_synchronize_full_domain - synchronize all asynchronous function within a certain domain | ||
| 259 | * @list: running list to synchronize on | ||
| 260 | * | ||
| 261 | * This function waits until all asynchronous function calls for the | ||
| 262 | * synchronization domain specified by the running list @list have been done. | ||
| 263 | */ | ||
| 264 | void async_synchronize_full_domain(struct list_head *list) | ||
| 229 | { | 265 | { |
| 230 | async_synchronize_cookie_special(next_cookie, list); | 266 | async_synchronize_cookie_domain(next_cookie, list); |
| 231 | } | 267 | } |
| 232 | EXPORT_SYMBOL_GPL(async_synchronize_full_special); | 268 | EXPORT_SYMBOL_GPL(async_synchronize_full_domain); |
| 233 | 269 | ||
| 234 | void async_synchronize_cookie_special(async_cookie_t cookie, struct list_head *running) | 270 | /** |
| 271 | * async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing | ||
| 272 | * @cookie: async_cookie_t to use as checkpoint | ||
| 273 | * @running: running list to synchronize on | ||
| 274 | * | ||
| 275 | * This function waits until all asynchronous function calls for the | ||
| 276 | * synchronization domain specified by the running list @list submitted | ||
| 277 | * prior to @cookie have been done. | ||
| 278 | */ | ||
| 279 | void async_synchronize_cookie_domain(async_cookie_t cookie, | ||
| 280 | struct list_head *running) | ||
| 235 | { | 281 | { |
| 236 | ktime_t starttime, delta, endtime; | 282 | ktime_t starttime, delta, endtime; |
| 237 | 283 | ||
| @@ -247,14 +293,22 @@ void async_synchronize_cookie_special(async_cookie_t cookie, struct list_head *r | |||
| 247 | delta = ktime_sub(endtime, starttime); | 293 | delta = ktime_sub(endtime, starttime); |
| 248 | 294 | ||
| 249 | printk("async_continuing @ %i after %lli usec\n", | 295 | printk("async_continuing @ %i after %lli usec\n", |
| 250 | task_pid_nr(current), ktime_to_ns(delta) >> 10); | 296 | task_pid_nr(current), |
| 297 | (long long)ktime_to_ns(delta) >> 10); | ||
| 251 | } | 298 | } |
| 252 | } | 299 | } |
| 253 | EXPORT_SYMBOL_GPL(async_synchronize_cookie_special); | 300 | EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain); |
| 254 | 301 | ||
| 302 | /** | ||
| 303 | * async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing | ||
| 304 | * @cookie: async_cookie_t to use as checkpoint | ||
| 305 | * | ||
| 306 | * This function waits until all asynchronous function calls prior to @cookie | ||
| 307 | * have been done. | ||
| 308 | */ | ||
| 255 | void async_synchronize_cookie(async_cookie_t cookie) | 309 | void async_synchronize_cookie(async_cookie_t cookie) |
| 256 | { | 310 | { |
| 257 | async_synchronize_cookie_special(cookie, &async_running); | 311 | async_synchronize_cookie_domain(cookie, &async_running); |
| 258 | } | 312 | } |
| 259 | EXPORT_SYMBOL_GPL(async_synchronize_cookie); | 313 | EXPORT_SYMBOL_GPL(async_synchronize_cookie); |
| 260 | 314 | ||
| @@ -315,7 +369,11 @@ static int async_manager_thread(void *unused) | |||
| 315 | ec = atomic_read(&entry_count); | 369 | ec = atomic_read(&entry_count); |
| 316 | 370 | ||
| 317 | while (tc < ec && tc < MAX_THREADS) { | 371 | while (tc < ec && tc < MAX_THREADS) { |
| 318 | kthread_run(async_thread, NULL, "async/%i", tc); | 372 | if (IS_ERR(kthread_run(async_thread, NULL, "async/%i", |
| 373 | tc))) { | ||
| 374 | msleep(100); | ||
| 375 | continue; | ||
| 376 | } | ||
| 319 | atomic_inc(&thread_count); | 377 | atomic_inc(&thread_count); |
| 320 | tc++; | 378 | tc++; |
| 321 | } | 379 | } |
| @@ -330,7 +388,9 @@ static int async_manager_thread(void *unused) | |||
| 330 | static int __init async_init(void) | 388 | static int __init async_init(void) |
| 331 | { | 389 | { |
| 332 | if (async_enabled) | 390 | if (async_enabled) |
| 333 | kthread_run(async_manager_thread, NULL, "async/mgr"); | 391 | if (IS_ERR(kthread_run(async_manager_thread, NULL, |
| 392 | "async/mgr"))) | ||
| 393 | async_enabled = 0; | ||
| 334 | return 0; | 394 | return 0; |
| 335 | } | 395 | } |
| 336 | 396 | ||
diff --git a/kernel/cgroup.c b/kernel/cgroup.c index c29831076e7a..e14db9c089b9 100644 --- a/kernel/cgroup.c +++ b/kernel/cgroup.c | |||
| @@ -1115,8 +1115,10 @@ static void cgroup_kill_sb(struct super_block *sb) { | |||
| 1115 | } | 1115 | } |
| 1116 | write_unlock(&css_set_lock); | 1116 | write_unlock(&css_set_lock); |
| 1117 | 1117 | ||
| 1118 | list_del(&root->root_list); | 1118 | if (!list_empty(&root->root_list)) { |
| 1119 | root_count--; | 1119 | list_del(&root->root_list); |
| 1120 | root_count--; | ||
| 1121 | } | ||
| 1120 | 1122 | ||
| 1121 | mutex_unlock(&cgroup_mutex); | 1123 | mutex_unlock(&cgroup_mutex); |
| 1122 | 1124 | ||
| @@ -2349,7 +2351,7 @@ static void cgroup_lock_hierarchy(struct cgroupfs_root *root) | |||
| 2349 | for (i = 0; i < CGROUP_SUBSYS_COUNT; i++) { | 2351 | for (i = 0; i < CGROUP_SUBSYS_COUNT; i++) { |
| 2350 | struct cgroup_subsys *ss = subsys[i]; | 2352 | struct cgroup_subsys *ss = subsys[i]; |
| 2351 | if (ss->root == root) | 2353 | if (ss->root == root) |
| 2352 | mutex_lock_nested(&ss->hierarchy_mutex, i); | 2354 | mutex_lock(&ss->hierarchy_mutex); |
| 2353 | } | 2355 | } |
| 2354 | } | 2356 | } |
| 2355 | 2357 | ||
| @@ -2434,7 +2436,9 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry, | |||
| 2434 | 2436 | ||
| 2435 | err_remove: | 2437 | err_remove: |
| 2436 | 2438 | ||
| 2439 | cgroup_lock_hierarchy(root); | ||
| 2437 | list_del(&cgrp->sibling); | 2440 | list_del(&cgrp->sibling); |
| 2441 | cgroup_unlock_hierarchy(root); | ||
| 2438 | root->number_of_cgroups--; | 2442 | root->number_of_cgroups--; |
| 2439 | 2443 | ||
| 2440 | err_destroy: | 2444 | err_destroy: |
| @@ -2507,7 +2511,7 @@ static int cgroup_clear_css_refs(struct cgroup *cgrp) | |||
| 2507 | for_each_subsys(cgrp->root, ss) { | 2511 | for_each_subsys(cgrp->root, ss) { |
| 2508 | struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id]; | 2512 | struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id]; |
| 2509 | int refcnt; | 2513 | int refcnt; |
| 2510 | do { | 2514 | while (1) { |
| 2511 | /* We can only remove a CSS with a refcnt==1 */ | 2515 | /* We can only remove a CSS with a refcnt==1 */ |
| 2512 | refcnt = atomic_read(&css->refcnt); | 2516 | refcnt = atomic_read(&css->refcnt); |
| 2513 | if (refcnt > 1) { | 2517 | if (refcnt > 1) { |
| @@ -2521,7 +2525,10 @@ static int cgroup_clear_css_refs(struct cgroup *cgrp) | |||
| 2521 | * css_tryget() to spin until we set the | 2525 | * css_tryget() to spin until we set the |
| 2522 | * CSS_REMOVED bits or abort | 2526 | * CSS_REMOVED bits or abort |
| 2523 | */ | 2527 | */ |
| 2524 | } while (atomic_cmpxchg(&css->refcnt, refcnt, 0) != refcnt); | 2528 | if (atomic_cmpxchg(&css->refcnt, refcnt, 0) == refcnt) |
| 2529 | break; | ||
| 2530 | cpu_relax(); | ||
| 2531 | } | ||
| 2525 | } | 2532 | } |
| 2526 | done: | 2533 | done: |
| 2527 | for_each_subsys(cgrp->root, ss) { | 2534 | for_each_subsys(cgrp->root, ss) { |
| @@ -2630,6 +2637,7 @@ static void __init cgroup_init_subsys(struct cgroup_subsys *ss) | |||
| 2630 | BUG_ON(!list_empty(&init_task.tasks)); | 2637 | BUG_ON(!list_empty(&init_task.tasks)); |
| 2631 | 2638 | ||
| 2632 | mutex_init(&ss->hierarchy_mutex); | 2639 | mutex_init(&ss->hierarchy_mutex); |
| 2640 | lockdep_set_class(&ss->hierarchy_mutex, &ss->subsys_key); | ||
| 2633 | ss->active = 1; | 2641 | ss->active = 1; |
| 2634 | } | 2642 | } |
| 2635 | 2643 | ||
| @@ -2991,20 +2999,21 @@ int cgroup_clone(struct task_struct *tsk, struct cgroup_subsys *subsys, | |||
| 2991 | mutex_unlock(&cgroup_mutex); | 2999 | mutex_unlock(&cgroup_mutex); |
| 2992 | return 0; | 3000 | return 0; |
| 2993 | } | 3001 | } |
| 2994 | task_lock(tsk); | ||
| 2995 | cg = tsk->cgroups; | ||
| 2996 | parent = task_cgroup(tsk, subsys->subsys_id); | ||
| 2997 | 3002 | ||
| 2998 | /* Pin the hierarchy */ | 3003 | /* Pin the hierarchy */ |
| 2999 | if (!atomic_inc_not_zero(&parent->root->sb->s_active)) { | 3004 | if (!atomic_inc_not_zero(&root->sb->s_active)) { |
| 3000 | /* We race with the final deactivate_super() */ | 3005 | /* We race with the final deactivate_super() */ |
| 3001 | mutex_unlock(&cgroup_mutex); | 3006 | mutex_unlock(&cgroup_mutex); |
| 3002 | return 0; | 3007 | return 0; |
| 3003 | } | 3008 | } |
| 3004 | 3009 | ||
| 3005 | /* Keep the cgroup alive */ | 3010 | /* Keep the cgroup alive */ |
| 3011 | task_lock(tsk); | ||
| 3012 | parent = task_cgroup(tsk, subsys->subsys_id); | ||
| 3013 | cg = tsk->cgroups; | ||
| 3006 | get_css_set(cg); | 3014 | get_css_set(cg); |
| 3007 | task_unlock(tsk); | 3015 | task_unlock(tsk); |
| 3016 | |||
| 3008 | mutex_unlock(&cgroup_mutex); | 3017 | mutex_unlock(&cgroup_mutex); |
| 3009 | 3018 | ||
| 3010 | /* Now do the VFS work to create a cgroup */ | 3019 | /* Now do the VFS work to create a cgroup */ |
| @@ -3043,7 +3052,7 @@ int cgroup_clone(struct task_struct *tsk, struct cgroup_subsys *subsys, | |||
| 3043 | mutex_unlock(&inode->i_mutex); | 3052 | mutex_unlock(&inode->i_mutex); |
| 3044 | put_css_set(cg); | 3053 | put_css_set(cg); |
| 3045 | 3054 | ||
| 3046 | deactivate_super(parent->root->sb); | 3055 | deactivate_super(root->sb); |
| 3047 | /* The cgroup is still accessible in the VFS, but | 3056 | /* The cgroup is still accessible in the VFS, but |
| 3048 | * we're not going to try to rmdir() it at this | 3057 | * we're not going to try to rmdir() it at this |
| 3049 | * point. */ | 3058 | * point. */ |
| @@ -3069,7 +3078,7 @@ int cgroup_clone(struct task_struct *tsk, struct cgroup_subsys *subsys, | |||
| 3069 | mutex_lock(&cgroup_mutex); | 3078 | mutex_lock(&cgroup_mutex); |
| 3070 | put_css_set(cg); | 3079 | put_css_set(cg); |
| 3071 | mutex_unlock(&cgroup_mutex); | 3080 | mutex_unlock(&cgroup_mutex); |
| 3072 | deactivate_super(parent->root->sb); | 3081 | deactivate_super(root->sb); |
| 3073 | return ret; | 3082 | return ret; |
| 3074 | } | 3083 | } |
| 3075 | 3084 | ||
diff --git a/kernel/cpuset.c b/kernel/cpuset.c index a85678865c5e..f76db9dcaa05 100644 --- a/kernel/cpuset.c +++ b/kernel/cpuset.c | |||
| @@ -61,6 +61,14 @@ | |||
| 61 | #include <linux/cgroup.h> | 61 | #include <linux/cgroup.h> |
| 62 | 62 | ||
| 63 | /* | 63 | /* |
| 64 | * Workqueue for cpuset related tasks. | ||
| 65 | * | ||
| 66 | * Using kevent workqueue may cause deadlock when memory_migrate | ||
| 67 | * is set. So we create a separate workqueue thread for cpuset. | ||
| 68 | */ | ||
| 69 | static struct workqueue_struct *cpuset_wq; | ||
| 70 | |||
| 71 | /* | ||
| 64 | * Tracks how many cpusets are currently defined in system. | 72 | * Tracks how many cpusets are currently defined in system. |
| 65 | * When there is only one cpuset (the root cpuset) we can | 73 | * When there is only one cpuset (the root cpuset) we can |
| 66 | * short circuit some hooks. | 74 | * short circuit some hooks. |
| @@ -831,7 +839,7 @@ static DECLARE_WORK(rebuild_sched_domains_work, do_rebuild_sched_domains); | |||
| 831 | */ | 839 | */ |
| 832 | static void async_rebuild_sched_domains(void) | 840 | static void async_rebuild_sched_domains(void) |
| 833 | { | 841 | { |
| 834 | schedule_work(&rebuild_sched_domains_work); | 842 | queue_work(cpuset_wq, &rebuild_sched_domains_work); |
| 835 | } | 843 | } |
| 836 | 844 | ||
| 837 | /* | 845 | /* |
| @@ -2111,6 +2119,9 @@ void __init cpuset_init_smp(void) | |||
| 2111 | 2119 | ||
| 2112 | hotcpu_notifier(cpuset_track_online_cpus, 0); | 2120 | hotcpu_notifier(cpuset_track_online_cpus, 0); |
| 2113 | hotplug_memory_notifier(cpuset_track_online_nodes, 10); | 2121 | hotplug_memory_notifier(cpuset_track_online_nodes, 10); |
| 2122 | |||
| 2123 | cpuset_wq = create_singlethread_workqueue("cpuset"); | ||
| 2124 | BUG_ON(!cpuset_wq); | ||
| 2114 | } | 2125 | } |
| 2115 | 2126 | ||
| 2116 | /** | 2127 | /** |
diff --git a/kernel/exit.c b/kernel/exit.c index f80dec3f1875..167e1e3ad7c6 100644 --- a/kernel/exit.c +++ b/kernel/exit.c | |||
| @@ -118,6 +118,8 @@ static void __exit_signal(struct task_struct *tsk) | |||
| 118 | * We won't ever get here for the group leader, since it | 118 | * We won't ever get here for the group leader, since it |
| 119 | * will have been the last reference on the signal_struct. | 119 | * will have been the last reference on the signal_struct. |
| 120 | */ | 120 | */ |
| 121 | sig->utime = cputime_add(sig->utime, task_utime(tsk)); | ||
| 122 | sig->stime = cputime_add(sig->stime, task_stime(tsk)); | ||
| 121 | sig->gtime = cputime_add(sig->gtime, task_gtime(tsk)); | 123 | sig->gtime = cputime_add(sig->gtime, task_gtime(tsk)); |
| 122 | sig->min_flt += tsk->min_flt; | 124 | sig->min_flt += tsk->min_flt; |
| 123 | sig->maj_flt += tsk->maj_flt; | 125 | sig->maj_flt += tsk->maj_flt; |
| @@ -126,6 +128,7 @@ static void __exit_signal(struct task_struct *tsk) | |||
| 126 | sig->inblock += task_io_get_inblock(tsk); | 128 | sig->inblock += task_io_get_inblock(tsk); |
| 127 | sig->oublock += task_io_get_oublock(tsk); | 129 | sig->oublock += task_io_get_oublock(tsk); |
| 128 | task_io_accounting_add(&sig->ioac, &tsk->ioac); | 130 | task_io_accounting_add(&sig->ioac, &tsk->ioac); |
| 131 | sig->sum_sched_runtime += tsk->se.sum_exec_runtime; | ||
| 129 | sig = NULL; /* Marker for below. */ | 132 | sig = NULL; /* Marker for below. */ |
| 130 | } | 133 | } |
| 131 | 134 | ||
| @@ -977,12 +980,9 @@ static void check_stack_usage(void) | |||
| 977 | { | 980 | { |
| 978 | static DEFINE_SPINLOCK(low_water_lock); | 981 | static DEFINE_SPINLOCK(low_water_lock); |
| 979 | static int lowest_to_date = THREAD_SIZE; | 982 | static int lowest_to_date = THREAD_SIZE; |
| 980 | unsigned long *n = end_of_stack(current); | ||
| 981 | unsigned long free; | 983 | unsigned long free; |
| 982 | 984 | ||
| 983 | while (*n == 0) | 985 | free = stack_not_used(current); |
| 984 | n++; | ||
| 985 | free = (unsigned long)n - (unsigned long)end_of_stack(current); | ||
| 986 | 986 | ||
| 987 | if (free >= lowest_to_date) | 987 | if (free >= lowest_to_date) |
| 988 | return; | 988 | return; |
diff --git a/kernel/fork.c b/kernel/fork.c index 242a706e7721..8de303bdd4e5 100644 --- a/kernel/fork.c +++ b/kernel/fork.c | |||
| @@ -61,6 +61,7 @@ | |||
| 61 | #include <linux/proc_fs.h> | 61 | #include <linux/proc_fs.h> |
| 62 | #include <linux/blkdev.h> | 62 | #include <linux/blkdev.h> |
| 63 | #include <trace/sched.h> | 63 | #include <trace/sched.h> |
| 64 | #include <linux/magic.h> | ||
| 64 | 65 | ||
| 65 | #include <asm/pgtable.h> | 66 | #include <asm/pgtable.h> |
| 66 | #include <asm/pgalloc.h> | 67 | #include <asm/pgalloc.h> |
| @@ -212,6 +213,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) | |||
| 212 | { | 213 | { |
| 213 | struct task_struct *tsk; | 214 | struct task_struct *tsk; |
| 214 | struct thread_info *ti; | 215 | struct thread_info *ti; |
| 216 | unsigned long *stackend; | ||
| 217 | |||
| 215 | int err; | 218 | int err; |
| 216 | 219 | ||
| 217 | prepare_to_copy(orig); | 220 | prepare_to_copy(orig); |
| @@ -237,6 +240,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) | |||
| 237 | goto out; | 240 | goto out; |
| 238 | 241 | ||
| 239 | setup_thread_stack(tsk, orig); | 242 | setup_thread_stack(tsk, orig); |
| 243 | stackend = end_of_stack(tsk); | ||
| 244 | *stackend = STACK_END_MAGIC; /* for overflow detection */ | ||
| 240 | 245 | ||
| 241 | #ifdef CONFIG_CC_STACKPROTECTOR | 246 | #ifdef CONFIG_CC_STACKPROTECTOR |
| 242 | tsk->stack_canary = get_random_int(); | 247 | tsk->stack_canary = get_random_int(); |
| @@ -851,13 +856,14 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk) | |||
| 851 | sig->tty_old_pgrp = NULL; | 856 | sig->tty_old_pgrp = NULL; |
| 852 | sig->tty = NULL; | 857 | sig->tty = NULL; |
| 853 | 858 | ||
| 854 | sig->cutime = sig->cstime = cputime_zero; | 859 | sig->utime = sig->stime = sig->cutime = sig->cstime = cputime_zero; |
| 855 | sig->gtime = cputime_zero; | 860 | sig->gtime = cputime_zero; |
| 856 | sig->cgtime = cputime_zero; | 861 | sig->cgtime = cputime_zero; |
| 857 | sig->nvcsw = sig->nivcsw = sig->cnvcsw = sig->cnivcsw = 0; | 862 | sig->nvcsw = sig->nivcsw = sig->cnvcsw = sig->cnivcsw = 0; |
| 858 | sig->min_flt = sig->maj_flt = sig->cmin_flt = sig->cmaj_flt = 0; | 863 | sig->min_flt = sig->maj_flt = sig->cmin_flt = sig->cmaj_flt = 0; |
| 859 | sig->inblock = sig->oublock = sig->cinblock = sig->coublock = 0; | 864 | sig->inblock = sig->oublock = sig->cinblock = sig->coublock = 0; |
| 860 | task_io_accounting_init(&sig->ioac); | 865 | task_io_accounting_init(&sig->ioac); |
| 866 | sig->sum_sched_runtime = 0; | ||
| 861 | taskstats_tgid_init(sig); | 867 | taskstats_tgid_init(sig); |
| 862 | 868 | ||
| 863 | task_lock(current->group_leader); | 869 | task_lock(current->group_leader); |
| @@ -1005,6 +1011,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, | |||
| 1005 | * triggers too late. This doesn't hurt, the check is only there | 1011 | * triggers too late. This doesn't hurt, the check is only there |
| 1006 | * to stop root fork bombs. | 1012 | * to stop root fork bombs. |
| 1007 | */ | 1013 | */ |
| 1014 | retval = -EAGAIN; | ||
| 1008 | if (nr_threads >= max_threads) | 1015 | if (nr_threads >= max_threads) |
| 1009 | goto bad_fork_cleanup_count; | 1016 | goto bad_fork_cleanup_count; |
| 1010 | 1017 | ||
| @@ -1093,7 +1100,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, | |||
| 1093 | #ifdef CONFIG_DEBUG_MUTEXES | 1100 | #ifdef CONFIG_DEBUG_MUTEXES |
| 1094 | p->blocked_on = NULL; /* not blocked yet */ | 1101 | p->blocked_on = NULL; /* not blocked yet */ |
| 1095 | #endif | 1102 | #endif |
| 1096 | if (unlikely(ptrace_reparented(current))) | 1103 | if (unlikely(current->ptrace)) |
| 1097 | ptrace_fork(p, clone_flags); | 1104 | ptrace_fork(p, clone_flags); |
| 1098 | 1105 | ||
| 1099 | /* Perform scheduler related setup. Assign this task to a CPU. */ | 1106 | /* Perform scheduler related setup. Assign this task to a CPU. */ |
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index f33afb0407bc..f394d2a42ca3 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c | |||
| @@ -501,6 +501,13 @@ static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base) | |||
| 501 | continue; | 501 | continue; |
| 502 | timer = rb_entry(base->first, struct hrtimer, node); | 502 | timer = rb_entry(base->first, struct hrtimer, node); |
| 503 | expires = ktime_sub(hrtimer_get_expires(timer), base->offset); | 503 | expires = ktime_sub(hrtimer_get_expires(timer), base->offset); |
| 504 | /* | ||
| 505 | * clock_was_set() has changed base->offset so the | ||
| 506 | * result might be negative. Fix it up to prevent a | ||
| 507 | * false positive in clockevents_program_event() | ||
| 508 | */ | ||
| 509 | if (expires.tv64 < 0) | ||
| 510 | expires.tv64 = 0; | ||
| 504 | if (expires.tv64 < cpu_base->expires_next.tv64) | 511 | if (expires.tv64 < cpu_base->expires_next.tv64) |
| 505 | cpu_base->expires_next = expires; | 512 | cpu_base->expires_next = expires; |
| 506 | } | 513 | } |
| @@ -1158,6 +1165,29 @@ static void __run_hrtimer(struct hrtimer *timer) | |||
| 1158 | 1165 | ||
| 1159 | #ifdef CONFIG_HIGH_RES_TIMERS | 1166 | #ifdef CONFIG_HIGH_RES_TIMERS |
| 1160 | 1167 | ||
| 1168 | static int force_clock_reprogram; | ||
| 1169 | |||
| 1170 | /* | ||
| 1171 | * After 5 iteration's attempts, we consider that hrtimer_interrupt() | ||
| 1172 | * is hanging, which could happen with something that slows the interrupt | ||
| 1173 | * such as the tracing. Then we force the clock reprogramming for each future | ||
| 1174 | * hrtimer interrupts to avoid infinite loops and use the min_delta_ns | ||
| 1175 | * threshold that we will overwrite. | ||
| 1176 | * The next tick event will be scheduled to 3 times we currently spend on | ||
| 1177 | * hrtimer_interrupt(). This gives a good compromise, the cpus will spend | ||
| 1178 | * 1/4 of their time to process the hrtimer interrupts. This is enough to | ||
| 1179 | * let it running without serious starvation. | ||
| 1180 | */ | ||
| 1181 | |||
| 1182 | static inline void | ||
| 1183 | hrtimer_interrupt_hanging(struct clock_event_device *dev, | ||
| 1184 | ktime_t try_time) | ||
| 1185 | { | ||
| 1186 | force_clock_reprogram = 1; | ||
| 1187 | dev->min_delta_ns = (unsigned long)try_time.tv64 * 3; | ||
| 1188 | printk(KERN_WARNING "hrtimer: interrupt too slow, " | ||
| 1189 | "forcing clock min delta to %lu ns\n", dev->min_delta_ns); | ||
| 1190 | } | ||
| 1161 | /* | 1191 | /* |
| 1162 | * High resolution timer interrupt | 1192 | * High resolution timer interrupt |
| 1163 | * Called with interrupts disabled | 1193 | * Called with interrupts disabled |
| @@ -1167,6 +1197,7 @@ void hrtimer_interrupt(struct clock_event_device *dev) | |||
| 1167 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); | 1197 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); |
| 1168 | struct hrtimer_clock_base *base; | 1198 | struct hrtimer_clock_base *base; |
| 1169 | ktime_t expires_next, now; | 1199 | ktime_t expires_next, now; |
| 1200 | int nr_retries = 0; | ||
| 1170 | int i; | 1201 | int i; |
| 1171 | 1202 | ||
| 1172 | BUG_ON(!cpu_base->hres_active); | 1203 | BUG_ON(!cpu_base->hres_active); |
| @@ -1174,6 +1205,10 @@ void hrtimer_interrupt(struct clock_event_device *dev) | |||
| 1174 | dev->next_event.tv64 = KTIME_MAX; | 1205 | dev->next_event.tv64 = KTIME_MAX; |
| 1175 | 1206 | ||
| 1176 | retry: | 1207 | retry: |
| 1208 | /* 5 retries is enough to notice a hang */ | ||
| 1209 | if (!(++nr_retries % 5)) | ||
| 1210 | hrtimer_interrupt_hanging(dev, ktime_sub(ktime_get(), now)); | ||
| 1211 | |||
| 1177 | now = ktime_get(); | 1212 | now = ktime_get(); |
| 1178 | 1213 | ||
| 1179 | expires_next.tv64 = KTIME_MAX; | 1214 | expires_next.tv64 = KTIME_MAX; |
| @@ -1226,7 +1261,7 @@ void hrtimer_interrupt(struct clock_event_device *dev) | |||
| 1226 | 1261 | ||
| 1227 | /* Reprogramming necessary ? */ | 1262 | /* Reprogramming necessary ? */ |
| 1228 | if (expires_next.tv64 != KTIME_MAX) { | 1263 | if (expires_next.tv64 != KTIME_MAX) { |
| 1229 | if (tick_program_event(expires_next, 0)) | 1264 | if (tick_program_event(expires_next, force_clock_reprogram)) |
| 1230 | goto retry; | 1265 | goto retry; |
| 1231 | } | 1266 | } |
| 1232 | } | 1267 | } |
| @@ -1580,6 +1615,10 @@ static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self, | |||
| 1580 | break; | 1615 | break; |
| 1581 | 1616 | ||
| 1582 | #ifdef CONFIG_HOTPLUG_CPU | 1617 | #ifdef CONFIG_HOTPLUG_CPU |
| 1618 | case CPU_DYING: | ||
| 1619 | case CPU_DYING_FROZEN: | ||
| 1620 | clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DYING, &scpu); | ||
| 1621 | break; | ||
| 1583 | case CPU_DEAD: | 1622 | case CPU_DEAD: |
| 1584 | case CPU_DEAD_FROZEN: | 1623 | case CPU_DEAD_FROZEN: |
| 1585 | { | 1624 | { |
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index f63c706d25e1..122fef4b0bd3 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c | |||
| @@ -46,7 +46,10 @@ void dynamic_irq_init(unsigned int irq) | |||
| 46 | desc->irq_count = 0; | 46 | desc->irq_count = 0; |
| 47 | desc->irqs_unhandled = 0; | 47 | desc->irqs_unhandled = 0; |
| 48 | #ifdef CONFIG_SMP | 48 | #ifdef CONFIG_SMP |
| 49 | cpumask_setall(&desc->affinity); | 49 | cpumask_setall(desc->affinity); |
| 50 | #ifdef CONFIG_GENERIC_PENDING_IRQ | ||
| 51 | cpumask_clear(desc->pending_mask); | ||
| 52 | #endif | ||
| 50 | #endif | 53 | #endif |
| 51 | spin_unlock_irqrestore(&desc->lock, flags); | 54 | spin_unlock_irqrestore(&desc->lock, flags); |
| 52 | } | 55 | } |
| @@ -383,6 +386,7 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc) | |||
| 383 | out_unlock: | 386 | out_unlock: |
| 384 | spin_unlock(&desc->lock); | 387 | spin_unlock(&desc->lock); |
| 385 | } | 388 | } |
| 389 | EXPORT_SYMBOL_GPL(handle_level_irq); | ||
| 386 | 390 | ||
| 387 | /** | 391 | /** |
| 388 | * handle_fasteoi_irq - irq handler for transparent controllers | 392 | * handle_fasteoi_irq - irq handler for transparent controllers |
| @@ -593,6 +597,7 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, | |||
| 593 | } | 597 | } |
| 594 | spin_unlock_irqrestore(&desc->lock, flags); | 598 | spin_unlock_irqrestore(&desc->lock, flags); |
| 595 | } | 599 | } |
| 600 | EXPORT_SYMBOL_GPL(__set_irq_handler); | ||
| 596 | 601 | ||
| 597 | void | 602 | void |
| 598 | set_irq_chip_and_handler(unsigned int irq, struct irq_chip *chip, | 603 | set_irq_chip_and_handler(unsigned int irq, struct irq_chip *chip, |
diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c index c20db0be9173..f51eaee921b6 100644 --- a/kernel/irq/handle.c +++ b/kernel/irq/handle.c | |||
| @@ -17,6 +17,7 @@ | |||
| 17 | #include <linux/kernel_stat.h> | 17 | #include <linux/kernel_stat.h> |
| 18 | #include <linux/rculist.h> | 18 | #include <linux/rculist.h> |
| 19 | #include <linux/hash.h> | 19 | #include <linux/hash.h> |
| 20 | #include <linux/bootmem.h> | ||
| 20 | 21 | ||
| 21 | #include "internals.h" | 22 | #include "internals.h" |
| 22 | 23 | ||
| @@ -39,6 +40,18 @@ void handle_bad_irq(unsigned int irq, struct irq_desc *desc) | |||
| 39 | ack_bad_irq(irq); | 40 | ack_bad_irq(irq); |
| 40 | } | 41 | } |
| 41 | 42 | ||
| 43 | #if defined(CONFIG_SMP) && defined(CONFIG_GENERIC_HARDIRQS) | ||
| 44 | static void __init init_irq_default_affinity(void) | ||
| 45 | { | ||
| 46 | alloc_bootmem_cpumask_var(&irq_default_affinity); | ||
| 47 | cpumask_setall(irq_default_affinity); | ||
| 48 | } | ||
| 49 | #else | ||
| 50 | static void __init init_irq_default_affinity(void) | ||
| 51 | { | ||
| 52 | } | ||
| 53 | #endif | ||
| 54 | |||
| 42 | /* | 55 | /* |
| 43 | * Linux has a controller-independent interrupt architecture. | 56 | * Linux has a controller-independent interrupt architecture. |
| 44 | * Every controller has a 'controller-template', that is used | 57 | * Every controller has a 'controller-template', that is used |
| @@ -57,6 +70,7 @@ int nr_irqs = NR_IRQS; | |||
| 57 | EXPORT_SYMBOL_GPL(nr_irqs); | 70 | EXPORT_SYMBOL_GPL(nr_irqs); |
| 58 | 71 | ||
| 59 | #ifdef CONFIG_SPARSE_IRQ | 72 | #ifdef CONFIG_SPARSE_IRQ |
| 73 | |||
| 60 | static struct irq_desc irq_desc_init = { | 74 | static struct irq_desc irq_desc_init = { |
| 61 | .irq = -1, | 75 | .irq = -1, |
| 62 | .status = IRQ_DISABLED, | 76 | .status = IRQ_DISABLED, |
| @@ -64,9 +78,6 @@ static struct irq_desc irq_desc_init = { | |||
| 64 | .handle_irq = handle_bad_irq, | 78 | .handle_irq = handle_bad_irq, |
| 65 | .depth = 1, | 79 | .depth = 1, |
| 66 | .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock), | 80 | .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock), |
| 67 | #ifdef CONFIG_SMP | ||
| 68 | .affinity = CPU_MASK_ALL | ||
| 69 | #endif | ||
| 70 | }; | 81 | }; |
| 71 | 82 | ||
| 72 | void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr) | 83 | void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr) |
| @@ -101,6 +112,10 @@ static void init_one_irq_desc(int irq, struct irq_desc *desc, int cpu) | |||
| 101 | printk(KERN_ERR "can not alloc kstat_irqs\n"); | 112 | printk(KERN_ERR "can not alloc kstat_irqs\n"); |
| 102 | BUG_ON(1); | 113 | BUG_ON(1); |
| 103 | } | 114 | } |
| 115 | if (!init_alloc_desc_masks(desc, cpu, false)) { | ||
| 116 | printk(KERN_ERR "can not alloc irq_desc cpumasks\n"); | ||
| 117 | BUG_ON(1); | ||
| 118 | } | ||
| 104 | arch_init_chip_data(desc, cpu); | 119 | arch_init_chip_data(desc, cpu); |
| 105 | } | 120 | } |
| 106 | 121 | ||
| @@ -109,7 +124,7 @@ static void init_one_irq_desc(int irq, struct irq_desc *desc, int cpu) | |||
| 109 | */ | 124 | */ |
| 110 | DEFINE_SPINLOCK(sparse_irq_lock); | 125 | DEFINE_SPINLOCK(sparse_irq_lock); |
| 111 | 126 | ||
| 112 | struct irq_desc *irq_desc_ptrs[NR_IRQS] __read_mostly; | 127 | struct irq_desc **irq_desc_ptrs __read_mostly; |
| 113 | 128 | ||
| 114 | static struct irq_desc irq_desc_legacy[NR_IRQS_LEGACY] __cacheline_aligned_in_smp = { | 129 | static struct irq_desc irq_desc_legacy[NR_IRQS_LEGACY] __cacheline_aligned_in_smp = { |
| 115 | [0 ... NR_IRQS_LEGACY-1] = { | 130 | [0 ... NR_IRQS_LEGACY-1] = { |
| @@ -119,14 +134,10 @@ static struct irq_desc irq_desc_legacy[NR_IRQS_LEGACY] __cacheline_aligned_in_sm | |||
| 119 | .handle_irq = handle_bad_irq, | 134 | .handle_irq = handle_bad_irq, |
| 120 | .depth = 1, | 135 | .depth = 1, |
| 121 | .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock), | 136 | .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock), |
| 122 | #ifdef CONFIG_SMP | ||
| 123 | .affinity = CPU_MASK_ALL | ||
| 124 | #endif | ||
| 125 | } | 137 | } |
| 126 | }; | 138 | }; |
| 127 | 139 | ||
| 128 | /* FIXME: use bootmem alloc ...*/ | 140 | static unsigned int *kstat_irqs_legacy; |
| 129 | static unsigned int kstat_irqs_legacy[NR_IRQS_LEGACY][NR_CPUS]; | ||
| 130 | 141 | ||
| 131 | int __init early_irq_init(void) | 142 | int __init early_irq_init(void) |
| 132 | { | 143 | { |
| @@ -134,18 +145,32 @@ int __init early_irq_init(void) | |||
| 134 | int legacy_count; | 145 | int legacy_count; |
| 135 | int i; | 146 | int i; |
| 136 | 147 | ||
| 148 | init_irq_default_affinity(); | ||
| 149 | |||
| 150 | /* initialize nr_irqs based on nr_cpu_ids */ | ||
| 151 | arch_probe_nr_irqs(); | ||
| 152 | printk(KERN_INFO "NR_IRQS:%d nr_irqs:%d\n", NR_IRQS, nr_irqs); | ||
| 153 | |||
| 137 | desc = irq_desc_legacy; | 154 | desc = irq_desc_legacy; |
| 138 | legacy_count = ARRAY_SIZE(irq_desc_legacy); | 155 | legacy_count = ARRAY_SIZE(irq_desc_legacy); |
| 139 | 156 | ||
| 157 | /* allocate irq_desc_ptrs array based on nr_irqs */ | ||
| 158 | irq_desc_ptrs = alloc_bootmem(nr_irqs * sizeof(void *)); | ||
| 159 | |||
| 160 | /* allocate based on nr_cpu_ids */ | ||
| 161 | /* FIXME: invert kstat_irgs, and it'd be a per_cpu_alloc'd thing */ | ||
| 162 | kstat_irqs_legacy = alloc_bootmem(NR_IRQS_LEGACY * nr_cpu_ids * | ||
| 163 | sizeof(int)); | ||
| 164 | |||
| 140 | for (i = 0; i < legacy_count; i++) { | 165 | for (i = 0; i < legacy_count; i++) { |
| 141 | desc[i].irq = i; | 166 | desc[i].irq = i; |
| 142 | desc[i].kstat_irqs = kstat_irqs_legacy[i]; | 167 | desc[i].kstat_irqs = kstat_irqs_legacy + i * nr_cpu_ids; |
| 143 | lockdep_set_class(&desc[i].lock, &irq_desc_lock_class); | 168 | lockdep_set_class(&desc[i].lock, &irq_desc_lock_class); |
| 144 | 169 | init_alloc_desc_masks(&desc[i], 0, true); | |
| 145 | irq_desc_ptrs[i] = desc + i; | 170 | irq_desc_ptrs[i] = desc + i; |
| 146 | } | 171 | } |
| 147 | 172 | ||
| 148 | for (i = legacy_count; i < NR_IRQS; i++) | 173 | for (i = legacy_count; i < nr_irqs; i++) |
| 149 | irq_desc_ptrs[i] = NULL; | 174 | irq_desc_ptrs[i] = NULL; |
| 150 | 175 | ||
| 151 | return arch_early_irq_init(); | 176 | return arch_early_irq_init(); |
| @@ -153,7 +178,10 @@ int __init early_irq_init(void) | |||
| 153 | 178 | ||
| 154 | struct irq_desc *irq_to_desc(unsigned int irq) | 179 | struct irq_desc *irq_to_desc(unsigned int irq) |
| 155 | { | 180 | { |
| 156 | return (irq < NR_IRQS) ? irq_desc_ptrs[irq] : NULL; | 181 | if (irq_desc_ptrs && irq < nr_irqs) |
| 182 | return irq_desc_ptrs[irq]; | ||
| 183 | |||
| 184 | return NULL; | ||
| 157 | } | 185 | } |
| 158 | 186 | ||
| 159 | struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu) | 187 | struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu) |
| @@ -162,10 +190,9 @@ struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu) | |||
| 162 | unsigned long flags; | 190 | unsigned long flags; |
| 163 | int node; | 191 | int node; |
| 164 | 192 | ||
| 165 | if (irq >= NR_IRQS) { | 193 | if (irq >= nr_irqs) { |
| 166 | printk(KERN_WARNING "irq >= NR_IRQS in irq_to_desc_alloc: %d %d\n", | 194 | WARN(1, "irq (%d) >= nr_irqs (%d) in irq_to_desc_alloc\n", |
| 167 | irq, NR_IRQS); | 195 | irq, nr_irqs); |
| 168 | WARN_ON(1); | ||
| 169 | return NULL; | 196 | return NULL; |
| 170 | } | 197 | } |
| 171 | 198 | ||
| @@ -207,9 +234,6 @@ struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = { | |||
| 207 | .handle_irq = handle_bad_irq, | 234 | .handle_irq = handle_bad_irq, |
| 208 | .depth = 1, | 235 | .depth = 1, |
| 209 | .lock = __SPIN_LOCK_UNLOCKED(irq_desc->lock), | 236 | .lock = __SPIN_LOCK_UNLOCKED(irq_desc->lock), |
| 210 | #ifdef CONFIG_SMP | ||
| 211 | .affinity = CPU_MASK_ALL | ||
| 212 | #endif | ||
| 213 | } | 237 | } |
| 214 | }; | 238 | }; |
| 215 | 239 | ||
| @@ -219,12 +243,17 @@ int __init early_irq_init(void) | |||
| 219 | int count; | 243 | int count; |
| 220 | int i; | 244 | int i; |
| 221 | 245 | ||
| 246 | init_irq_default_affinity(); | ||
| 247 | |||
| 248 | printk(KERN_INFO "NR_IRQS:%d\n", NR_IRQS); | ||
| 249 | |||
| 222 | desc = irq_desc; | 250 | desc = irq_desc; |
| 223 | count = ARRAY_SIZE(irq_desc); | 251 | count = ARRAY_SIZE(irq_desc); |
| 224 | 252 | ||
| 225 | for (i = 0; i < count; i++) | 253 | for (i = 0; i < count; i++) { |
| 226 | desc[i].irq = i; | 254 | desc[i].irq = i; |
| 227 | 255 | init_alloc_desc_masks(&desc[i], 0, true); | |
| 256 | } | ||
| 228 | return arch_early_irq_init(); | 257 | return arch_early_irq_init(); |
| 229 | } | 258 | } |
| 230 | 259 | ||
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index e6d0a43cc125..40416a81a0f5 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h | |||
| @@ -16,7 +16,14 @@ extern int __irq_set_trigger(struct irq_desc *desc, unsigned int irq, | |||
| 16 | extern struct lock_class_key irq_desc_lock_class; | 16 | extern struct lock_class_key irq_desc_lock_class; |
| 17 | extern void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr); | 17 | extern void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr); |
| 18 | extern spinlock_t sparse_irq_lock; | 18 | extern spinlock_t sparse_irq_lock; |
| 19 | |||
| 20 | #ifdef CONFIG_SPARSE_IRQ | ||
| 21 | /* irq_desc_ptrs allocated at boot time */ | ||
| 22 | extern struct irq_desc **irq_desc_ptrs; | ||
| 23 | #else | ||
| 24 | /* irq_desc_ptrs is a fixed size array */ | ||
| 19 | extern struct irq_desc *irq_desc_ptrs[NR_IRQS]; | 25 | extern struct irq_desc *irq_desc_ptrs[NR_IRQS]; |
| 26 | #endif | ||
| 20 | 27 | ||
| 21 | #ifdef CONFIG_PROC_FS | 28 | #ifdef CONFIG_PROC_FS |
| 22 | extern void register_irq_proc(unsigned int irq, struct irq_desc *desc); | 29 | extern void register_irq_proc(unsigned int irq, struct irq_desc *desc); |
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index cd0cd8dcb345..a3a5dc9ef346 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c | |||
| @@ -15,17 +15,9 @@ | |||
| 15 | 15 | ||
| 16 | #include "internals.h" | 16 | #include "internals.h" |
| 17 | 17 | ||
| 18 | #ifdef CONFIG_SMP | 18 | #if defined(CONFIG_SMP) && defined(CONFIG_GENERIC_HARDIRQS) |
| 19 | cpumask_var_t irq_default_affinity; | 19 | cpumask_var_t irq_default_affinity; |
| 20 | 20 | ||
| 21 | static int init_irq_default_affinity(void) | ||
| 22 | { | ||
| 23 | alloc_cpumask_var(&irq_default_affinity, GFP_KERNEL); | ||
| 24 | cpumask_setall(irq_default_affinity); | ||
| 25 | return 0; | ||
| 26 | } | ||
| 27 | core_initcall(init_irq_default_affinity); | ||
| 28 | |||
| 29 | /** | 21 | /** |
| 30 | * synchronize_irq - wait for pending IRQ handlers (on other CPUs) | 22 | * synchronize_irq - wait for pending IRQ handlers (on other CPUs) |
| 31 | * @irq: interrupt number to wait for | 23 | * @irq: interrupt number to wait for |
| @@ -98,14 +90,14 @@ int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask) | |||
| 98 | 90 | ||
| 99 | #ifdef CONFIG_GENERIC_PENDING_IRQ | 91 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
| 100 | if (desc->status & IRQ_MOVE_PCNTXT || desc->status & IRQ_DISABLED) { | 92 | if (desc->status & IRQ_MOVE_PCNTXT || desc->status & IRQ_DISABLED) { |
| 101 | cpumask_copy(&desc->affinity, cpumask); | 93 | cpumask_copy(desc->affinity, cpumask); |
| 102 | desc->chip->set_affinity(irq, cpumask); | 94 | desc->chip->set_affinity(irq, cpumask); |
| 103 | } else { | 95 | } else { |
| 104 | desc->status |= IRQ_MOVE_PENDING; | 96 | desc->status |= IRQ_MOVE_PENDING; |
| 105 | cpumask_copy(&desc->pending_mask, cpumask); | 97 | cpumask_copy(desc->pending_mask, cpumask); |
| 106 | } | 98 | } |
| 107 | #else | 99 | #else |
| 108 | cpumask_copy(&desc->affinity, cpumask); | 100 | cpumask_copy(desc->affinity, cpumask); |
| 109 | desc->chip->set_affinity(irq, cpumask); | 101 | desc->chip->set_affinity(irq, cpumask); |
| 110 | #endif | 102 | #endif |
| 111 | desc->status |= IRQ_AFFINITY_SET; | 103 | desc->status |= IRQ_AFFINITY_SET; |
| @@ -127,16 +119,16 @@ int do_irq_select_affinity(unsigned int irq, struct irq_desc *desc) | |||
| 127 | * one of the targets is online. | 119 | * one of the targets is online. |
| 128 | */ | 120 | */ |
| 129 | if (desc->status & (IRQ_AFFINITY_SET | IRQ_NO_BALANCING)) { | 121 | if (desc->status & (IRQ_AFFINITY_SET | IRQ_NO_BALANCING)) { |
| 130 | if (cpumask_any_and(&desc->affinity, cpu_online_mask) | 122 | if (cpumask_any_and(desc->affinity, cpu_online_mask) |
| 131 | < nr_cpu_ids) | 123 | < nr_cpu_ids) |
| 132 | goto set_affinity; | 124 | goto set_affinity; |
| 133 | else | 125 | else |
| 134 | desc->status &= ~IRQ_AFFINITY_SET; | 126 | desc->status &= ~IRQ_AFFINITY_SET; |
| 135 | } | 127 | } |
| 136 | 128 | ||
| 137 | cpumask_and(&desc->affinity, cpu_online_mask, irq_default_affinity); | 129 | cpumask_and(desc->affinity, cpu_online_mask, irq_default_affinity); |
| 138 | set_affinity: | 130 | set_affinity: |
| 139 | desc->chip->set_affinity(irq, &desc->affinity); | 131 | desc->chip->set_affinity(irq, desc->affinity); |
| 140 | 132 | ||
| 141 | return 0; | 133 | return 0; |
| 142 | } | 134 | } |
diff --git a/kernel/irq/migration.c b/kernel/irq/migration.c index bd72329e630c..e05ad9be43b7 100644 --- a/kernel/irq/migration.c +++ b/kernel/irq/migration.c | |||
| @@ -18,7 +18,7 @@ void move_masked_irq(int irq) | |||
| 18 | 18 | ||
| 19 | desc->status &= ~IRQ_MOVE_PENDING; | 19 | desc->status &= ~IRQ_MOVE_PENDING; |
| 20 | 20 | ||
| 21 | if (unlikely(cpumask_empty(&desc->pending_mask))) | 21 | if (unlikely(cpumask_empty(desc->pending_mask))) |
| 22 | return; | 22 | return; |
| 23 | 23 | ||
| 24 | if (!desc->chip->set_affinity) | 24 | if (!desc->chip->set_affinity) |
| @@ -38,13 +38,13 @@ void move_masked_irq(int irq) | |||
| 38 | * For correct operation this depends on the caller | 38 | * For correct operation this depends on the caller |
| 39 | * masking the irqs. | 39 | * masking the irqs. |
| 40 | */ | 40 | */ |
| 41 | if (likely(cpumask_any_and(&desc->pending_mask, cpu_online_mask) | 41 | if (likely(cpumask_any_and(desc->pending_mask, cpu_online_mask) |
| 42 | < nr_cpu_ids)) { | 42 | < nr_cpu_ids)) { |
| 43 | cpumask_and(&desc->affinity, | 43 | cpumask_and(desc->affinity, |
| 44 | &desc->pending_mask, cpu_online_mask); | 44 | desc->pending_mask, cpu_online_mask); |
| 45 | desc->chip->set_affinity(irq, &desc->affinity); | 45 | desc->chip->set_affinity(irq, desc->affinity); |
| 46 | } | 46 | } |
| 47 | cpumask_clear(&desc->pending_mask); | 47 | cpumask_clear(desc->pending_mask); |
| 48 | } | 48 | } |
| 49 | 49 | ||
| 50 | void move_native_irq(int irq) | 50 | void move_native_irq(int irq) |
diff --git a/kernel/irq/numa_migrate.c b/kernel/irq/numa_migrate.c index ecf765c6a77a..7f9b80434e32 100644 --- a/kernel/irq/numa_migrate.c +++ b/kernel/irq/numa_migrate.c | |||
| @@ -38,15 +38,22 @@ static void free_kstat_irqs(struct irq_desc *old_desc, struct irq_desc *desc) | |||
| 38 | old_desc->kstat_irqs = NULL; | 38 | old_desc->kstat_irqs = NULL; |
| 39 | } | 39 | } |
| 40 | 40 | ||
| 41 | static void init_copy_one_irq_desc(int irq, struct irq_desc *old_desc, | 41 | static bool init_copy_one_irq_desc(int irq, struct irq_desc *old_desc, |
| 42 | struct irq_desc *desc, int cpu) | 42 | struct irq_desc *desc, int cpu) |
| 43 | { | 43 | { |
| 44 | memcpy(desc, old_desc, sizeof(struct irq_desc)); | 44 | memcpy(desc, old_desc, sizeof(struct irq_desc)); |
| 45 | if (!init_alloc_desc_masks(desc, cpu, false)) { | ||
| 46 | printk(KERN_ERR "irq %d: can not get new irq_desc cpumask " | ||
| 47 | "for migration.\n", irq); | ||
| 48 | return false; | ||
| 49 | } | ||
| 45 | spin_lock_init(&desc->lock); | 50 | spin_lock_init(&desc->lock); |
| 46 | desc->cpu = cpu; | 51 | desc->cpu = cpu; |
| 47 | lockdep_set_class(&desc->lock, &irq_desc_lock_class); | 52 | lockdep_set_class(&desc->lock, &irq_desc_lock_class); |
| 48 | init_copy_kstat_irqs(old_desc, desc, cpu, nr_cpu_ids); | 53 | init_copy_kstat_irqs(old_desc, desc, cpu, nr_cpu_ids); |
| 54 | init_copy_desc_masks(old_desc, desc); | ||
| 49 | arch_init_copy_chip_data(old_desc, desc, cpu); | 55 | arch_init_copy_chip_data(old_desc, desc, cpu); |
| 56 | return true; | ||
| 50 | } | 57 | } |
| 51 | 58 | ||
| 52 | static void free_one_irq_desc(struct irq_desc *old_desc, struct irq_desc *desc) | 59 | static void free_one_irq_desc(struct irq_desc *old_desc, struct irq_desc *desc) |
| @@ -71,23 +78,34 @@ static struct irq_desc *__real_move_irq_desc(struct irq_desc *old_desc, | |||
| 71 | desc = irq_desc_ptrs[irq]; | 78 | desc = irq_desc_ptrs[irq]; |
| 72 | 79 | ||
| 73 | if (desc && old_desc != desc) | 80 | if (desc && old_desc != desc) |
| 74 | goto out_unlock; | 81 | goto out_unlock; |
| 75 | 82 | ||
| 76 | node = cpu_to_node(cpu); | 83 | node = cpu_to_node(cpu); |
| 77 | desc = kzalloc_node(sizeof(*desc), GFP_ATOMIC, node); | 84 | desc = kzalloc_node(sizeof(*desc), GFP_ATOMIC, node); |
| 78 | if (!desc) { | 85 | if (!desc) { |
| 79 | printk(KERN_ERR "irq %d: can not get new irq_desc for migration.\n", irq); | 86 | printk(KERN_ERR "irq %d: can not get new irq_desc " |
| 87 | "for migration.\n", irq); | ||
| 80 | /* still use old one */ | 88 | /* still use old one */ |
| 81 | desc = old_desc; | 89 | desc = old_desc; |
| 82 | goto out_unlock; | 90 | goto out_unlock; |
| 83 | } | 91 | } |
| 84 | init_copy_one_irq_desc(irq, old_desc, desc, cpu); | 92 | if (!init_copy_one_irq_desc(irq, old_desc, desc, cpu)) { |
| 93 | /* still use old one */ | ||
| 94 | kfree(desc); | ||
| 95 | desc = old_desc; | ||
| 96 | goto out_unlock; | ||
| 97 | } | ||
| 85 | 98 | ||
| 86 | irq_desc_ptrs[irq] = desc; | 99 | irq_desc_ptrs[irq] = desc; |
| 100 | spin_unlock_irqrestore(&sparse_irq_lock, flags); | ||
| 87 | 101 | ||
| 88 | /* free the old one */ | 102 | /* free the old one */ |
| 89 | free_one_irq_desc(old_desc, desc); | 103 | free_one_irq_desc(old_desc, desc); |
| 104 | spin_unlock(&old_desc->lock); | ||
| 90 | kfree(old_desc); | 105 | kfree(old_desc); |
| 106 | spin_lock(&desc->lock); | ||
| 107 | |||
| 108 | return desc; | ||
| 91 | 109 | ||
| 92 | out_unlock: | 110 | out_unlock: |
| 93 | spin_unlock_irqrestore(&sparse_irq_lock, flags); | 111 | spin_unlock_irqrestore(&sparse_irq_lock, flags); |
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index aae3f742bcec..692363dd591f 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c | |||
| @@ -20,11 +20,11 @@ static struct proc_dir_entry *root_irq_dir; | |||
| 20 | static int irq_affinity_proc_show(struct seq_file *m, void *v) | 20 | static int irq_affinity_proc_show(struct seq_file *m, void *v) |
| 21 | { | 21 | { |
| 22 | struct irq_desc *desc = irq_to_desc((long)m->private); | 22 | struct irq_desc *desc = irq_to_desc((long)m->private); |
| 23 | const struct cpumask *mask = &desc->affinity; | 23 | const struct cpumask *mask = desc->affinity; |
| 24 | 24 | ||
| 25 | #ifdef CONFIG_GENERIC_PENDING_IRQ | 25 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
| 26 | if (desc->status & IRQ_MOVE_PENDING) | 26 | if (desc->status & IRQ_MOVE_PENDING) |
| 27 | mask = &desc->pending_mask; | 27 | mask = desc->pending_mask; |
| 28 | #endif | 28 | #endif |
| 29 | seq_cpumask(m, mask); | 29 | seq_cpumask(m, mask); |
| 30 | seq_putc(m, '\n'); | 30 | seq_putc(m, '\n'); |
diff --git a/kernel/itimer.c b/kernel/itimer.c index 6a5fe93dd8bd..58762f7077ec 100644 --- a/kernel/itimer.c +++ b/kernel/itimer.c | |||
| @@ -62,7 +62,7 @@ int do_getitimer(int which, struct itimerval *value) | |||
| 62 | struct task_cputime cputime; | 62 | struct task_cputime cputime; |
| 63 | cputime_t utime; | 63 | cputime_t utime; |
| 64 | 64 | ||
| 65 | thread_group_cputime(tsk, &cputime); | 65 | thread_group_cputimer(tsk, &cputime); |
| 66 | utime = cputime.utime; | 66 | utime = cputime.utime; |
| 67 | if (cputime_le(cval, utime)) { /* about to fire */ | 67 | if (cputime_le(cval, utime)) { /* about to fire */ |
| 68 | cval = jiffies_to_cputime(1); | 68 | cval = jiffies_to_cputime(1); |
| @@ -82,7 +82,7 @@ int do_getitimer(int which, struct itimerval *value) | |||
| 82 | struct task_cputime times; | 82 | struct task_cputime times; |
| 83 | cputime_t ptime; | 83 | cputime_t ptime; |
| 84 | 84 | ||
| 85 | thread_group_cputime(tsk, ×); | 85 | thread_group_cputimer(tsk, ×); |
| 86 | ptime = cputime_add(times.utime, times.stime); | 86 | ptime = cputime_add(times.utime, times.stime); |
| 87 | if (cputime_le(cval, ptime)) { /* about to fire */ | 87 | if (cputime_le(cval, ptime)) { /* about to fire */ |
| 88 | cval = jiffies_to_cputime(1); | 88 | cval = jiffies_to_cputime(1); |
diff --git a/kernel/kexec.c b/kernel/kexec.c index 8a6d7b08864e..795e7b67a228 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c | |||
| @@ -1130,7 +1130,7 @@ void crash_save_cpu(struct pt_regs *regs, int cpu) | |||
| 1130 | return; | 1130 | return; |
| 1131 | memset(&prstatus, 0, sizeof(prstatus)); | 1131 | memset(&prstatus, 0, sizeof(prstatus)); |
| 1132 | prstatus.pr_pid = current->pid; | 1132 | prstatus.pr_pid = current->pid; |
| 1133 | elf_core_copy_regs(&prstatus.pr_reg, regs); | 1133 | elf_core_copy_kernel_regs(&prstatus.pr_reg, regs); |
| 1134 | buf = append_elf_note(buf, KEXEC_CORE_NOTE_NAME, NT_PRSTATUS, | 1134 | buf = append_elf_note(buf, KEXEC_CORE_NOTE_NAME, NT_PRSTATUS, |
| 1135 | &prstatus, sizeof(prstatus)); | 1135 | &prstatus, sizeof(prstatus)); |
| 1136 | final_note(buf); | 1136 | final_note(buf); |
diff --git a/kernel/module.c b/kernel/module.c index e8b51d41dd72..ba22484a987e 100644 --- a/kernel/module.c +++ b/kernel/module.c | |||
| @@ -573,13 +573,13 @@ static char last_unloaded_module[MODULE_NAME_LEN+1]; | |||
| 573 | /* Init the unload section of the module. */ | 573 | /* Init the unload section of the module. */ |
| 574 | static void module_unload_init(struct module *mod) | 574 | static void module_unload_init(struct module *mod) |
| 575 | { | 575 | { |
| 576 | unsigned int i; | 576 | int cpu; |
| 577 | 577 | ||
| 578 | INIT_LIST_HEAD(&mod->modules_which_use_me); | 578 | INIT_LIST_HEAD(&mod->modules_which_use_me); |
| 579 | for (i = 0; i < NR_CPUS; i++) | 579 | for_each_possible_cpu(cpu) |
| 580 | local_set(&mod->ref[i].count, 0); | 580 | local_set(__module_ref_addr(mod, cpu), 0); |
| 581 | /* Hold reference count during initialization. */ | 581 | /* Hold reference count during initialization. */ |
| 582 | local_set(&mod->ref[raw_smp_processor_id()].count, 1); | 582 | local_set(__module_ref_addr(mod, raw_smp_processor_id()), 1); |
| 583 | /* Backwards compatibility macros put refcount during init. */ | 583 | /* Backwards compatibility macros put refcount during init. */ |
| 584 | mod->waiter = current; | 584 | mod->waiter = current; |
| 585 | } | 585 | } |
| @@ -717,10 +717,11 @@ static int try_stop_module(struct module *mod, int flags, int *forced) | |||
| 717 | 717 | ||
| 718 | unsigned int module_refcount(struct module *mod) | 718 | unsigned int module_refcount(struct module *mod) |
| 719 | { | 719 | { |
| 720 | unsigned int i, total = 0; | 720 | unsigned int total = 0; |
| 721 | int cpu; | ||
| 721 | 722 | ||
| 722 | for (i = 0; i < NR_CPUS; i++) | 723 | for_each_possible_cpu(cpu) |
| 723 | total += local_read(&mod->ref[i].count); | 724 | total += local_read(__module_ref_addr(mod, cpu)); |
| 724 | return total; | 725 | return total; |
| 725 | } | 726 | } |
| 726 | EXPORT_SYMBOL(module_refcount); | 727 | EXPORT_SYMBOL(module_refcount); |
| @@ -894,7 +895,7 @@ void module_put(struct module *module) | |||
| 894 | { | 895 | { |
| 895 | if (module) { | 896 | if (module) { |
| 896 | unsigned int cpu = get_cpu(); | 897 | unsigned int cpu = get_cpu(); |
| 897 | local_dec(&module->ref[cpu].count); | 898 | local_dec(__module_ref_addr(module, cpu)); |
| 898 | /* Maybe they're waiting for us to drop reference? */ | 899 | /* Maybe they're waiting for us to drop reference? */ |
| 899 | if (unlikely(!module_is_live(module))) | 900 | if (unlikely(!module_is_live(module))) |
| 900 | wake_up_process(module->waiter); | 901 | wake_up_process(module->waiter); |
| @@ -1464,7 +1465,10 @@ static void free_module(struct module *mod) | |||
| 1464 | kfree(mod->args); | 1465 | kfree(mod->args); |
| 1465 | if (mod->percpu) | 1466 | if (mod->percpu) |
| 1466 | percpu_modfree(mod->percpu); | 1467 | percpu_modfree(mod->percpu); |
| 1467 | 1468 | #if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP) | |
| 1469 | if (mod->refptr) | ||
| 1470 | percpu_modfree(mod->refptr); | ||
| 1471 | #endif | ||
| 1468 | /* Free lock-classes: */ | 1472 | /* Free lock-classes: */ |
| 1469 | lockdep_free_key_range(mod->module_core, mod->core_size); | 1473 | lockdep_free_key_range(mod->module_core, mod->core_size); |
| 1470 | 1474 | ||
| @@ -2011,6 +2015,14 @@ static noinline struct module *load_module(void __user *umod, | |||
| 2011 | if (err < 0) | 2015 | if (err < 0) |
| 2012 | goto free_mod; | 2016 | goto free_mod; |
| 2013 | 2017 | ||
| 2018 | #if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP) | ||
| 2019 | mod->refptr = percpu_modalloc(sizeof(local_t), __alignof__(local_t), | ||
| 2020 | mod->name); | ||
| 2021 | if (!mod->refptr) { | ||
| 2022 | err = -ENOMEM; | ||
| 2023 | goto free_mod; | ||
| 2024 | } | ||
| 2025 | #endif | ||
| 2014 | if (pcpuindex) { | 2026 | if (pcpuindex) { |
| 2015 | /* We have a special allocation for this section. */ | 2027 | /* We have a special allocation for this section. */ |
| 2016 | percpu = percpu_modalloc(sechdrs[pcpuindex].sh_size, | 2028 | percpu = percpu_modalloc(sechdrs[pcpuindex].sh_size, |
| @@ -2018,7 +2030,7 @@ static noinline struct module *load_module(void __user *umod, | |||
| 2018 | mod->name); | 2030 | mod->name); |
| 2019 | if (!percpu) { | 2031 | if (!percpu) { |
| 2020 | err = -ENOMEM; | 2032 | err = -ENOMEM; |
| 2021 | goto free_mod; | 2033 | goto free_percpu; |
| 2022 | } | 2034 | } |
| 2023 | sechdrs[pcpuindex].sh_flags &= ~(unsigned long)SHF_ALLOC; | 2035 | sechdrs[pcpuindex].sh_flags &= ~(unsigned long)SHF_ALLOC; |
| 2024 | mod->percpu = percpu; | 2036 | mod->percpu = percpu; |
| @@ -2282,6 +2294,9 @@ static noinline struct module *load_module(void __user *umod, | |||
| 2282 | free_percpu: | 2294 | free_percpu: |
| 2283 | if (percpu) | 2295 | if (percpu) |
| 2284 | percpu_modfree(percpu); | 2296 | percpu_modfree(percpu); |
| 2297 | #if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP) | ||
| 2298 | percpu_modfree(mod->refptr); | ||
| 2299 | #endif | ||
| 2285 | free_mod: | 2300 | free_mod: |
| 2286 | kfree(args); | 2301 | kfree(args); |
| 2287 | free_hdr: | 2302 | free_hdr: |
diff --git a/kernel/panic.c b/kernel/panic.c index 2a2ff36ff44d..32fe4eff1b89 100644 --- a/kernel/panic.c +++ b/kernel/panic.c | |||
| @@ -74,6 +74,9 @@ NORET_TYPE void panic(const char * fmt, ...) | |||
| 74 | vsnprintf(buf, sizeof(buf), fmt, args); | 74 | vsnprintf(buf, sizeof(buf), fmt, args); |
| 75 | va_end(args); | 75 | va_end(args); |
| 76 | printk(KERN_EMERG "Kernel panic - not syncing: %s\n",buf); | 76 | printk(KERN_EMERG "Kernel panic - not syncing: %s\n",buf); |
| 77 | #ifdef CONFIG_DEBUG_BUGVERBOSE | ||
| 78 | dump_stack(); | ||
| 79 | #endif | ||
| 77 | bust_spinlocks(0); | 80 | bust_spinlocks(0); |
| 78 | 81 | ||
| 79 | /* | 82 | /* |
| @@ -355,15 +358,18 @@ EXPORT_SYMBOL(warn_slowpath); | |||
| 355 | #endif | 358 | #endif |
| 356 | 359 | ||
| 357 | #ifdef CONFIG_CC_STACKPROTECTOR | 360 | #ifdef CONFIG_CC_STACKPROTECTOR |
| 361 | |||
| 358 | /* | 362 | /* |
| 359 | * Called when gcc's -fstack-protector feature is used, and | 363 | * Called when gcc's -fstack-protector feature is used, and |
| 360 | * gcc detects corruption of the on-stack canary value | 364 | * gcc detects corruption of the on-stack canary value |
| 361 | */ | 365 | */ |
| 362 | void __stack_chk_fail(void) | 366 | void __stack_chk_fail(void) |
| 363 | { | 367 | { |
| 364 | panic("stack-protector: Kernel stack is corrupted"); | 368 | panic("stack-protector: Kernel stack is corrupted in: %p\n", |
| 369 | __builtin_return_address(0)); | ||
| 365 | } | 370 | } |
| 366 | EXPORT_SYMBOL(__stack_chk_fail); | 371 | EXPORT_SYMBOL(__stack_chk_fail); |
| 372 | |||
| 367 | #endif | 373 | #endif |
| 368 | 374 | ||
| 369 | core_param(panic, panic_timeout, int, 0644); | 375 | core_param(panic, panic_timeout, int, 0644); |
diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c index fa07da94d7be..2313a4cc14ea 100644 --- a/kernel/posix-cpu-timers.c +++ b/kernel/posix-cpu-timers.c | |||
| @@ -230,6 +230,71 @@ static int cpu_clock_sample(const clockid_t which_clock, struct task_struct *p, | |||
| 230 | return 0; | 230 | return 0; |
| 231 | } | 231 | } |
| 232 | 232 | ||
| 233 | void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times) | ||
| 234 | { | ||
| 235 | struct sighand_struct *sighand; | ||
| 236 | struct signal_struct *sig; | ||
| 237 | struct task_struct *t; | ||
| 238 | |||
| 239 | *times = INIT_CPUTIME; | ||
| 240 | |||
| 241 | rcu_read_lock(); | ||
| 242 | sighand = rcu_dereference(tsk->sighand); | ||
| 243 | if (!sighand) | ||
| 244 | goto out; | ||
| 245 | |||
| 246 | sig = tsk->signal; | ||
| 247 | |||
| 248 | t = tsk; | ||
| 249 | do { | ||
| 250 | times->utime = cputime_add(times->utime, t->utime); | ||
| 251 | times->stime = cputime_add(times->stime, t->stime); | ||
| 252 | times->sum_exec_runtime += t->se.sum_exec_runtime; | ||
| 253 | |||
| 254 | t = next_thread(t); | ||
| 255 | } while (t != tsk); | ||
| 256 | |||
| 257 | times->utime = cputime_add(times->utime, sig->utime); | ||
| 258 | times->stime = cputime_add(times->stime, sig->stime); | ||
| 259 | times->sum_exec_runtime += sig->sum_sched_runtime; | ||
| 260 | out: | ||
| 261 | rcu_read_unlock(); | ||
| 262 | } | ||
| 263 | |||
| 264 | static void update_gt_cputime(struct task_cputime *a, struct task_cputime *b) | ||
| 265 | { | ||
| 266 | if (cputime_gt(b->utime, a->utime)) | ||
| 267 | a->utime = b->utime; | ||
| 268 | |||
| 269 | if (cputime_gt(b->stime, a->stime)) | ||
| 270 | a->stime = b->stime; | ||
| 271 | |||
| 272 | if (b->sum_exec_runtime > a->sum_exec_runtime) | ||
| 273 | a->sum_exec_runtime = b->sum_exec_runtime; | ||
| 274 | } | ||
| 275 | |||
| 276 | void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times) | ||
| 277 | { | ||
| 278 | struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; | ||
| 279 | struct task_cputime sum; | ||
| 280 | unsigned long flags; | ||
| 281 | |||
| 282 | spin_lock_irqsave(&cputimer->lock, flags); | ||
| 283 | if (!cputimer->running) { | ||
| 284 | cputimer->running = 1; | ||
| 285 | /* | ||
| 286 | * The POSIX timer interface allows for absolute time expiry | ||
| 287 | * values through the TIMER_ABSTIME flag, therefore we have | ||
| 288 | * to synchronize the timer to the clock every time we start | ||
| 289 | * it. | ||
| 290 | */ | ||
| 291 | thread_group_cputime(tsk, &sum); | ||
| 292 | update_gt_cputime(&cputimer->cputime, &sum); | ||
| 293 | } | ||
| 294 | *times = cputimer->cputime; | ||
| 295 | spin_unlock_irqrestore(&cputimer->lock, flags); | ||
| 296 | } | ||
| 297 | |||
| 233 | /* | 298 | /* |
| 234 | * Sample a process (thread group) clock for the given group_leader task. | 299 | * Sample a process (thread group) clock for the given group_leader task. |
| 235 | * Must be called with tasklist_lock held for reading. | 300 | * Must be called with tasklist_lock held for reading. |
| @@ -457,7 +522,7 @@ void posix_cpu_timers_exit_group(struct task_struct *tsk) | |||
| 457 | { | 522 | { |
| 458 | struct task_cputime cputime; | 523 | struct task_cputime cputime; |
| 459 | 524 | ||
| 460 | thread_group_cputime(tsk, &cputime); | 525 | thread_group_cputimer(tsk, &cputime); |
| 461 | cleanup_timers(tsk->signal->cpu_timers, | 526 | cleanup_timers(tsk->signal->cpu_timers, |
| 462 | cputime.utime, cputime.stime, cputime.sum_exec_runtime); | 527 | cputime.utime, cputime.stime, cputime.sum_exec_runtime); |
| 463 | } | 528 | } |
| @@ -964,6 +1029,19 @@ static void check_thread_timers(struct task_struct *tsk, | |||
| 964 | } | 1029 | } |
| 965 | } | 1030 | } |
| 966 | 1031 | ||
| 1032 | static void stop_process_timers(struct task_struct *tsk) | ||
| 1033 | { | ||
| 1034 | struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; | ||
| 1035 | unsigned long flags; | ||
| 1036 | |||
| 1037 | if (!cputimer->running) | ||
| 1038 | return; | ||
| 1039 | |||
| 1040 | spin_lock_irqsave(&cputimer->lock, flags); | ||
| 1041 | cputimer->running = 0; | ||
| 1042 | spin_unlock_irqrestore(&cputimer->lock, flags); | ||
| 1043 | } | ||
| 1044 | |||
| 967 | /* | 1045 | /* |
| 968 | * Check for any per-thread CPU timers that have fired and move them | 1046 | * Check for any per-thread CPU timers that have fired and move them |
| 969 | * off the tsk->*_timers list onto the firing list. Per-thread timers | 1047 | * off the tsk->*_timers list onto the firing list. Per-thread timers |
| @@ -987,13 +1065,15 @@ static void check_process_timers(struct task_struct *tsk, | |||
| 987 | sig->rlim[RLIMIT_CPU].rlim_cur == RLIM_INFINITY && | 1065 | sig->rlim[RLIMIT_CPU].rlim_cur == RLIM_INFINITY && |
| 988 | list_empty(&timers[CPUCLOCK_VIRT]) && | 1066 | list_empty(&timers[CPUCLOCK_VIRT]) && |
| 989 | cputime_eq(sig->it_virt_expires, cputime_zero) && | 1067 | cputime_eq(sig->it_virt_expires, cputime_zero) && |
| 990 | list_empty(&timers[CPUCLOCK_SCHED])) | 1068 | list_empty(&timers[CPUCLOCK_SCHED])) { |
| 1069 | stop_process_timers(tsk); | ||
| 991 | return; | 1070 | return; |
| 1071 | } | ||
| 992 | 1072 | ||
| 993 | /* | 1073 | /* |
| 994 | * Collect the current process totals. | 1074 | * Collect the current process totals. |
| 995 | */ | 1075 | */ |
| 996 | thread_group_cputime(tsk, &cputime); | 1076 | thread_group_cputimer(tsk, &cputime); |
| 997 | utime = cputime.utime; | 1077 | utime = cputime.utime; |
| 998 | ptime = cputime_add(utime, cputime.stime); | 1078 | ptime = cputime_add(utime, cputime.stime); |
| 999 | sum_sched_runtime = cputime.sum_exec_runtime; | 1079 | sum_sched_runtime = cputime.sum_exec_runtime; |
| @@ -1259,7 +1339,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk) | |||
| 1259 | if (!task_cputime_zero(&sig->cputime_expires)) { | 1339 | if (!task_cputime_zero(&sig->cputime_expires)) { |
| 1260 | struct task_cputime group_sample; | 1340 | struct task_cputime group_sample; |
| 1261 | 1341 | ||
| 1262 | thread_group_cputime(tsk, &group_sample); | 1342 | thread_group_cputimer(tsk, &group_sample); |
| 1263 | if (task_cputime_expired(&group_sample, &sig->cputime_expires)) | 1343 | if (task_cputime_expired(&group_sample, &sig->cputime_expires)) |
| 1264 | return 1; | 1344 | return 1; |
| 1265 | } | 1345 | } |
| @@ -1329,6 +1409,33 @@ void run_posix_cpu_timers(struct task_struct *tsk) | |||
| 1329 | } | 1409 | } |
| 1330 | 1410 | ||
| 1331 | /* | 1411 | /* |
| 1412 | * Sample a process (thread group) timer for the given group_leader task. | ||
| 1413 | * Must be called with tasklist_lock held for reading. | ||
| 1414 | */ | ||
| 1415 | static int cpu_timer_sample_group(const clockid_t which_clock, | ||
| 1416 | struct task_struct *p, | ||
| 1417 | union cpu_time_count *cpu) | ||
| 1418 | { | ||
| 1419 | struct task_cputime cputime; | ||
| 1420 | |||
| 1421 | thread_group_cputimer(p, &cputime); | ||
| 1422 | switch (CPUCLOCK_WHICH(which_clock)) { | ||
| 1423 | default: | ||
| 1424 | return -EINVAL; | ||
| 1425 | case CPUCLOCK_PROF: | ||
| 1426 | cpu->cpu = cputime_add(cputime.utime, cputime.stime); | ||
| 1427 | break; | ||
| 1428 | case CPUCLOCK_VIRT: | ||
| 1429 | cpu->cpu = cputime.utime; | ||
| 1430 | break; | ||
| 1431 | case CPUCLOCK_SCHED: | ||
| 1432 | cpu->sched = cputime.sum_exec_runtime + task_delta_exec(p); | ||
| 1433 | break; | ||
| 1434 | } | ||
| 1435 | return 0; | ||
| 1436 | } | ||
| 1437 | |||
| 1438 | /* | ||
| 1332 | * Set one of the process-wide special case CPU timers. | 1439 | * Set one of the process-wide special case CPU timers. |
| 1333 | * The tsk->sighand->siglock must be held by the caller. | 1440 | * The tsk->sighand->siglock must be held by the caller. |
| 1334 | * The *newval argument is relative and we update it to be absolute, *oldval | 1441 | * The *newval argument is relative and we update it to be absolute, *oldval |
| @@ -1341,7 +1448,7 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx, | |||
| 1341 | struct list_head *head; | 1448 | struct list_head *head; |
| 1342 | 1449 | ||
| 1343 | BUG_ON(clock_idx == CPUCLOCK_SCHED); | 1450 | BUG_ON(clock_idx == CPUCLOCK_SCHED); |
| 1344 | cpu_clock_sample_group(clock_idx, tsk, &now); | 1451 | cpu_timer_sample_group(clock_idx, tsk, &now); |
| 1345 | 1452 | ||
| 1346 | if (oldval) { | 1453 | if (oldval) { |
| 1347 | if (!cputime_eq(*oldval, cputime_zero)) { | 1454 | if (!cputime_eq(*oldval, cputime_zero)) { |
diff --git a/kernel/power/disk.c b/kernel/power/disk.c index 45e8541ab7e3..432ee575c9ee 100644 --- a/kernel/power/disk.c +++ b/kernel/power/disk.c | |||
| @@ -71,6 +71,14 @@ void hibernation_set_ops(struct platform_hibernation_ops *ops) | |||
| 71 | mutex_unlock(&pm_mutex); | 71 | mutex_unlock(&pm_mutex); |
| 72 | } | 72 | } |
| 73 | 73 | ||
| 74 | static bool entering_platform_hibernation; | ||
| 75 | |||
| 76 | bool system_entering_hibernation(void) | ||
| 77 | { | ||
| 78 | return entering_platform_hibernation; | ||
| 79 | } | ||
| 80 | EXPORT_SYMBOL(system_entering_hibernation); | ||
| 81 | |||
| 74 | #ifdef CONFIG_PM_DEBUG | 82 | #ifdef CONFIG_PM_DEBUG |
| 75 | static void hibernation_debug_sleep(void) | 83 | static void hibernation_debug_sleep(void) |
| 76 | { | 84 | { |
| @@ -411,6 +419,7 @@ int hibernation_platform_enter(void) | |||
| 411 | if (error) | 419 | if (error) |
| 412 | goto Close; | 420 | goto Close; |
| 413 | 421 | ||
| 422 | entering_platform_hibernation = true; | ||
| 414 | suspend_console(); | 423 | suspend_console(); |
| 415 | error = device_suspend(PMSG_HIBERNATE); | 424 | error = device_suspend(PMSG_HIBERNATE); |
| 416 | if (error) { | 425 | if (error) { |
| @@ -445,6 +454,7 @@ int hibernation_platform_enter(void) | |||
| 445 | Finish: | 454 | Finish: |
| 446 | hibernation_ops->finish(); | 455 | hibernation_ops->finish(); |
| 447 | Resume_devices: | 456 | Resume_devices: |
| 457 | entering_platform_hibernation = false; | ||
| 448 | device_resume(PMSG_RESTORE); | 458 | device_resume(PMSG_RESTORE); |
| 449 | resume_console(); | 459 | resume_console(); |
| 450 | Close: | 460 | Close: |
diff --git a/kernel/power/main.c b/kernel/power/main.c index 239988873971..b4d219016b6c 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c | |||
| @@ -57,16 +57,6 @@ int pm_notifier_call_chain(unsigned long val) | |||
| 57 | #ifdef CONFIG_PM_DEBUG | 57 | #ifdef CONFIG_PM_DEBUG |
| 58 | int pm_test_level = TEST_NONE; | 58 | int pm_test_level = TEST_NONE; |
| 59 | 59 | ||
| 60 | static int suspend_test(int level) | ||
| 61 | { | ||
| 62 | if (pm_test_level == level) { | ||
| 63 | printk(KERN_INFO "suspend debug: Waiting for 5 seconds.\n"); | ||
| 64 | mdelay(5000); | ||
| 65 | return 1; | ||
| 66 | } | ||
| 67 | return 0; | ||
| 68 | } | ||
| 69 | |||
| 70 | static const char * const pm_tests[__TEST_AFTER_LAST] = { | 60 | static const char * const pm_tests[__TEST_AFTER_LAST] = { |
| 71 | [TEST_NONE] = "none", | 61 | [TEST_NONE] = "none", |
| 72 | [TEST_CORE] = "core", | 62 | [TEST_CORE] = "core", |
| @@ -125,14 +115,24 @@ static ssize_t pm_test_store(struct kobject *kobj, struct kobj_attribute *attr, | |||
| 125 | } | 115 | } |
| 126 | 116 | ||
| 127 | power_attr(pm_test); | 117 | power_attr(pm_test); |
| 128 | #else /* !CONFIG_PM_DEBUG */ | 118 | #endif /* CONFIG_PM_DEBUG */ |
| 129 | static inline int suspend_test(int level) { return 0; } | ||
| 130 | #endif /* !CONFIG_PM_DEBUG */ | ||
| 131 | 119 | ||
| 132 | #endif /* CONFIG_PM_SLEEP */ | 120 | #endif /* CONFIG_PM_SLEEP */ |
| 133 | 121 | ||
| 134 | #ifdef CONFIG_SUSPEND | 122 | #ifdef CONFIG_SUSPEND |
| 135 | 123 | ||
| 124 | static int suspend_test(int level) | ||
| 125 | { | ||
| 126 | #ifdef CONFIG_PM_DEBUG | ||
| 127 | if (pm_test_level == level) { | ||
| 128 | printk(KERN_INFO "suspend debug: Waiting for 5 seconds.\n"); | ||
| 129 | mdelay(5000); | ||
| 130 | return 1; | ||
| 131 | } | ||
| 132 | #endif /* !CONFIG_PM_DEBUG */ | ||
| 133 | return 0; | ||
| 134 | } | ||
| 135 | |||
| 136 | #ifdef CONFIG_PM_TEST_SUSPEND | 136 | #ifdef CONFIG_PM_TEST_SUSPEND |
| 137 | 137 | ||
| 138 | /* | 138 | /* |
diff --git a/kernel/profile.c b/kernel/profile.c index 784933acf5b8..7724e0409bae 100644 --- a/kernel/profile.c +++ b/kernel/profile.c | |||
| @@ -114,12 +114,15 @@ int __ref profile_init(void) | |||
| 114 | if (!slab_is_available()) { | 114 | if (!slab_is_available()) { |
| 115 | prof_buffer = alloc_bootmem(buffer_bytes); | 115 | prof_buffer = alloc_bootmem(buffer_bytes); |
| 116 | alloc_bootmem_cpumask_var(&prof_cpu_mask); | 116 | alloc_bootmem_cpumask_var(&prof_cpu_mask); |
| 117 | cpumask_copy(prof_cpu_mask, cpu_possible_mask); | ||
| 117 | return 0; | 118 | return 0; |
| 118 | } | 119 | } |
| 119 | 120 | ||
| 120 | if (!alloc_cpumask_var(&prof_cpu_mask, GFP_KERNEL)) | 121 | if (!alloc_cpumask_var(&prof_cpu_mask, GFP_KERNEL)) |
| 121 | return -ENOMEM; | 122 | return -ENOMEM; |
| 122 | 123 | ||
| 124 | cpumask_copy(prof_cpu_mask, cpu_possible_mask); | ||
| 125 | |||
| 123 | prof_buffer = kzalloc(buffer_bytes, GFP_KERNEL); | 126 | prof_buffer = kzalloc(buffer_bytes, GFP_KERNEL); |
| 124 | if (prof_buffer) | 127 | if (prof_buffer) |
| 125 | return 0; | 128 | return 0; |
diff --git a/kernel/sched.c b/kernel/sched.c index 52bbf1c842a8..61245b8d0f16 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
| @@ -3880,19 +3880,24 @@ int select_nohz_load_balancer(int stop_tick) | |||
| 3880 | int cpu = smp_processor_id(); | 3880 | int cpu = smp_processor_id(); |
| 3881 | 3881 | ||
| 3882 | if (stop_tick) { | 3882 | if (stop_tick) { |
| 3883 | cpumask_set_cpu(cpu, nohz.cpu_mask); | ||
| 3884 | cpu_rq(cpu)->in_nohz_recently = 1; | 3883 | cpu_rq(cpu)->in_nohz_recently = 1; |
| 3885 | 3884 | ||
| 3886 | /* | 3885 | if (!cpu_active(cpu)) { |
| 3887 | * If we are going offline and still the leader, give up! | 3886 | if (atomic_read(&nohz.load_balancer) != cpu) |
| 3888 | */ | 3887 | return 0; |
| 3889 | if (!cpu_active(cpu) && | 3888 | |
| 3890 | atomic_read(&nohz.load_balancer) == cpu) { | 3889 | /* |
| 3890 | * If we are going offline and still the leader, | ||
| 3891 | * give up! | ||
| 3892 | */ | ||
| 3891 | if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu) | 3893 | if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu) |
| 3892 | BUG(); | 3894 | BUG(); |
| 3895 | |||
| 3893 | return 0; | 3896 | return 0; |
| 3894 | } | 3897 | } |
| 3895 | 3898 | ||
| 3899 | cpumask_set_cpu(cpu, nohz.cpu_mask); | ||
| 3900 | |||
| 3896 | /* time for ilb owner also to sleep */ | 3901 | /* time for ilb owner also to sleep */ |
| 3897 | if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) { | 3902 | if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) { |
| 3898 | if (atomic_read(&nohz.load_balancer) == cpu) | 3903 | if (atomic_read(&nohz.load_balancer) == cpu) |
| @@ -4687,8 +4692,8 @@ EXPORT_SYMBOL(default_wake_function); | |||
| 4687 | * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns | 4692 | * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns |
| 4688 | * zero in this (rare) case, and we handle it by continuing to scan the queue. | 4693 | * zero in this (rare) case, and we handle it by continuing to scan the queue. |
| 4689 | */ | 4694 | */ |
| 4690 | static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, | 4695 | void __wake_up_common(wait_queue_head_t *q, unsigned int mode, |
| 4691 | int nr_exclusive, int sync, void *key) | 4696 | int nr_exclusive, int sync, void *key) |
| 4692 | { | 4697 | { |
| 4693 | wait_queue_t *curr, *next; | 4698 | wait_queue_t *curr, *next; |
| 4694 | 4699 | ||
| @@ -5939,12 +5944,7 @@ void sched_show_task(struct task_struct *p) | |||
| 5939 | printk(KERN_CONT " %016lx ", thread_saved_pc(p)); | 5944 | printk(KERN_CONT " %016lx ", thread_saved_pc(p)); |
| 5940 | #endif | 5945 | #endif |
| 5941 | #ifdef CONFIG_DEBUG_STACK_USAGE | 5946 | #ifdef CONFIG_DEBUG_STACK_USAGE |
| 5942 | { | 5947 | free = stack_not_used(p); |
| 5943 | unsigned long *n = end_of_stack(p); | ||
| 5944 | while (!*n) | ||
| 5945 | n++; | ||
| 5946 | free = (unsigned long)n - (unsigned long)end_of_stack(p); | ||
| 5947 | } | ||
| 5948 | #endif | 5948 | #endif |
| 5949 | printk(KERN_CONT "%5lu %5d %6d\n", free, | 5949 | printk(KERN_CONT "%5lu %5d %6d\n", free, |
| 5950 | task_pid_nr(p), task_pid_nr(p->real_parent)); | 5950 | task_pid_nr(p), task_pid_nr(p->real_parent)); |
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 5cc1c162044f..0566f2a03c42 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c | |||
| @@ -719,7 +719,7 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int wakeup) | |||
| 719 | __enqueue_entity(cfs_rq, se); | 719 | __enqueue_entity(cfs_rq, se); |
| 720 | } | 720 | } |
| 721 | 721 | ||
| 722 | static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) | 722 | static void __clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 723 | { | 723 | { |
| 724 | if (cfs_rq->last == se) | 724 | if (cfs_rq->last == se) |
| 725 | cfs_rq->last = NULL; | 725 | cfs_rq->last = NULL; |
| @@ -728,6 +728,12 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) | |||
| 728 | cfs_rq->next = NULL; | 728 | cfs_rq->next = NULL; |
| 729 | } | 729 | } |
| 730 | 730 | ||
| 731 | static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) | ||
| 732 | { | ||
| 733 | for_each_sched_entity(se) | ||
| 734 | __clear_buddies(cfs_rq_of(se), se); | ||
| 735 | } | ||
| 736 | |||
| 731 | static void | 737 | static void |
| 732 | dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) | 738 | dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) |
| 733 | { | 739 | { |
| @@ -768,8 +774,14 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) | |||
| 768 | 774 | ||
| 769 | ideal_runtime = sched_slice(cfs_rq, curr); | 775 | ideal_runtime = sched_slice(cfs_rq, curr); |
| 770 | delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; | 776 | delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; |
| 771 | if (delta_exec > ideal_runtime) | 777 | if (delta_exec > ideal_runtime) { |
| 772 | resched_task(rq_of(cfs_rq)->curr); | 778 | resched_task(rq_of(cfs_rq)->curr); |
| 779 | /* | ||
| 780 | * The current task ran long enough, ensure it doesn't get | ||
| 781 | * re-elected due to buddy favours. | ||
| 782 | */ | ||
| 783 | clear_buddies(cfs_rq, curr); | ||
| 784 | } | ||
| 773 | } | 785 | } |
| 774 | 786 | ||
| 775 | static void | 787 | static void |
| @@ -1452,6 +1464,11 @@ static struct task_struct *pick_next_task_fair(struct rq *rq) | |||
| 1452 | 1464 | ||
| 1453 | do { | 1465 | do { |
| 1454 | se = pick_next_entity(cfs_rq); | 1466 | se = pick_next_entity(cfs_rq); |
| 1467 | /* | ||
| 1468 | * If se was a buddy, clear it so that it will have to earn | ||
| 1469 | * the favour again. | ||
| 1470 | */ | ||
| 1471 | __clear_buddies(cfs_rq, se); | ||
| 1455 | set_next_entity(cfs_rq, se); | 1472 | set_next_entity(cfs_rq, se); |
| 1456 | cfs_rq = group_cfs_rq(se); | 1473 | cfs_rq = group_cfs_rq(se); |
| 1457 | } while (cfs_rq); | 1474 | } while (cfs_rq); |
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 954e1a81b796..da932f4c8524 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c | |||
| @@ -960,16 +960,17 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu) | |||
| 960 | 960 | ||
| 961 | static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask); | 961 | static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask); |
| 962 | 962 | ||
| 963 | static inline int pick_optimal_cpu(int this_cpu, cpumask_t *mask) | 963 | static inline int pick_optimal_cpu(int this_cpu, |
| 964 | const struct cpumask *mask) | ||
| 964 | { | 965 | { |
| 965 | int first; | 966 | int first; |
| 966 | 967 | ||
| 967 | /* "this_cpu" is cheaper to preempt than a remote processor */ | 968 | /* "this_cpu" is cheaper to preempt than a remote processor */ |
| 968 | if ((this_cpu != -1) && cpu_isset(this_cpu, *mask)) | 969 | if ((this_cpu != -1) && cpumask_test_cpu(this_cpu, mask)) |
| 969 | return this_cpu; | 970 | return this_cpu; |
| 970 | 971 | ||
| 971 | first = first_cpu(*mask); | 972 | first = cpumask_first(mask); |
| 972 | if (first != NR_CPUS) | 973 | if (first < nr_cpu_ids) |
| 973 | return first; | 974 | return first; |
| 974 | 975 | ||
| 975 | return -1; | 976 | return -1; |
| @@ -981,6 +982,7 @@ static int find_lowest_rq(struct task_struct *task) | |||
| 981 | struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask); | 982 | struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask); |
| 982 | int this_cpu = smp_processor_id(); | 983 | int this_cpu = smp_processor_id(); |
| 983 | int cpu = task_cpu(task); | 984 | int cpu = task_cpu(task); |
| 985 | cpumask_var_t domain_mask; | ||
| 984 | 986 | ||
| 985 | if (task->rt.nr_cpus_allowed == 1) | 987 | if (task->rt.nr_cpus_allowed == 1) |
| 986 | return -1; /* No other targets possible */ | 988 | return -1; /* No other targets possible */ |
| @@ -1013,19 +1015,25 @@ static int find_lowest_rq(struct task_struct *task) | |||
| 1013 | if (this_cpu == cpu) | 1015 | if (this_cpu == cpu) |
| 1014 | this_cpu = -1; /* Skip this_cpu opt if the same */ | 1016 | this_cpu = -1; /* Skip this_cpu opt if the same */ |
| 1015 | 1017 | ||
| 1016 | for_each_domain(cpu, sd) { | 1018 | if (alloc_cpumask_var(&domain_mask, GFP_ATOMIC)) { |
| 1017 | if (sd->flags & SD_WAKE_AFFINE) { | 1019 | for_each_domain(cpu, sd) { |
| 1018 | cpumask_t domain_mask; | 1020 | if (sd->flags & SD_WAKE_AFFINE) { |
| 1019 | int best_cpu; | 1021 | int best_cpu; |
| 1020 | 1022 | ||
| 1021 | cpumask_and(&domain_mask, sched_domain_span(sd), | 1023 | cpumask_and(domain_mask, |
| 1022 | lowest_mask); | 1024 | sched_domain_span(sd), |
| 1025 | lowest_mask); | ||
| 1023 | 1026 | ||
| 1024 | best_cpu = pick_optimal_cpu(this_cpu, | 1027 | best_cpu = pick_optimal_cpu(this_cpu, |
| 1025 | &domain_mask); | 1028 | domain_mask); |
| 1026 | if (best_cpu != -1) | 1029 | |
| 1027 | return best_cpu; | 1030 | if (best_cpu != -1) { |
| 1031 | free_cpumask_var(domain_mask); | ||
| 1032 | return best_cpu; | ||
| 1033 | } | ||
| 1034 | } | ||
| 1028 | } | 1035 | } |
| 1036 | free_cpumask_var(domain_mask); | ||
| 1029 | } | 1037 | } |
| 1030 | 1038 | ||
| 1031 | /* | 1039 | /* |
diff --git a/kernel/sched_stats.h b/kernel/sched_stats.h index 8ab0cef8ecab..a8f93dd374e1 100644 --- a/kernel/sched_stats.h +++ b/kernel/sched_stats.h | |||
| @@ -296,19 +296,21 @@ sched_info_switch(struct task_struct *prev, struct task_struct *next) | |||
| 296 | static inline void account_group_user_time(struct task_struct *tsk, | 296 | static inline void account_group_user_time(struct task_struct *tsk, |
| 297 | cputime_t cputime) | 297 | cputime_t cputime) |
| 298 | { | 298 | { |
| 299 | struct task_cputime *times; | 299 | struct thread_group_cputimer *cputimer; |
| 300 | struct signal_struct *sig; | ||
| 301 | 300 | ||
| 302 | /* tsk == current, ensure it is safe to use ->signal */ | 301 | /* tsk == current, ensure it is safe to use ->signal */ |
| 303 | if (unlikely(tsk->exit_state)) | 302 | if (unlikely(tsk->exit_state)) |
| 304 | return; | 303 | return; |
| 305 | 304 | ||
| 306 | sig = tsk->signal; | 305 | cputimer = &tsk->signal->cputimer; |
| 307 | times = &sig->cputime.totals; | ||
| 308 | 306 | ||
| 309 | spin_lock(×->lock); | 307 | if (!cputimer->running) |
| 310 | times->utime = cputime_add(times->utime, cputime); | 308 | return; |
| 311 | spin_unlock(×->lock); | 309 | |
| 310 | spin_lock(&cputimer->lock); | ||
| 311 | cputimer->cputime.utime = | ||
| 312 | cputime_add(cputimer->cputime.utime, cputime); | ||
| 313 | spin_unlock(&cputimer->lock); | ||
| 312 | } | 314 | } |
| 313 | 315 | ||
| 314 | /** | 316 | /** |
| @@ -324,19 +326,21 @@ static inline void account_group_user_time(struct task_struct *tsk, | |||
| 324 | static inline void account_group_system_time(struct task_struct *tsk, | 326 | static inline void account_group_system_time(struct task_struct *tsk, |
| 325 | cputime_t cputime) | 327 | cputime_t cputime) |
| 326 | { | 328 | { |
| 327 | struct task_cputime *times; | 329 | struct thread_group_cputimer *cputimer; |
| 328 | struct signal_struct *sig; | ||
| 329 | 330 | ||
| 330 | /* tsk == current, ensure it is safe to use ->signal */ | 331 | /* tsk == current, ensure it is safe to use ->signal */ |
| 331 | if (unlikely(tsk->exit_state)) | 332 | if (unlikely(tsk->exit_state)) |
| 332 | return; | 333 | return; |
| 333 | 334 | ||
| 334 | sig = tsk->signal; | 335 | cputimer = &tsk->signal->cputimer; |
| 335 | times = &sig->cputime.totals; | 336 | |
| 337 | if (!cputimer->running) | ||
| 338 | return; | ||
| 336 | 339 | ||
| 337 | spin_lock(×->lock); | 340 | spin_lock(&cputimer->lock); |
| 338 | times->stime = cputime_add(times->stime, cputime); | 341 | cputimer->cputime.stime = |
| 339 | spin_unlock(×->lock); | 342 | cputime_add(cputimer->cputime.stime, cputime); |
| 343 | spin_unlock(&cputimer->lock); | ||
| 340 | } | 344 | } |
| 341 | 345 | ||
| 342 | /** | 346 | /** |
| @@ -352,7 +356,7 @@ static inline void account_group_system_time(struct task_struct *tsk, | |||
| 352 | static inline void account_group_exec_runtime(struct task_struct *tsk, | 356 | static inline void account_group_exec_runtime(struct task_struct *tsk, |
| 353 | unsigned long long ns) | 357 | unsigned long long ns) |
| 354 | { | 358 | { |
| 355 | struct task_cputime *times; | 359 | struct thread_group_cputimer *cputimer; |
| 356 | struct signal_struct *sig; | 360 | struct signal_struct *sig; |
| 357 | 361 | ||
| 358 | sig = tsk->signal; | 362 | sig = tsk->signal; |
| @@ -361,9 +365,12 @@ static inline void account_group_exec_runtime(struct task_struct *tsk, | |||
| 361 | if (unlikely(!sig)) | 365 | if (unlikely(!sig)) |
| 362 | return; | 366 | return; |
| 363 | 367 | ||
| 364 | times = &sig->cputime.totals; | 368 | cputimer = &sig->cputimer; |
| 369 | |||
| 370 | if (!cputimer->running) | ||
| 371 | return; | ||
| 365 | 372 | ||
| 366 | spin_lock(×->lock); | 373 | spin_lock(&cputimer->lock); |
| 367 | times->sum_exec_runtime += ns; | 374 | cputimer->cputime.sum_exec_runtime += ns; |
| 368 | spin_unlock(×->lock); | 375 | spin_unlock(&cputimer->lock); |
| 369 | } | 376 | } |
diff --git a/kernel/signal.c b/kernel/signal.c index e73759783dc8..2a74fe87c0dd 100644 --- a/kernel/signal.c +++ b/kernel/signal.c | |||
| @@ -909,7 +909,9 @@ static void print_fatal_signal(struct pt_regs *regs, int signr) | |||
| 909 | } | 909 | } |
| 910 | #endif | 910 | #endif |
| 911 | printk("\n"); | 911 | printk("\n"); |
| 912 | preempt_disable(); | ||
| 912 | show_regs(regs); | 913 | show_regs(regs); |
| 914 | preempt_enable(); | ||
| 913 | } | 915 | } |
| 914 | 916 | ||
| 915 | static int __init setup_print_fatal_signals(char *str) | 917 | static int __init setup_print_fatal_signals(char *str) |
| @@ -1365,7 +1367,6 @@ int do_notify_parent(struct task_struct *tsk, int sig) | |||
| 1365 | struct siginfo info; | 1367 | struct siginfo info; |
| 1366 | unsigned long flags; | 1368 | unsigned long flags; |
| 1367 | struct sighand_struct *psig; | 1369 | struct sighand_struct *psig; |
| 1368 | struct task_cputime cputime; | ||
| 1369 | int ret = sig; | 1370 | int ret = sig; |
| 1370 | 1371 | ||
| 1371 | BUG_ON(sig == -1); | 1372 | BUG_ON(sig == -1); |
| @@ -1395,9 +1396,10 @@ int do_notify_parent(struct task_struct *tsk, int sig) | |||
| 1395 | info.si_uid = __task_cred(tsk)->uid; | 1396 | info.si_uid = __task_cred(tsk)->uid; |
| 1396 | rcu_read_unlock(); | 1397 | rcu_read_unlock(); |
| 1397 | 1398 | ||
| 1398 | thread_group_cputime(tsk, &cputime); | 1399 | info.si_utime = cputime_to_clock_t(cputime_add(tsk->utime, |
| 1399 | info.si_utime = cputime_to_jiffies(cputime.utime); | 1400 | tsk->signal->utime)); |
| 1400 | info.si_stime = cputime_to_jiffies(cputime.stime); | 1401 | info.si_stime = cputime_to_clock_t(cputime_add(tsk->stime, |
| 1402 | tsk->signal->stime)); | ||
| 1401 | 1403 | ||
| 1402 | info.si_status = tsk->exit_code & 0x7f; | 1404 | info.si_status = tsk->exit_code & 0x7f; |
| 1403 | if (tsk->exit_code & 0x80) | 1405 | if (tsk->exit_code & 0x80) |
diff --git a/kernel/smp.c b/kernel/smp.c index 5cfa0e5e3e88..bbedbb7efe32 100644 --- a/kernel/smp.c +++ b/kernel/smp.c | |||
| @@ -18,6 +18,7 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(call_function_lock); | |||
| 18 | enum { | 18 | enum { |
| 19 | CSD_FLAG_WAIT = 0x01, | 19 | CSD_FLAG_WAIT = 0x01, |
| 20 | CSD_FLAG_ALLOC = 0x02, | 20 | CSD_FLAG_ALLOC = 0x02, |
| 21 | CSD_FLAG_LOCK = 0x04, | ||
| 21 | }; | 22 | }; |
| 22 | 23 | ||
| 23 | struct call_function_data { | 24 | struct call_function_data { |
| @@ -186,6 +187,9 @@ void generic_smp_call_function_single_interrupt(void) | |||
| 186 | if (data_flags & CSD_FLAG_WAIT) { | 187 | if (data_flags & CSD_FLAG_WAIT) { |
| 187 | smp_wmb(); | 188 | smp_wmb(); |
| 188 | data->flags &= ~CSD_FLAG_WAIT; | 189 | data->flags &= ~CSD_FLAG_WAIT; |
| 190 | } else if (data_flags & CSD_FLAG_LOCK) { | ||
| 191 | smp_wmb(); | ||
| 192 | data->flags &= ~CSD_FLAG_LOCK; | ||
| 189 | } else if (data_flags & CSD_FLAG_ALLOC) | 193 | } else if (data_flags & CSD_FLAG_ALLOC) |
| 190 | kfree(data); | 194 | kfree(data); |
| 191 | } | 195 | } |
| @@ -196,6 +200,8 @@ void generic_smp_call_function_single_interrupt(void) | |||
| 196 | } | 200 | } |
| 197 | } | 201 | } |
| 198 | 202 | ||
| 203 | static DEFINE_PER_CPU(struct call_single_data, csd_data); | ||
| 204 | |||
| 199 | /* | 205 | /* |
| 200 | * smp_call_function_single - Run a function on a specific CPU | 206 | * smp_call_function_single - Run a function on a specific CPU |
| 201 | * @func: The function to run. This must be fast and non-blocking. | 207 | * @func: The function to run. This must be fast and non-blocking. |
| @@ -224,14 +230,38 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info, | |||
| 224 | func(info); | 230 | func(info); |
| 225 | local_irq_restore(flags); | 231 | local_irq_restore(flags); |
| 226 | } else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) { | 232 | } else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) { |
| 227 | struct call_single_data *data = NULL; | 233 | struct call_single_data *data; |
| 228 | 234 | ||
| 229 | if (!wait) { | 235 | if (!wait) { |
| 236 | /* | ||
| 237 | * We are calling a function on a single CPU | ||
| 238 | * and we are not going to wait for it to finish. | ||
| 239 | * We first try to allocate the data, but if we | ||
| 240 | * fail, we fall back to use a per cpu data to pass | ||
| 241 | * the information to that CPU. Since all callers | ||
| 242 | * of this code will use the same data, we must | ||
| 243 | * synchronize the callers to prevent a new caller | ||
| 244 | * from corrupting the data before the callee | ||
| 245 | * can access it. | ||
| 246 | * | ||
| 247 | * The CSD_FLAG_LOCK is used to let us know when | ||
| 248 | * the IPI handler is done with the data. | ||
| 249 | * The first caller will set it, and the callee | ||
| 250 | * will clear it. The next caller must wait for | ||
| 251 | * it to clear before we set it again. This | ||
| 252 | * will make sure the callee is done with the | ||
| 253 | * data before a new caller will use it. | ||
| 254 | */ | ||
| 230 | data = kmalloc(sizeof(*data), GFP_ATOMIC); | 255 | data = kmalloc(sizeof(*data), GFP_ATOMIC); |
| 231 | if (data) | 256 | if (data) |
| 232 | data->flags = CSD_FLAG_ALLOC; | 257 | data->flags = CSD_FLAG_ALLOC; |
| 233 | } | 258 | else { |
| 234 | if (!data) { | 259 | data = &per_cpu(csd_data, me); |
| 260 | while (data->flags & CSD_FLAG_LOCK) | ||
| 261 | cpu_relax(); | ||
| 262 | data->flags = CSD_FLAG_LOCK; | ||
| 263 | } | ||
| 264 | } else { | ||
| 235 | data = &d; | 265 | data = &d; |
| 236 | data->flags = CSD_FLAG_WAIT; | 266 | data->flags = CSD_FLAG_WAIT; |
| 237 | } | 267 | } |
diff --git a/kernel/softirq.c b/kernel/softirq.c index bdbe9de9cd8d..0365b4899a3d 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c | |||
| @@ -795,6 +795,11 @@ int __init __weak early_irq_init(void) | |||
| 795 | return 0; | 795 | return 0; |
| 796 | } | 796 | } |
| 797 | 797 | ||
| 798 | int __init __weak arch_probe_nr_irqs(void) | ||
| 799 | { | ||
| 800 | return 0; | ||
| 801 | } | ||
| 802 | |||
| 798 | int __init __weak arch_early_irq_init(void) | 803 | int __init __weak arch_early_irq_init(void) |
| 799 | { | 804 | { |
| 800 | return 0; | 805 | return 0; |
diff --git a/kernel/sys.c b/kernel/sys.c index e7dc0e10a485..f145c415bc16 100644 --- a/kernel/sys.c +++ b/kernel/sys.c | |||
| @@ -1525,22 +1525,14 @@ SYSCALL_DEFINE2(setrlimit, unsigned int, resource, struct rlimit __user *, rlim) | |||
| 1525 | return -EINVAL; | 1525 | return -EINVAL; |
| 1526 | if (copy_from_user(&new_rlim, rlim, sizeof(*rlim))) | 1526 | if (copy_from_user(&new_rlim, rlim, sizeof(*rlim))) |
| 1527 | return -EFAULT; | 1527 | return -EFAULT; |
| 1528 | if (new_rlim.rlim_cur > new_rlim.rlim_max) | ||
| 1529 | return -EINVAL; | ||
| 1528 | old_rlim = current->signal->rlim + resource; | 1530 | old_rlim = current->signal->rlim + resource; |
| 1529 | if ((new_rlim.rlim_max > old_rlim->rlim_max) && | 1531 | if ((new_rlim.rlim_max > old_rlim->rlim_max) && |
| 1530 | !capable(CAP_SYS_RESOURCE)) | 1532 | !capable(CAP_SYS_RESOURCE)) |
| 1531 | return -EPERM; | 1533 | return -EPERM; |
| 1532 | 1534 | if (resource == RLIMIT_NOFILE && new_rlim.rlim_max > sysctl_nr_open) | |
| 1533 | if (resource == RLIMIT_NOFILE) { | 1535 | return -EPERM; |
| 1534 | if (new_rlim.rlim_max == RLIM_INFINITY) | ||
| 1535 | new_rlim.rlim_max = sysctl_nr_open; | ||
| 1536 | if (new_rlim.rlim_cur == RLIM_INFINITY) | ||
| 1537 | new_rlim.rlim_cur = sysctl_nr_open; | ||
| 1538 | if (new_rlim.rlim_max > sysctl_nr_open) | ||
| 1539 | return -EPERM; | ||
| 1540 | } | ||
| 1541 | |||
| 1542 | if (new_rlim.rlim_cur > new_rlim.rlim_max) | ||
| 1543 | return -EINVAL; | ||
| 1544 | 1536 | ||
| 1545 | retval = security_task_setrlimit(resource, &new_rlim); | 1537 | retval = security_task_setrlimit(resource, &new_rlim); |
| 1546 | if (retval) | 1538 | if (retval) |
diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 790f9d785663..c5ef44ff850f 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c | |||
| @@ -101,6 +101,7 @@ static int two = 2; | |||
| 101 | 101 | ||
| 102 | static int zero; | 102 | static int zero; |
| 103 | static int one = 1; | 103 | static int one = 1; |
| 104 | static unsigned long one_ul = 1; | ||
| 104 | static int one_hundred = 100; | 105 | static int one_hundred = 100; |
| 105 | 106 | ||
| 106 | /* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */ | 107 | /* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */ |
| @@ -974,7 +975,7 @@ static struct ctl_table vm_table[] = { | |||
| 974 | .mode = 0644, | 975 | .mode = 0644, |
| 975 | .proc_handler = &dirty_background_bytes_handler, | 976 | .proc_handler = &dirty_background_bytes_handler, |
| 976 | .strategy = &sysctl_intvec, | 977 | .strategy = &sysctl_intvec, |
| 977 | .extra1 = &one, | 978 | .extra1 = &one_ul, |
| 978 | }, | 979 | }, |
| 979 | { | 980 | { |
| 980 | .ctl_name = VM_DIRTY_RATIO, | 981 | .ctl_name = VM_DIRTY_RATIO, |
| @@ -995,7 +996,7 @@ static struct ctl_table vm_table[] = { | |||
| 995 | .mode = 0644, | 996 | .mode = 0644, |
| 996 | .proc_handler = &dirty_bytes_handler, | 997 | .proc_handler = &dirty_bytes_handler, |
| 997 | .strategy = &sysctl_intvec, | 998 | .strategy = &sysctl_intvec, |
| 998 | .extra1 = &one, | 999 | .extra1 = &one_ul, |
| 999 | }, | 1000 | }, |
| 1000 | { | 1001 | { |
| 1001 | .procname = "dirty_writeback_centisecs", | 1002 | .procname = "dirty_writeback_centisecs", |
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c index 63e05d423a09..21a5ca849514 100644 --- a/kernel/time/tick-common.c +++ b/kernel/time/tick-common.c | |||
| @@ -274,6 +274,21 @@ out_bc: | |||
| 274 | } | 274 | } |
| 275 | 275 | ||
| 276 | /* | 276 | /* |
| 277 | * Transfer the do_timer job away from a dying cpu. | ||
| 278 | * | ||
| 279 | * Called with interrupts disabled. | ||
| 280 | */ | ||
| 281 | static void tick_handover_do_timer(int *cpup) | ||
| 282 | { | ||
| 283 | if (*cpup == tick_do_timer_cpu) { | ||
| 284 | int cpu = cpumask_first(cpu_online_mask); | ||
| 285 | |||
| 286 | tick_do_timer_cpu = (cpu < nr_cpu_ids) ? cpu : | ||
| 287 | TICK_DO_TIMER_NONE; | ||
| 288 | } | ||
| 289 | } | ||
| 290 | |||
| 291 | /* | ||
| 277 | * Shutdown an event device on a given cpu: | 292 | * Shutdown an event device on a given cpu: |
| 278 | * | 293 | * |
| 279 | * This is called on a life CPU, when a CPU is dead. So we cannot | 294 | * This is called on a life CPU, when a CPU is dead. So we cannot |
| @@ -297,13 +312,6 @@ static void tick_shutdown(unsigned int *cpup) | |||
| 297 | clockevents_exchange_device(dev, NULL); | 312 | clockevents_exchange_device(dev, NULL); |
| 298 | td->evtdev = NULL; | 313 | td->evtdev = NULL; |
| 299 | } | 314 | } |
| 300 | /* Transfer the do_timer job away from this cpu */ | ||
| 301 | if (*cpup == tick_do_timer_cpu) { | ||
| 302 | int cpu = cpumask_first(cpu_online_mask); | ||
| 303 | |||
| 304 | tick_do_timer_cpu = (cpu < nr_cpu_ids) ? cpu : | ||
| 305 | TICK_DO_TIMER_NONE; | ||
| 306 | } | ||
| 307 | spin_unlock_irqrestore(&tick_device_lock, flags); | 315 | spin_unlock_irqrestore(&tick_device_lock, flags); |
| 308 | } | 316 | } |
| 309 | 317 | ||
| @@ -357,6 +365,10 @@ static int tick_notify(struct notifier_block *nb, unsigned long reason, | |||
| 357 | tick_broadcast_oneshot_control(reason); | 365 | tick_broadcast_oneshot_control(reason); |
| 358 | break; | 366 | break; |
| 359 | 367 | ||
| 368 | case CLOCK_EVT_NOTIFY_CPU_DYING: | ||
| 369 | tick_handover_do_timer(dev); | ||
| 370 | break; | ||
| 371 | |||
| 360 | case CLOCK_EVT_NOTIFY_CPU_DEAD: | 372 | case CLOCK_EVT_NOTIFY_CPU_DEAD: |
| 361 | tick_shutdown_broadcast_oneshot(dev); | 373 | tick_shutdown_broadcast_oneshot(dev); |
| 362 | tick_shutdown_broadcast(dev); | 374 | tick_shutdown_broadcast(dev); |
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 2f32969c09df..9a236ffe2aa4 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
| @@ -17,6 +17,7 @@ | |||
| 17 | #include <linux/clocksource.h> | 17 | #include <linux/clocksource.h> |
| 18 | #include <linux/kallsyms.h> | 18 | #include <linux/kallsyms.h> |
| 19 | #include <linux/seq_file.h> | 19 | #include <linux/seq_file.h> |
| 20 | #include <linux/suspend.h> | ||
| 20 | #include <linux/debugfs.h> | 21 | #include <linux/debugfs.h> |
| 21 | #include <linux/hardirq.h> | 22 | #include <linux/hardirq.h> |
| 22 | #include <linux/kthread.h> | 23 | #include <linux/kthread.h> |
| @@ -1736,9 +1737,12 @@ static void clear_ftrace_pid(struct pid *pid) | |||
| 1736 | { | 1737 | { |
| 1737 | struct task_struct *p; | 1738 | struct task_struct *p; |
| 1738 | 1739 | ||
| 1740 | rcu_read_lock(); | ||
| 1739 | do_each_pid_task(pid, PIDTYPE_PID, p) { | 1741 | do_each_pid_task(pid, PIDTYPE_PID, p) { |
| 1740 | clear_tsk_trace_trace(p); | 1742 | clear_tsk_trace_trace(p); |
| 1741 | } while_each_pid_task(pid, PIDTYPE_PID, p); | 1743 | } while_each_pid_task(pid, PIDTYPE_PID, p); |
| 1744 | rcu_read_unlock(); | ||
| 1745 | |||
| 1742 | put_pid(pid); | 1746 | put_pid(pid); |
| 1743 | } | 1747 | } |
| 1744 | 1748 | ||
| @@ -1746,9 +1750,11 @@ static void set_ftrace_pid(struct pid *pid) | |||
| 1746 | { | 1750 | { |
| 1747 | struct task_struct *p; | 1751 | struct task_struct *p; |
| 1748 | 1752 | ||
| 1753 | rcu_read_lock(); | ||
| 1749 | do_each_pid_task(pid, PIDTYPE_PID, p) { | 1754 | do_each_pid_task(pid, PIDTYPE_PID, p) { |
| 1750 | set_tsk_trace_trace(p); | 1755 | set_tsk_trace_trace(p); |
| 1751 | } while_each_pid_task(pid, PIDTYPE_PID, p); | 1756 | } while_each_pid_task(pid, PIDTYPE_PID, p); |
| 1757 | rcu_read_unlock(); | ||
| 1752 | } | 1758 | } |
| 1753 | 1759 | ||
| 1754 | static void clear_ftrace_pid_task(struct pid **pid) | 1760 | static void clear_ftrace_pid_task(struct pid **pid) |
| @@ -1965,6 +1971,7 @@ ftrace_enable_sysctl(struct ctl_table *table, int write, | |||
| 1965 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 1971 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 1966 | 1972 | ||
| 1967 | static atomic_t ftrace_graph_active; | 1973 | static atomic_t ftrace_graph_active; |
| 1974 | static struct notifier_block ftrace_suspend_notifier; | ||
| 1968 | 1975 | ||
| 1969 | int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) | 1976 | int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) |
| 1970 | { | 1977 | { |
| @@ -2043,6 +2050,27 @@ static int start_graph_tracing(void) | |||
| 2043 | return ret; | 2050 | return ret; |
| 2044 | } | 2051 | } |
| 2045 | 2052 | ||
| 2053 | /* | ||
| 2054 | * Hibernation protection. | ||
| 2055 | * The state of the current task is too much unstable during | ||
| 2056 | * suspend/restore to disk. We want to protect against that. | ||
| 2057 | */ | ||
| 2058 | static int | ||
| 2059 | ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state, | ||
| 2060 | void *unused) | ||
| 2061 | { | ||
| 2062 | switch (state) { | ||
| 2063 | case PM_HIBERNATION_PREPARE: | ||
| 2064 | pause_graph_tracing(); | ||
| 2065 | break; | ||
| 2066 | |||
| 2067 | case PM_POST_HIBERNATION: | ||
| 2068 | unpause_graph_tracing(); | ||
| 2069 | break; | ||
| 2070 | } | ||
| 2071 | return NOTIFY_DONE; | ||
| 2072 | } | ||
| 2073 | |||
| 2046 | int register_ftrace_graph(trace_func_graph_ret_t retfunc, | 2074 | int register_ftrace_graph(trace_func_graph_ret_t retfunc, |
| 2047 | trace_func_graph_ent_t entryfunc) | 2075 | trace_func_graph_ent_t entryfunc) |
| 2048 | { | 2076 | { |
| @@ -2050,6 +2078,9 @@ int register_ftrace_graph(trace_func_graph_ret_t retfunc, | |||
| 2050 | 2078 | ||
| 2051 | mutex_lock(&ftrace_sysctl_lock); | 2079 | mutex_lock(&ftrace_sysctl_lock); |
| 2052 | 2080 | ||
| 2081 | ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call; | ||
| 2082 | register_pm_notifier(&ftrace_suspend_notifier); | ||
| 2083 | |||
| 2053 | atomic_inc(&ftrace_graph_active); | 2084 | atomic_inc(&ftrace_graph_active); |
| 2054 | ret = start_graph_tracing(); | 2085 | ret = start_graph_tracing(); |
| 2055 | if (ret) { | 2086 | if (ret) { |
| @@ -2075,6 +2106,7 @@ void unregister_ftrace_graph(void) | |||
| 2075 | ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; | 2106 | ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; |
| 2076 | ftrace_graph_entry = ftrace_graph_entry_stub; | 2107 | ftrace_graph_entry = ftrace_graph_entry_stub; |
| 2077 | ftrace_shutdown(FTRACE_STOP_FUNC_RET); | 2108 | ftrace_shutdown(FTRACE_STOP_FUNC_RET); |
| 2109 | unregister_pm_notifier(&ftrace_suspend_notifier); | ||
| 2078 | 2110 | ||
| 2079 | mutex_unlock(&ftrace_sysctl_lock); | 2111 | mutex_unlock(&ftrace_sysctl_lock); |
| 2080 | } | 2112 | } |
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 8b0daf0662ef..bd38c5cfd8ad 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c | |||
| @@ -246,7 +246,7 @@ static inline int test_time_stamp(u64 delta) | |||
| 246 | return 0; | 246 | return 0; |
| 247 | } | 247 | } |
| 248 | 248 | ||
| 249 | #define BUF_PAGE_SIZE (PAGE_SIZE - sizeof(struct buffer_data_page)) | 249 | #define BUF_PAGE_SIZE (PAGE_SIZE - offsetof(struct buffer_data_page, data)) |
| 250 | 250 | ||
| 251 | /* | 251 | /* |
| 252 | * head_page == tail_page && head == tail then buffer is empty. | 252 | * head_page == tail_page && head == tail then buffer is empty. |
| @@ -1025,12 +1025,8 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, | |||
| 1025 | } | 1025 | } |
| 1026 | 1026 | ||
| 1027 | if (next_page == head_page) { | 1027 | if (next_page == head_page) { |
| 1028 | if (!(buffer->flags & RB_FL_OVERWRITE)) { | 1028 | if (!(buffer->flags & RB_FL_OVERWRITE)) |
| 1029 | /* reset write */ | ||
| 1030 | if (tail <= BUF_PAGE_SIZE) | ||
| 1031 | local_set(&tail_page->write, tail); | ||
| 1032 | goto out_unlock; | 1029 | goto out_unlock; |
| 1033 | } | ||
| 1034 | 1030 | ||
| 1035 | /* tail_page has not moved yet? */ | 1031 | /* tail_page has not moved yet? */ |
| 1036 | if (tail_page == cpu_buffer->tail_page) { | 1032 | if (tail_page == cpu_buffer->tail_page) { |
| @@ -1105,6 +1101,10 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, | |||
| 1105 | return event; | 1101 | return event; |
| 1106 | 1102 | ||
| 1107 | out_unlock: | 1103 | out_unlock: |
| 1104 | /* reset write */ | ||
| 1105 | if (tail <= BUF_PAGE_SIZE) | ||
| 1106 | local_set(&tail_page->write, tail); | ||
| 1107 | |||
| 1108 | __raw_spin_unlock(&cpu_buffer->lock); | 1108 | __raw_spin_unlock(&cpu_buffer->lock); |
| 1109 | local_irq_restore(flags); | 1109 | local_irq_restore(flags); |
| 1110 | return NULL; | 1110 | return NULL; |
| @@ -2174,6 +2174,9 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer) | |||
| 2174 | 2174 | ||
| 2175 | cpu_buffer->overrun = 0; | 2175 | cpu_buffer->overrun = 0; |
| 2176 | cpu_buffer->entries = 0; | 2176 | cpu_buffer->entries = 0; |
| 2177 | |||
| 2178 | cpu_buffer->write_stamp = 0; | ||
| 2179 | cpu_buffer->read_stamp = 0; | ||
| 2177 | } | 2180 | } |
| 2178 | 2181 | ||
| 2179 | /** | 2182 | /** |
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index c580233add95..17bb88d86ac2 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
| @@ -40,7 +40,7 @@ | |||
| 40 | 40 | ||
| 41 | #define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE) | 41 | #define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE) |
| 42 | 42 | ||
| 43 | unsigned long __read_mostly tracing_max_latency = (cycle_t)ULONG_MAX; | 43 | unsigned long __read_mostly tracing_max_latency; |
| 44 | unsigned long __read_mostly tracing_thresh; | 44 | unsigned long __read_mostly tracing_thresh; |
| 45 | 45 | ||
| 46 | /* | 46 | /* |
| @@ -3736,7 +3736,7 @@ static struct notifier_block trace_die_notifier = { | |||
| 3736 | * it if we decide to change what log level the ftrace dump | 3736 | * it if we decide to change what log level the ftrace dump |
| 3737 | * should be at. | 3737 | * should be at. |
| 3738 | */ | 3738 | */ |
| 3739 | #define KERN_TRACE KERN_INFO | 3739 | #define KERN_TRACE KERN_EMERG |
| 3740 | 3740 | ||
| 3741 | static void | 3741 | static void |
| 3742 | trace_printk_seq(struct trace_seq *s) | 3742 | trace_printk_seq(struct trace_seq *s) |
| @@ -3770,6 +3770,7 @@ void ftrace_dump(void) | |||
| 3770 | dump_ran = 1; | 3770 | dump_ran = 1; |
| 3771 | 3771 | ||
| 3772 | /* No turning back! */ | 3772 | /* No turning back! */ |
| 3773 | tracing_off(); | ||
| 3773 | ftrace_kill(); | 3774 | ftrace_kill(); |
| 3774 | 3775 | ||
| 3775 | for_each_tracing_cpu(cpu) { | 3776 | for_each_tracing_cpu(cpu) { |
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c index 7c2e326bbc8b..62a78d943534 100644 --- a/kernel/trace/trace_irqsoff.c +++ b/kernel/trace/trace_irqsoff.c | |||
| @@ -380,6 +380,7 @@ static void stop_irqsoff_tracer(struct trace_array *tr) | |||
| 380 | 380 | ||
| 381 | static void __irqsoff_tracer_init(struct trace_array *tr) | 381 | static void __irqsoff_tracer_init(struct trace_array *tr) |
| 382 | { | 382 | { |
| 383 | tracing_max_latency = 0; | ||
| 383 | irqsoff_trace = tr; | 384 | irqsoff_trace = tr; |
| 384 | /* make sure that the tracer is visible */ | 385 | /* make sure that the tracer is visible */ |
| 385 | smp_wmb(); | 386 | smp_wmb(); |
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c index 43586b689e31..42ae1e77b6b3 100644 --- a/kernel/trace/trace_sched_wakeup.c +++ b/kernel/trace/trace_sched_wakeup.c | |||
| @@ -333,6 +333,7 @@ static void stop_wakeup_tracer(struct trace_array *tr) | |||
| 333 | 333 | ||
| 334 | static int wakeup_tracer_init(struct trace_array *tr) | 334 | static int wakeup_tracer_init(struct trace_array *tr) |
| 335 | { | 335 | { |
| 336 | tracing_max_latency = 0; | ||
| 336 | wakeup_trace = tr; | 337 | wakeup_trace = tr; |
| 337 | start_wakeup_tracer(tr); | 338 | start_wakeup_tracer(tr); |
| 338 | return 0; | 339 | return 0; |
diff --git a/kernel/wait.c b/kernel/wait.c index cd87131f2fc2..42a2dbc181c8 100644 --- a/kernel/wait.c +++ b/kernel/wait.c | |||
| @@ -91,6 +91,15 @@ prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state) | |||
| 91 | } | 91 | } |
| 92 | EXPORT_SYMBOL(prepare_to_wait_exclusive); | 92 | EXPORT_SYMBOL(prepare_to_wait_exclusive); |
| 93 | 93 | ||
| 94 | /* | ||
| 95 | * finish_wait - clean up after waiting in a queue | ||
| 96 | * @q: waitqueue waited on | ||
| 97 | * @wait: wait descriptor | ||
| 98 | * | ||
| 99 | * Sets current thread back to running state and removes | ||
| 100 | * the wait descriptor from the given waitqueue if still | ||
| 101 | * queued. | ||
| 102 | */ | ||
| 94 | void finish_wait(wait_queue_head_t *q, wait_queue_t *wait) | 103 | void finish_wait(wait_queue_head_t *q, wait_queue_t *wait) |
| 95 | { | 104 | { |
| 96 | unsigned long flags; | 105 | unsigned long flags; |
| @@ -117,6 +126,39 @@ void finish_wait(wait_queue_head_t *q, wait_queue_t *wait) | |||
| 117 | } | 126 | } |
| 118 | EXPORT_SYMBOL(finish_wait); | 127 | EXPORT_SYMBOL(finish_wait); |
| 119 | 128 | ||
| 129 | /* | ||
| 130 | * abort_exclusive_wait - abort exclusive waiting in a queue | ||
| 131 | * @q: waitqueue waited on | ||
| 132 | * @wait: wait descriptor | ||
| 133 | * @state: runstate of the waiter to be woken | ||
| 134 | * @key: key to identify a wait bit queue or %NULL | ||
| 135 | * | ||
| 136 | * Sets current thread back to running state and removes | ||
| 137 | * the wait descriptor from the given waitqueue if still | ||
| 138 | * queued. | ||
| 139 | * | ||
| 140 | * Wakes up the next waiter if the caller is concurrently | ||
| 141 | * woken up through the queue. | ||
| 142 | * | ||
| 143 | * This prevents waiter starvation where an exclusive waiter | ||
| 144 | * aborts and is woken up concurrently and noone wakes up | ||
| 145 | * the next waiter. | ||
| 146 | */ | ||
| 147 | void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait, | ||
| 148 | unsigned int mode, void *key) | ||
| 149 | { | ||
| 150 | unsigned long flags; | ||
| 151 | |||
| 152 | __set_current_state(TASK_RUNNING); | ||
| 153 | spin_lock_irqsave(&q->lock, flags); | ||
| 154 | if (!list_empty(&wait->task_list)) | ||
| 155 | list_del_init(&wait->task_list); | ||
| 156 | else if (waitqueue_active(q)) | ||
| 157 | __wake_up_common(q, mode, 1, 0, key); | ||
| 158 | spin_unlock_irqrestore(&q->lock, flags); | ||
| 159 | } | ||
| 160 | EXPORT_SYMBOL(abort_exclusive_wait); | ||
| 161 | |||
| 120 | int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key) | 162 | int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key) |
| 121 | { | 163 | { |
| 122 | int ret = default_wake_function(wait, mode, sync, key); | 164 | int ret = default_wake_function(wait, mode, sync, key); |
| @@ -177,17 +219,20 @@ int __sched | |||
| 177 | __wait_on_bit_lock(wait_queue_head_t *wq, struct wait_bit_queue *q, | 219 | __wait_on_bit_lock(wait_queue_head_t *wq, struct wait_bit_queue *q, |
| 178 | int (*action)(void *), unsigned mode) | 220 | int (*action)(void *), unsigned mode) |
| 179 | { | 221 | { |
| 180 | int ret = 0; | ||
| 181 | |||
| 182 | do { | 222 | do { |
| 223 | int ret; | ||
| 224 | |||
| 183 | prepare_to_wait_exclusive(wq, &q->wait, mode); | 225 | prepare_to_wait_exclusive(wq, &q->wait, mode); |
| 184 | if (test_bit(q->key.bit_nr, q->key.flags)) { | 226 | if (!test_bit(q->key.bit_nr, q->key.flags)) |
| 185 | if ((ret = (*action)(q->key.flags))) | 227 | continue; |
| 186 | break; | 228 | ret = action(q->key.flags); |
| 187 | } | 229 | if (!ret) |
| 230 | continue; | ||
| 231 | abort_exclusive_wait(wq, &q->wait, mode, &q->key); | ||
| 232 | return ret; | ||
| 188 | } while (test_and_set_bit(q->key.bit_nr, q->key.flags)); | 233 | } while (test_and_set_bit(q->key.bit_nr, q->key.flags)); |
| 189 | finish_wait(wq, &q->wait); | 234 | finish_wait(wq, &q->wait); |
| 190 | return ret; | 235 | return 0; |
| 191 | } | 236 | } |
| 192 | EXPORT_SYMBOL(__wait_on_bit_lock); | 237 | EXPORT_SYMBOL(__wait_on_bit_lock); |
| 193 | 238 | ||
