diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/Makefile | 5 | ||||
| -rw-r--r-- | kernel/audit_tree.c | 91 | ||||
| -rw-r--r-- | kernel/auditfilter.c | 14 | ||||
| -rw-r--r-- | kernel/cgroup.c | 22 | ||||
| -rw-r--r-- | kernel/cgroup_freezer.c | 19 | ||||
| -rw-r--r-- | kernel/cpu.c | 3 | ||||
| -rw-r--r-- | kernel/cpuset.c | 31 | ||||
| -rw-r--r-- | kernel/exit.c | 14 | ||||
| -rw-r--r-- | kernel/fork.c | 11 | ||||
| -rw-r--r-- | kernel/hrtimer.c | 26 | ||||
| -rw-r--r-- | kernel/kallsyms.c | 17 | ||||
| -rw-r--r-- | kernel/kprobes.c | 23 | ||||
| -rw-r--r-- | kernel/posix-cpu-timers.c | 7 | ||||
| -rw-r--r-- | kernel/power/main.c | 2 | ||||
| -rw-r--r-- | kernel/profile.c | 2 | ||||
| -rw-r--r-- | kernel/relay.c | 9 | ||||
| -rw-r--r-- | kernel/sched.c | 231 | ||||
| -rw-r--r-- | kernel/sched_debug.c | 97 | ||||
| -rw-r--r-- | kernel/sched_fair.c | 83 | ||||
| -rw-r--r-- | kernel/sched_features.h | 1 | ||||
| -rw-r--r-- | kernel/sched_stats.h | 15 | ||||
| -rw-r--r-- | kernel/smp.c | 18 | ||||
| -rw-r--r-- | kernel/softirq.c | 7 | ||||
| -rw-r--r-- | kernel/stop_machine.c | 5 | ||||
| -rw-r--r-- | kernel/sys_ni.c | 2 | ||||
| -rw-r--r-- | kernel/time/tick-sched.c | 4 | ||||
| -rw-r--r-- | kernel/timer.c | 129 | ||||
| -rw-r--r-- | kernel/trace/Kconfig | 2 | ||||
| -rw-r--r-- | kernel/trace/ftrace.c | 147 | ||||
| -rw-r--r-- | kernel/trace/ring_buffer.c | 174 | ||||
| -rw-r--r-- | kernel/trace/trace.c | 61 | ||||
| -rw-r--r-- | kernel/workqueue.c | 45 |
32 files changed, 877 insertions, 440 deletions
diff --git a/kernel/Makefile b/kernel/Makefile index 9a3ec66a9d84..6a212b842d86 100644 --- a/kernel/Makefile +++ b/kernel/Makefile | |||
| @@ -11,8 +11,6 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o \ | |||
| 11 | hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \ | 11 | hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \ |
| 12 | notifier.o ksysfs.o pm_qos_params.o sched_clock.o | 12 | notifier.o ksysfs.o pm_qos_params.o sched_clock.o |
| 13 | 13 | ||
| 14 | CFLAGS_REMOVE_sched.o = -mno-spe | ||
| 15 | |||
| 16 | ifdef CONFIG_FUNCTION_TRACER | 14 | ifdef CONFIG_FUNCTION_TRACER |
| 17 | # Do not trace debug files and internal ftrace files | 15 | # Do not trace debug files and internal ftrace files |
| 18 | CFLAGS_REMOVE_lockdep.o = -pg | 16 | CFLAGS_REMOVE_lockdep.o = -pg |
| @@ -21,7 +19,6 @@ CFLAGS_REMOVE_mutex-debug.o = -pg | |||
| 21 | CFLAGS_REMOVE_rtmutex-debug.o = -pg | 19 | CFLAGS_REMOVE_rtmutex-debug.o = -pg |
| 22 | CFLAGS_REMOVE_cgroup-debug.o = -pg | 20 | CFLAGS_REMOVE_cgroup-debug.o = -pg |
| 23 | CFLAGS_REMOVE_sched_clock.o = -pg | 21 | CFLAGS_REMOVE_sched_clock.o = -pg |
| 24 | CFLAGS_REMOVE_sched.o = -mno-spe -pg | ||
| 25 | endif | 22 | endif |
| 26 | 23 | ||
| 27 | obj-$(CONFIG_FREEZER) += freezer.o | 24 | obj-$(CONFIG_FREEZER) += freezer.o |
| @@ -92,7 +89,7 @@ obj-$(CONFIG_FUNCTION_TRACER) += trace/ | |||
| 92 | obj-$(CONFIG_TRACING) += trace/ | 89 | obj-$(CONFIG_TRACING) += trace/ |
| 93 | obj-$(CONFIG_SMP) += sched_cpupri.o | 90 | obj-$(CONFIG_SMP) += sched_cpupri.o |
| 94 | 91 | ||
| 95 | ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y) | 92 | ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) |
| 96 | # According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is | 93 | # According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is |
| 97 | # needed for x86 only. Why this used to be enabled for all architectures is beyond | 94 | # needed for x86 only. Why this used to be enabled for all architectures is beyond |
| 98 | # me. I suspect most platforms don't need this, but until we know that for sure | 95 | # me. I suspect most platforms don't need this, but until we know that for sure |
diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c index 8ba0e0d934f2..8b509441f49a 100644 --- a/kernel/audit_tree.c +++ b/kernel/audit_tree.c | |||
| @@ -24,6 +24,7 @@ struct audit_chunk { | |||
| 24 | struct list_head trees; /* with root here */ | 24 | struct list_head trees; /* with root here */ |
| 25 | int dead; | 25 | int dead; |
| 26 | int count; | 26 | int count; |
| 27 | atomic_long_t refs; | ||
| 27 | struct rcu_head head; | 28 | struct rcu_head head; |
| 28 | struct node { | 29 | struct node { |
| 29 | struct list_head list; | 30 | struct list_head list; |
| @@ -56,7 +57,8 @@ static LIST_HEAD(prune_list); | |||
| 56 | * tree is refcounted; one reference for "some rules on rules_list refer to | 57 | * tree is refcounted; one reference for "some rules on rules_list refer to |
| 57 | * it", one for each chunk with pointer to it. | 58 | * it", one for each chunk with pointer to it. |
| 58 | * | 59 | * |
| 59 | * chunk is refcounted by embedded inotify_watch. | 60 | * chunk is refcounted by embedded inotify_watch + .refs (non-zero refcount |
| 61 | * of watch contributes 1 to .refs). | ||
| 60 | * | 62 | * |
| 61 | * node.index allows to get from node.list to containing chunk. | 63 | * node.index allows to get from node.list to containing chunk. |
| 62 | * MSB of that sucker is stolen to mark taggings that we might have to | 64 | * MSB of that sucker is stolen to mark taggings that we might have to |
| @@ -121,6 +123,7 @@ static struct audit_chunk *alloc_chunk(int count) | |||
| 121 | INIT_LIST_HEAD(&chunk->hash); | 123 | INIT_LIST_HEAD(&chunk->hash); |
| 122 | INIT_LIST_HEAD(&chunk->trees); | 124 | INIT_LIST_HEAD(&chunk->trees); |
| 123 | chunk->count = count; | 125 | chunk->count = count; |
| 126 | atomic_long_set(&chunk->refs, 1); | ||
| 124 | for (i = 0; i < count; i++) { | 127 | for (i = 0; i < count; i++) { |
| 125 | INIT_LIST_HEAD(&chunk->owners[i].list); | 128 | INIT_LIST_HEAD(&chunk->owners[i].list); |
| 126 | chunk->owners[i].index = i; | 129 | chunk->owners[i].index = i; |
| @@ -129,9 +132,8 @@ static struct audit_chunk *alloc_chunk(int count) | |||
| 129 | return chunk; | 132 | return chunk; |
| 130 | } | 133 | } |
| 131 | 134 | ||
| 132 | static void __free_chunk(struct rcu_head *rcu) | 135 | static void free_chunk(struct audit_chunk *chunk) |
| 133 | { | 136 | { |
| 134 | struct audit_chunk *chunk = container_of(rcu, struct audit_chunk, head); | ||
| 135 | int i; | 137 | int i; |
| 136 | 138 | ||
| 137 | for (i = 0; i < chunk->count; i++) { | 139 | for (i = 0; i < chunk->count; i++) { |
| @@ -141,14 +143,16 @@ static void __free_chunk(struct rcu_head *rcu) | |||
| 141 | kfree(chunk); | 143 | kfree(chunk); |
| 142 | } | 144 | } |
| 143 | 145 | ||
| 144 | static inline void free_chunk(struct audit_chunk *chunk) | 146 | void audit_put_chunk(struct audit_chunk *chunk) |
| 145 | { | 147 | { |
| 146 | call_rcu(&chunk->head, __free_chunk); | 148 | if (atomic_long_dec_and_test(&chunk->refs)) |
| 149 | free_chunk(chunk); | ||
| 147 | } | 150 | } |
| 148 | 151 | ||
| 149 | void audit_put_chunk(struct audit_chunk *chunk) | 152 | static void __put_chunk(struct rcu_head *rcu) |
| 150 | { | 153 | { |
| 151 | put_inotify_watch(&chunk->watch); | 154 | struct audit_chunk *chunk = container_of(rcu, struct audit_chunk, head); |
| 155 | audit_put_chunk(chunk); | ||
| 152 | } | 156 | } |
| 153 | 157 | ||
| 154 | enum {HASH_SIZE = 128}; | 158 | enum {HASH_SIZE = 128}; |
| @@ -176,7 +180,7 @@ struct audit_chunk *audit_tree_lookup(const struct inode *inode) | |||
| 176 | 180 | ||
| 177 | list_for_each_entry_rcu(p, list, hash) { | 181 | list_for_each_entry_rcu(p, list, hash) { |
| 178 | if (p->watch.inode == inode) { | 182 | if (p->watch.inode == inode) { |
| 179 | get_inotify_watch(&p->watch); | 183 | atomic_long_inc(&p->refs); |
| 180 | return p; | 184 | return p; |
| 181 | } | 185 | } |
| 182 | } | 186 | } |
| @@ -194,17 +198,49 @@ int audit_tree_match(struct audit_chunk *chunk, struct audit_tree *tree) | |||
| 194 | 198 | ||
| 195 | /* tagging and untagging inodes with trees */ | 199 | /* tagging and untagging inodes with trees */ |
| 196 | 200 | ||
| 197 | static void untag_chunk(struct audit_chunk *chunk, struct node *p) | 201 | static struct audit_chunk *find_chunk(struct node *p) |
| 202 | { | ||
| 203 | int index = p->index & ~(1U<<31); | ||
| 204 | p -= index; | ||
| 205 | return container_of(p, struct audit_chunk, owners[0]); | ||
| 206 | } | ||
| 207 | |||
| 208 | static void untag_chunk(struct node *p) | ||
| 198 | { | 209 | { |
| 210 | struct audit_chunk *chunk = find_chunk(p); | ||
| 199 | struct audit_chunk *new; | 211 | struct audit_chunk *new; |
| 200 | struct audit_tree *owner; | 212 | struct audit_tree *owner; |
| 201 | int size = chunk->count - 1; | 213 | int size = chunk->count - 1; |
| 202 | int i, j; | 214 | int i, j; |
| 203 | 215 | ||
| 216 | if (!pin_inotify_watch(&chunk->watch)) { | ||
| 217 | /* | ||
| 218 | * Filesystem is shutting down; all watches are getting | ||
| 219 | * evicted, just take it off the node list for this | ||
| 220 | * tree and let the eviction logics take care of the | ||
| 221 | * rest. | ||
| 222 | */ | ||
| 223 | owner = p->owner; | ||
| 224 | if (owner->root == chunk) { | ||
| 225 | list_del_init(&owner->same_root); | ||
| 226 | owner->root = NULL; | ||
| 227 | } | ||
| 228 | list_del_init(&p->list); | ||
| 229 | p->owner = NULL; | ||
| 230 | put_tree(owner); | ||
| 231 | return; | ||
| 232 | } | ||
| 233 | |||
| 234 | spin_unlock(&hash_lock); | ||
| 235 | |||
| 236 | /* | ||
| 237 | * pin_inotify_watch() succeeded, so the watch won't go away | ||
| 238 | * from under us. | ||
| 239 | */ | ||
| 204 | mutex_lock(&chunk->watch.inode->inotify_mutex); | 240 | mutex_lock(&chunk->watch.inode->inotify_mutex); |
| 205 | if (chunk->dead) { | 241 | if (chunk->dead) { |
| 206 | mutex_unlock(&chunk->watch.inode->inotify_mutex); | 242 | mutex_unlock(&chunk->watch.inode->inotify_mutex); |
| 207 | return; | 243 | goto out; |
| 208 | } | 244 | } |
| 209 | 245 | ||
| 210 | owner = p->owner; | 246 | owner = p->owner; |
| @@ -221,7 +257,7 @@ static void untag_chunk(struct audit_chunk *chunk, struct node *p) | |||
| 221 | inotify_evict_watch(&chunk->watch); | 257 | inotify_evict_watch(&chunk->watch); |
| 222 | mutex_unlock(&chunk->watch.inode->inotify_mutex); | 258 | mutex_unlock(&chunk->watch.inode->inotify_mutex); |
| 223 | put_inotify_watch(&chunk->watch); | 259 | put_inotify_watch(&chunk->watch); |
| 224 | return; | 260 | goto out; |
| 225 | } | 261 | } |
| 226 | 262 | ||
| 227 | new = alloc_chunk(size); | 263 | new = alloc_chunk(size); |
| @@ -263,7 +299,7 @@ static void untag_chunk(struct audit_chunk *chunk, struct node *p) | |||
| 263 | inotify_evict_watch(&chunk->watch); | 299 | inotify_evict_watch(&chunk->watch); |
| 264 | mutex_unlock(&chunk->watch.inode->inotify_mutex); | 300 | mutex_unlock(&chunk->watch.inode->inotify_mutex); |
| 265 | put_inotify_watch(&chunk->watch); | 301 | put_inotify_watch(&chunk->watch); |
| 266 | return; | 302 | goto out; |
| 267 | 303 | ||
| 268 | Fallback: | 304 | Fallback: |
| 269 | // do the best we can | 305 | // do the best we can |
| @@ -277,6 +313,9 @@ Fallback: | |||
| 277 | put_tree(owner); | 313 | put_tree(owner); |
| 278 | spin_unlock(&hash_lock); | 314 | spin_unlock(&hash_lock); |
| 279 | mutex_unlock(&chunk->watch.inode->inotify_mutex); | 315 | mutex_unlock(&chunk->watch.inode->inotify_mutex); |
| 316 | out: | ||
| 317 | unpin_inotify_watch(&chunk->watch); | ||
| 318 | spin_lock(&hash_lock); | ||
| 280 | } | 319 | } |
| 281 | 320 | ||
| 282 | static int create_chunk(struct inode *inode, struct audit_tree *tree) | 321 | static int create_chunk(struct inode *inode, struct audit_tree *tree) |
| @@ -387,13 +426,6 @@ static int tag_chunk(struct inode *inode, struct audit_tree *tree) | |||
| 387 | return 0; | 426 | return 0; |
| 388 | } | 427 | } |
| 389 | 428 | ||
| 390 | static struct audit_chunk *find_chunk(struct node *p) | ||
| 391 | { | ||
| 392 | int index = p->index & ~(1U<<31); | ||
| 393 | p -= index; | ||
| 394 | return container_of(p, struct audit_chunk, owners[0]); | ||
| 395 | } | ||
| 396 | |||
| 397 | static void kill_rules(struct audit_tree *tree) | 429 | static void kill_rules(struct audit_tree *tree) |
| 398 | { | 430 | { |
| 399 | struct audit_krule *rule, *next; | 431 | struct audit_krule *rule, *next; |
| @@ -431,17 +463,10 @@ static void prune_one(struct audit_tree *victim) | |||
| 431 | spin_lock(&hash_lock); | 463 | spin_lock(&hash_lock); |
| 432 | while (!list_empty(&victim->chunks)) { | 464 | while (!list_empty(&victim->chunks)) { |
| 433 | struct node *p; | 465 | struct node *p; |
| 434 | struct audit_chunk *chunk; | ||
| 435 | 466 | ||
| 436 | p = list_entry(victim->chunks.next, struct node, list); | 467 | p = list_entry(victim->chunks.next, struct node, list); |
| 437 | chunk = find_chunk(p); | ||
| 438 | get_inotify_watch(&chunk->watch); | ||
| 439 | spin_unlock(&hash_lock); | ||
| 440 | |||
| 441 | untag_chunk(chunk, p); | ||
| 442 | 468 | ||
| 443 | put_inotify_watch(&chunk->watch); | 469 | untag_chunk(p); |
| 444 | spin_lock(&hash_lock); | ||
| 445 | } | 470 | } |
| 446 | spin_unlock(&hash_lock); | 471 | spin_unlock(&hash_lock); |
| 447 | put_tree(victim); | 472 | put_tree(victim); |
| @@ -469,7 +494,6 @@ static void trim_marked(struct audit_tree *tree) | |||
| 469 | 494 | ||
| 470 | while (!list_empty(&tree->chunks)) { | 495 | while (!list_empty(&tree->chunks)) { |
| 471 | struct node *node; | 496 | struct node *node; |
| 472 | struct audit_chunk *chunk; | ||
| 473 | 497 | ||
| 474 | node = list_entry(tree->chunks.next, struct node, list); | 498 | node = list_entry(tree->chunks.next, struct node, list); |
| 475 | 499 | ||
| @@ -477,14 +501,7 @@ static void trim_marked(struct audit_tree *tree) | |||
| 477 | if (!(node->index & (1U<<31))) | 501 | if (!(node->index & (1U<<31))) |
| 478 | break; | 502 | break; |
| 479 | 503 | ||
| 480 | chunk = find_chunk(node); | 504 | untag_chunk(node); |
| 481 | get_inotify_watch(&chunk->watch); | ||
| 482 | spin_unlock(&hash_lock); | ||
| 483 | |||
| 484 | untag_chunk(chunk, node); | ||
| 485 | |||
| 486 | put_inotify_watch(&chunk->watch); | ||
| 487 | spin_lock(&hash_lock); | ||
| 488 | } | 505 | } |
| 489 | if (!tree->root && !tree->goner) { | 506 | if (!tree->root && !tree->goner) { |
| 490 | tree->goner = 1; | 507 | tree->goner = 1; |
| @@ -878,7 +895,7 @@ static void handle_event(struct inotify_watch *watch, u32 wd, u32 mask, | |||
| 878 | static void destroy_watch(struct inotify_watch *watch) | 895 | static void destroy_watch(struct inotify_watch *watch) |
| 879 | { | 896 | { |
| 880 | struct audit_chunk *chunk = container_of(watch, struct audit_chunk, watch); | 897 | struct audit_chunk *chunk = container_of(watch, struct audit_chunk, watch); |
| 881 | free_chunk(chunk); | 898 | call_rcu(&chunk->head, __put_chunk); |
| 882 | } | 899 | } |
| 883 | 900 | ||
| 884 | static const struct inotify_operations rtree_inotify_ops = { | 901 | static const struct inotify_operations rtree_inotify_ops = { |
diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c index b7d354e2b0ef..9fd85a4640a0 100644 --- a/kernel/auditfilter.c +++ b/kernel/auditfilter.c | |||
| @@ -1094,8 +1094,8 @@ static void audit_inotify_unregister(struct list_head *in_list) | |||
| 1094 | list_for_each_entry_safe(p, n, in_list, ilist) { | 1094 | list_for_each_entry_safe(p, n, in_list, ilist) { |
| 1095 | list_del(&p->ilist); | 1095 | list_del(&p->ilist); |
| 1096 | inotify_rm_watch(audit_ih, &p->wdata); | 1096 | inotify_rm_watch(audit_ih, &p->wdata); |
| 1097 | /* the put matching the get in audit_do_del_rule() */ | 1097 | /* the unpin matching the pin in audit_do_del_rule() */ |
| 1098 | put_inotify_watch(&p->wdata); | 1098 | unpin_inotify_watch(&p->wdata); |
| 1099 | } | 1099 | } |
| 1100 | } | 1100 | } |
| 1101 | 1101 | ||
| @@ -1389,9 +1389,13 @@ static inline int audit_del_rule(struct audit_entry *entry, | |||
| 1389 | /* Put parent on the inotify un-registration | 1389 | /* Put parent on the inotify un-registration |
| 1390 | * list. Grab a reference before releasing | 1390 | * list. Grab a reference before releasing |
| 1391 | * audit_filter_mutex, to be released in | 1391 | * audit_filter_mutex, to be released in |
| 1392 | * audit_inotify_unregister(). */ | 1392 | * audit_inotify_unregister(). |
| 1393 | list_add(&parent->ilist, &inotify_list); | 1393 | * If filesystem is going away, just leave |
| 1394 | get_inotify_watch(&parent->wdata); | 1394 | * the sucker alone, eviction will take |
| 1395 | * care of it. | ||
| 1396 | */ | ||
| 1397 | if (pin_inotify_watch(&parent->wdata)) | ||
| 1398 | list_add(&parent->ilist, &inotify_list); | ||
| 1395 | } | 1399 | } |
| 1396 | } | 1400 | } |
| 1397 | } | 1401 | } |
diff --git a/kernel/cgroup.c b/kernel/cgroup.c index 35eebd5510c2..fe00b3b983a8 100644 --- a/kernel/cgroup.c +++ b/kernel/cgroup.c | |||
| @@ -2039,10 +2039,13 @@ int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry) | |||
| 2039 | struct cgroup *cgrp; | 2039 | struct cgroup *cgrp; |
| 2040 | struct cgroup_iter it; | 2040 | struct cgroup_iter it; |
| 2041 | struct task_struct *tsk; | 2041 | struct task_struct *tsk; |
| 2042 | |||
| 2042 | /* | 2043 | /* |
| 2043 | * Validate dentry by checking the superblock operations | 2044 | * Validate dentry by checking the superblock operations, |
| 2045 | * and make sure it's a directory. | ||
| 2044 | */ | 2046 | */ |
| 2045 | if (dentry->d_sb->s_op != &cgroup_ops) | 2047 | if (dentry->d_sb->s_op != &cgroup_ops || |
| 2048 | !S_ISDIR(dentry->d_inode->i_mode)) | ||
| 2046 | goto err; | 2049 | goto err; |
| 2047 | 2050 | ||
| 2048 | ret = 0; | 2051 | ret = 0; |
| @@ -2472,10 +2475,7 @@ static int cgroup_rmdir(struct inode *unused_dir, struct dentry *dentry) | |||
| 2472 | mutex_unlock(&cgroup_mutex); | 2475 | mutex_unlock(&cgroup_mutex); |
| 2473 | return -EBUSY; | 2476 | return -EBUSY; |
| 2474 | } | 2477 | } |
| 2475 | 2478 | mutex_unlock(&cgroup_mutex); | |
| 2476 | parent = cgrp->parent; | ||
| 2477 | root = cgrp->root; | ||
| 2478 | sb = root->sb; | ||
| 2479 | 2479 | ||
| 2480 | /* | 2480 | /* |
| 2481 | * Call pre_destroy handlers of subsys. Notify subsystems | 2481 | * Call pre_destroy handlers of subsys. Notify subsystems |
| @@ -2483,7 +2483,14 @@ static int cgroup_rmdir(struct inode *unused_dir, struct dentry *dentry) | |||
| 2483 | */ | 2483 | */ |
| 2484 | cgroup_call_pre_destroy(cgrp); | 2484 | cgroup_call_pre_destroy(cgrp); |
| 2485 | 2485 | ||
| 2486 | if (cgroup_has_css_refs(cgrp)) { | 2486 | mutex_lock(&cgroup_mutex); |
| 2487 | parent = cgrp->parent; | ||
| 2488 | root = cgrp->root; | ||
| 2489 | sb = root->sb; | ||
| 2490 | |||
| 2491 | if (atomic_read(&cgrp->count) | ||
| 2492 | || !list_empty(&cgrp->children) | ||
| 2493 | || cgroup_has_css_refs(cgrp)) { | ||
| 2487 | mutex_unlock(&cgroup_mutex); | 2494 | mutex_unlock(&cgroup_mutex); |
| 2488 | return -EBUSY; | 2495 | return -EBUSY; |
| 2489 | } | 2496 | } |
| @@ -2497,7 +2504,6 @@ static int cgroup_rmdir(struct inode *unused_dir, struct dentry *dentry) | |||
| 2497 | list_del(&cgrp->sibling); | 2504 | list_del(&cgrp->sibling); |
| 2498 | spin_lock(&cgrp->dentry->d_lock); | 2505 | spin_lock(&cgrp->dentry->d_lock); |
| 2499 | d = dget(cgrp->dentry); | 2506 | d = dget(cgrp->dentry); |
| 2500 | cgrp->dentry = NULL; | ||
| 2501 | spin_unlock(&d->d_lock); | 2507 | spin_unlock(&d->d_lock); |
| 2502 | 2508 | ||
| 2503 | cgroup_d_remove_dir(d); | 2509 | cgroup_d_remove_dir(d); |
diff --git a/kernel/cgroup_freezer.c b/kernel/cgroup_freezer.c index 7fa476f01d05..fb249e2bcada 100644 --- a/kernel/cgroup_freezer.c +++ b/kernel/cgroup_freezer.c | |||
| @@ -184,9 +184,20 @@ static void freezer_fork(struct cgroup_subsys *ss, struct task_struct *task) | |||
| 184 | { | 184 | { |
| 185 | struct freezer *freezer; | 185 | struct freezer *freezer; |
| 186 | 186 | ||
| 187 | task_lock(task); | 187 | /* |
| 188 | * No lock is needed, since the task isn't on tasklist yet, | ||
| 189 | * so it can't be moved to another cgroup, which means the | ||
| 190 | * freezer won't be removed and will be valid during this | ||
| 191 | * function call. | ||
| 192 | */ | ||
| 188 | freezer = task_freezer(task); | 193 | freezer = task_freezer(task); |
| 189 | task_unlock(task); | 194 | |
| 195 | /* | ||
| 196 | * The root cgroup is non-freezable, so we can skip the | ||
| 197 | * following check. | ||
| 198 | */ | ||
| 199 | if (!freezer->css.cgroup->parent) | ||
| 200 | return; | ||
| 190 | 201 | ||
| 191 | spin_lock_irq(&freezer->lock); | 202 | spin_lock_irq(&freezer->lock); |
| 192 | BUG_ON(freezer->state == CGROUP_FROZEN); | 203 | BUG_ON(freezer->state == CGROUP_FROZEN); |
| @@ -331,7 +342,7 @@ static int freezer_write(struct cgroup *cgroup, | |||
| 331 | else if (strcmp(buffer, freezer_state_strs[CGROUP_FROZEN]) == 0) | 342 | else if (strcmp(buffer, freezer_state_strs[CGROUP_FROZEN]) == 0) |
| 332 | goal_state = CGROUP_FROZEN; | 343 | goal_state = CGROUP_FROZEN; |
| 333 | else | 344 | else |
| 334 | return -EIO; | 345 | return -EINVAL; |
| 335 | 346 | ||
| 336 | if (!cgroup_lock_live_group(cgroup)) | 347 | if (!cgroup_lock_live_group(cgroup)) |
| 337 | return -ENODEV; | 348 | return -ENODEV; |
| @@ -350,6 +361,8 @@ static struct cftype files[] = { | |||
| 350 | 361 | ||
| 351 | static int freezer_populate(struct cgroup_subsys *ss, struct cgroup *cgroup) | 362 | static int freezer_populate(struct cgroup_subsys *ss, struct cgroup *cgroup) |
| 352 | { | 363 | { |
| 364 | if (!cgroup->parent) | ||
| 365 | return 0; | ||
| 353 | return cgroup_add_files(cgroup, ss, files, ARRAY_SIZE(files)); | 366 | return cgroup_add_files(cgroup, ss, files, ARRAY_SIZE(files)); |
| 354 | } | 367 | } |
| 355 | 368 | ||
diff --git a/kernel/cpu.c b/kernel/cpu.c index 86d49045daed..5a732c5ef08b 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c | |||
| @@ -499,3 +499,6 @@ const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = { | |||
| 499 | #endif | 499 | #endif |
| 500 | }; | 500 | }; |
| 501 | EXPORT_SYMBOL_GPL(cpu_bit_bitmap); | 501 | EXPORT_SYMBOL_GPL(cpu_bit_bitmap); |
| 502 | |||
| 503 | const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL; | ||
| 504 | EXPORT_SYMBOL(cpu_all_bits); | ||
diff --git a/kernel/cpuset.c b/kernel/cpuset.c index 3e00526f52ec..da7ff6137f37 100644 --- a/kernel/cpuset.c +++ b/kernel/cpuset.c | |||
| @@ -36,6 +36,7 @@ | |||
| 36 | #include <linux/list.h> | 36 | #include <linux/list.h> |
| 37 | #include <linux/mempolicy.h> | 37 | #include <linux/mempolicy.h> |
| 38 | #include <linux/mm.h> | 38 | #include <linux/mm.h> |
| 39 | #include <linux/memory.h> | ||
| 39 | #include <linux/module.h> | 40 | #include <linux/module.h> |
| 40 | #include <linux/mount.h> | 41 | #include <linux/mount.h> |
| 41 | #include <linux/namei.h> | 42 | #include <linux/namei.h> |
| @@ -587,7 +588,6 @@ static int generate_sched_domains(cpumask_t **domains, | |||
| 587 | int ndoms; /* number of sched domains in result */ | 588 | int ndoms; /* number of sched domains in result */ |
| 588 | int nslot; /* next empty doms[] cpumask_t slot */ | 589 | int nslot; /* next empty doms[] cpumask_t slot */ |
| 589 | 590 | ||
| 590 | ndoms = 0; | ||
| 591 | doms = NULL; | 591 | doms = NULL; |
| 592 | dattr = NULL; | 592 | dattr = NULL; |
| 593 | csa = NULL; | 593 | csa = NULL; |
| @@ -674,10 +674,8 @@ restart: | |||
| 674 | * Convert <csn, csa> to <ndoms, doms> and populate cpu masks. | 674 | * Convert <csn, csa> to <ndoms, doms> and populate cpu masks. |
| 675 | */ | 675 | */ |
| 676 | doms = kmalloc(ndoms * sizeof(cpumask_t), GFP_KERNEL); | 676 | doms = kmalloc(ndoms * sizeof(cpumask_t), GFP_KERNEL); |
| 677 | if (!doms) { | 677 | if (!doms) |
| 678 | ndoms = 0; | ||
| 679 | goto done; | 678 | goto done; |
| 680 | } | ||
| 681 | 679 | ||
| 682 | /* | 680 | /* |
| 683 | * The rest of the code, including the scheduler, can deal with | 681 | * The rest of the code, including the scheduler, can deal with |
| @@ -732,6 +730,13 @@ restart: | |||
| 732 | done: | 730 | done: |
| 733 | kfree(csa); | 731 | kfree(csa); |
| 734 | 732 | ||
| 733 | /* | ||
| 734 | * Fallback to the default domain if kmalloc() failed. | ||
| 735 | * See comments in partition_sched_domains(). | ||
| 736 | */ | ||
| 737 | if (doms == NULL) | ||
| 738 | ndoms = 1; | ||
| 739 | |||
| 735 | *domains = doms; | 740 | *domains = doms; |
| 736 | *attributes = dattr; | 741 | *attributes = dattr; |
| 737 | return ndoms; | 742 | return ndoms; |
| @@ -2011,12 +2016,23 @@ static int cpuset_track_online_cpus(struct notifier_block *unused_nb, | |||
| 2011 | * Call this routine anytime after node_states[N_HIGH_MEMORY] changes. | 2016 | * Call this routine anytime after node_states[N_HIGH_MEMORY] changes. |
| 2012 | * See also the previous routine cpuset_track_online_cpus(). | 2017 | * See also the previous routine cpuset_track_online_cpus(). |
| 2013 | */ | 2018 | */ |
| 2014 | void cpuset_track_online_nodes(void) | 2019 | static int cpuset_track_online_nodes(struct notifier_block *self, |
| 2020 | unsigned long action, void *arg) | ||
| 2015 | { | 2021 | { |
| 2016 | cgroup_lock(); | 2022 | cgroup_lock(); |
| 2017 | top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY]; | 2023 | switch (action) { |
| 2018 | scan_for_empty_cpusets(&top_cpuset); | 2024 | case MEM_ONLINE: |
| 2025 | top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY]; | ||
| 2026 | break; | ||
| 2027 | case MEM_OFFLINE: | ||
| 2028 | top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY]; | ||
| 2029 | scan_for_empty_cpusets(&top_cpuset); | ||
| 2030 | break; | ||
| 2031 | default: | ||
| 2032 | break; | ||
| 2033 | } | ||
| 2019 | cgroup_unlock(); | 2034 | cgroup_unlock(); |
| 2035 | return NOTIFY_OK; | ||
| 2020 | } | 2036 | } |
| 2021 | #endif | 2037 | #endif |
| 2022 | 2038 | ||
| @@ -2032,6 +2048,7 @@ void __init cpuset_init_smp(void) | |||
| 2032 | top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY]; | 2048 | top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY]; |
| 2033 | 2049 | ||
| 2034 | hotcpu_notifier(cpuset_track_online_cpus, 0); | 2050 | hotcpu_notifier(cpuset_track_online_cpus, 0); |
| 2051 | hotplug_memory_notifier(cpuset_track_online_nodes, 10); | ||
| 2035 | } | 2052 | } |
| 2036 | 2053 | ||
| 2037 | /** | 2054 | /** |
diff --git a/kernel/exit.c b/kernel/exit.c index 80137a5d9467..2d8be7ebb0f7 100644 --- a/kernel/exit.c +++ b/kernel/exit.c | |||
| @@ -40,7 +40,6 @@ | |||
| 40 | #include <linux/cn_proc.h> | 40 | #include <linux/cn_proc.h> |
| 41 | #include <linux/mutex.h> | 41 | #include <linux/mutex.h> |
| 42 | #include <linux/futex.h> | 42 | #include <linux/futex.h> |
| 43 | #include <linux/compat.h> | ||
| 44 | #include <linux/pipe_fs_i.h> | 43 | #include <linux/pipe_fs_i.h> |
| 45 | #include <linux/audit.h> /* for audit_free() */ | 44 | #include <linux/audit.h> /* for audit_free() */ |
| 46 | #include <linux/resource.h> | 45 | #include <linux/resource.h> |
| @@ -141,6 +140,11 @@ static void __exit_signal(struct task_struct *tsk) | |||
| 141 | if (sig) { | 140 | if (sig) { |
| 142 | flush_sigqueue(&sig->shared_pending); | 141 | flush_sigqueue(&sig->shared_pending); |
| 143 | taskstats_tgid_free(sig); | 142 | taskstats_tgid_free(sig); |
| 143 | /* | ||
| 144 | * Make sure ->signal can't go away under rq->lock, | ||
| 145 | * see account_group_exec_runtime(). | ||
| 146 | */ | ||
| 147 | task_rq_unlock_wait(tsk); | ||
| 144 | __cleanup_signal(sig); | 148 | __cleanup_signal(sig); |
| 145 | } | 149 | } |
| 146 | } | 150 | } |
| @@ -1054,14 +1058,6 @@ NORET_TYPE void do_exit(long code) | |||
| 1054 | exit_itimers(tsk->signal); | 1058 | exit_itimers(tsk->signal); |
| 1055 | } | 1059 | } |
| 1056 | acct_collect(code, group_dead); | 1060 | acct_collect(code, group_dead); |
| 1057 | #ifdef CONFIG_FUTEX | ||
| 1058 | if (unlikely(tsk->robust_list)) | ||
| 1059 | exit_robust_list(tsk); | ||
| 1060 | #ifdef CONFIG_COMPAT | ||
| 1061 | if (unlikely(tsk->compat_robust_list)) | ||
| 1062 | compat_exit_robust_list(tsk); | ||
| 1063 | #endif | ||
| 1064 | #endif | ||
| 1065 | if (group_dead) | 1061 | if (group_dead) |
| 1066 | tty_audit_exit(); | 1062 | tty_audit_exit(); |
| 1067 | if (unlikely(tsk->audit_context)) | 1063 | if (unlikely(tsk->audit_context)) |
diff --git a/kernel/fork.c b/kernel/fork.c index f6083561dfe0..2a372a0e206f 100644 --- a/kernel/fork.c +++ b/kernel/fork.c | |||
| @@ -40,6 +40,7 @@ | |||
| 40 | #include <linux/jiffies.h> | 40 | #include <linux/jiffies.h> |
| 41 | #include <linux/tracehook.h> | 41 | #include <linux/tracehook.h> |
| 42 | #include <linux/futex.h> | 42 | #include <linux/futex.h> |
| 43 | #include <linux/compat.h> | ||
| 43 | #include <linux/task_io_accounting_ops.h> | 44 | #include <linux/task_io_accounting_ops.h> |
| 44 | #include <linux/rcupdate.h> | 45 | #include <linux/rcupdate.h> |
| 45 | #include <linux/ptrace.h> | 46 | #include <linux/ptrace.h> |
| @@ -519,6 +520,16 @@ void mm_release(struct task_struct *tsk, struct mm_struct *mm) | |||
| 519 | { | 520 | { |
| 520 | struct completion *vfork_done = tsk->vfork_done; | 521 | struct completion *vfork_done = tsk->vfork_done; |
| 521 | 522 | ||
| 523 | /* Get rid of any futexes when releasing the mm */ | ||
| 524 | #ifdef CONFIG_FUTEX | ||
| 525 | if (unlikely(tsk->robust_list)) | ||
| 526 | exit_robust_list(tsk); | ||
| 527 | #ifdef CONFIG_COMPAT | ||
| 528 | if (unlikely(tsk->compat_robust_list)) | ||
| 529 | compat_exit_robust_list(tsk); | ||
| 530 | #endif | ||
| 531 | #endif | ||
| 532 | |||
| 522 | /* Get rid of any cached register state */ | 533 | /* Get rid of any cached register state */ |
| 523 | deactivate_mm(tsk, mm); | 534 | deactivate_mm(tsk, mm); |
| 524 | 535 | ||
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 2b465dfde426..47e63349d1b2 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c | |||
| @@ -664,14 +664,6 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, | |||
| 664 | 664 | ||
| 665 | /* Timer is expired, act upon the callback mode */ | 665 | /* Timer is expired, act upon the callback mode */ |
| 666 | switch(timer->cb_mode) { | 666 | switch(timer->cb_mode) { |
| 667 | case HRTIMER_CB_IRQSAFE_NO_RESTART: | ||
| 668 | debug_hrtimer_deactivate(timer); | ||
| 669 | /* | ||
| 670 | * We can call the callback from here. No restart | ||
| 671 | * happens, so no danger of recursion | ||
| 672 | */ | ||
| 673 | BUG_ON(timer->function(timer) != HRTIMER_NORESTART); | ||
| 674 | return 1; | ||
| 675 | case HRTIMER_CB_IRQSAFE_PERCPU: | 667 | case HRTIMER_CB_IRQSAFE_PERCPU: |
| 676 | case HRTIMER_CB_IRQSAFE_UNLOCKED: | 668 | case HRTIMER_CB_IRQSAFE_UNLOCKED: |
| 677 | /* | 669 | /* |
| @@ -683,7 +675,6 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, | |||
| 683 | */ | 675 | */ |
| 684 | debug_hrtimer_deactivate(timer); | 676 | debug_hrtimer_deactivate(timer); |
| 685 | return 1; | 677 | return 1; |
| 686 | case HRTIMER_CB_IRQSAFE: | ||
| 687 | case HRTIMER_CB_SOFTIRQ: | 678 | case HRTIMER_CB_SOFTIRQ: |
| 688 | /* | 679 | /* |
| 689 | * Move everything else into the softirq pending list ! | 680 | * Move everything else into the softirq pending list ! |
| @@ -1209,6 +1200,7 @@ static void run_hrtimer_pending(struct hrtimer_cpu_base *cpu_base) | |||
| 1209 | enum hrtimer_restart (*fn)(struct hrtimer *); | 1200 | enum hrtimer_restart (*fn)(struct hrtimer *); |
| 1210 | struct hrtimer *timer; | 1201 | struct hrtimer *timer; |
| 1211 | int restart; | 1202 | int restart; |
| 1203 | int emulate_hardirq_ctx = 0; | ||
| 1212 | 1204 | ||
| 1213 | timer = list_entry(cpu_base->cb_pending.next, | 1205 | timer = list_entry(cpu_base->cb_pending.next, |
| 1214 | struct hrtimer, cb_entry); | 1206 | struct hrtimer, cb_entry); |
| @@ -1217,10 +1209,24 @@ static void run_hrtimer_pending(struct hrtimer_cpu_base *cpu_base) | |||
| 1217 | timer_stats_account_hrtimer(timer); | 1209 | timer_stats_account_hrtimer(timer); |
| 1218 | 1210 | ||
| 1219 | fn = timer->function; | 1211 | fn = timer->function; |
| 1212 | /* | ||
| 1213 | * A timer might have been added to the cb_pending list | ||
| 1214 | * when it was migrated during a cpu-offline operation. | ||
| 1215 | * Emulate hardirq context for such timers. | ||
| 1216 | */ | ||
| 1217 | if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU || | ||
| 1218 | timer->cb_mode == HRTIMER_CB_IRQSAFE_UNLOCKED) | ||
| 1219 | emulate_hardirq_ctx = 1; | ||
| 1220 | |||
| 1220 | __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0); | 1221 | __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0); |
| 1221 | spin_unlock_irq(&cpu_base->lock); | 1222 | spin_unlock_irq(&cpu_base->lock); |
| 1222 | 1223 | ||
| 1223 | restart = fn(timer); | 1224 | if (unlikely(emulate_hardirq_ctx)) { |
| 1225 | local_irq_disable(); | ||
| 1226 | restart = fn(timer); | ||
| 1227 | local_irq_enable(); | ||
| 1228 | } else | ||
| 1229 | restart = fn(timer); | ||
| 1224 | 1230 | ||
| 1225 | spin_lock_irq(&cpu_base->lock); | 1231 | spin_lock_irq(&cpu_base->lock); |
| 1226 | 1232 | ||
diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 5072cf1685a2..7b8b0f21a5b1 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c | |||
| @@ -304,17 +304,24 @@ int sprint_symbol(char *buffer, unsigned long address) | |||
| 304 | char *modname; | 304 | char *modname; |
| 305 | const char *name; | 305 | const char *name; |
| 306 | unsigned long offset, size; | 306 | unsigned long offset, size; |
| 307 | char namebuf[KSYM_NAME_LEN]; | 307 | int len; |
| 308 | 308 | ||
| 309 | name = kallsyms_lookup(address, &size, &offset, &modname, namebuf); | 309 | name = kallsyms_lookup(address, &size, &offset, &modname, buffer); |
| 310 | if (!name) | 310 | if (!name) |
| 311 | return sprintf(buffer, "0x%lx", address); | 311 | return sprintf(buffer, "0x%lx", address); |
| 312 | 312 | ||
| 313 | if (name != buffer) | ||
| 314 | strcpy(buffer, name); | ||
| 315 | len = strlen(buffer); | ||
| 316 | buffer += len; | ||
| 317 | |||
| 313 | if (modname) | 318 | if (modname) |
| 314 | return sprintf(buffer, "%s+%#lx/%#lx [%s]", name, offset, | 319 | len += sprintf(buffer, "+%#lx/%#lx [%s]", |
| 315 | size, modname); | 320 | offset, size, modname); |
| 316 | else | 321 | else |
| 317 | return sprintf(buffer, "%s+%#lx/%#lx", name, offset, size); | 322 | len += sprintf(buffer, "+%#lx/%#lx", offset, size); |
| 323 | |||
| 324 | return len; | ||
| 318 | } | 325 | } |
| 319 | 326 | ||
| 320 | /* Look up a kernel symbol and print it to the kernel messages. */ | 327 | /* Look up a kernel symbol and print it to the kernel messages. */ |
diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 8b57a2597f21..9f8a3f25259a 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c | |||
| @@ -72,7 +72,7 @@ static bool kprobe_enabled; | |||
| 72 | DEFINE_MUTEX(kprobe_mutex); /* Protects kprobe_table */ | 72 | DEFINE_MUTEX(kprobe_mutex); /* Protects kprobe_table */ |
| 73 | static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; | 73 | static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; |
| 74 | static struct { | 74 | static struct { |
| 75 | spinlock_t lock ____cacheline_aligned; | 75 | spinlock_t lock ____cacheline_aligned_in_smp; |
| 76 | } kretprobe_table_locks[KPROBE_TABLE_SIZE]; | 76 | } kretprobe_table_locks[KPROBE_TABLE_SIZE]; |
| 77 | 77 | ||
| 78 | static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash) | 78 | static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash) |
| @@ -613,30 +613,37 @@ static int __kprobes __register_kprobe(struct kprobe *p, | |||
| 613 | return -EINVAL; | 613 | return -EINVAL; |
| 614 | p->addr = addr; | 614 | p->addr = addr; |
| 615 | 615 | ||
| 616 | if (!kernel_text_address((unsigned long) p->addr) || | 616 | preempt_disable(); |
| 617 | in_kprobes_functions((unsigned long) p->addr)) | 617 | if (!__kernel_text_address((unsigned long) p->addr) || |
| 618 | in_kprobes_functions((unsigned long) p->addr)) { | ||
| 619 | preempt_enable(); | ||
| 618 | return -EINVAL; | 620 | return -EINVAL; |
| 621 | } | ||
| 619 | 622 | ||
| 620 | p->mod_refcounted = 0; | 623 | p->mod_refcounted = 0; |
| 621 | 624 | ||
| 622 | /* | 625 | /* |
| 623 | * Check if are we probing a module. | 626 | * Check if are we probing a module. |
| 624 | */ | 627 | */ |
| 625 | probed_mod = module_text_address((unsigned long) p->addr); | 628 | probed_mod = __module_text_address((unsigned long) p->addr); |
| 626 | if (probed_mod) { | 629 | if (probed_mod) { |
| 627 | struct module *calling_mod = module_text_address(called_from); | 630 | struct module *calling_mod; |
| 631 | calling_mod = __module_text_address(called_from); | ||
| 628 | /* | 632 | /* |
| 629 | * We must allow modules to probe themself and in this case | 633 | * We must allow modules to probe themself and in this case |
| 630 | * avoid incrementing the module refcount, so as to allow | 634 | * avoid incrementing the module refcount, so as to allow |
| 631 | * unloading of self probing modules. | 635 | * unloading of self probing modules. |
| 632 | */ | 636 | */ |
| 633 | if (calling_mod && calling_mod != probed_mod) { | 637 | if (calling_mod && calling_mod != probed_mod) { |
| 634 | if (unlikely(!try_module_get(probed_mod))) | 638 | if (unlikely(!try_module_get(probed_mod))) { |
| 639 | preempt_enable(); | ||
| 635 | return -EINVAL; | 640 | return -EINVAL; |
| 641 | } | ||
| 636 | p->mod_refcounted = 1; | 642 | p->mod_refcounted = 1; |
| 637 | } else | 643 | } else |
| 638 | probed_mod = NULL; | 644 | probed_mod = NULL; |
| 639 | } | 645 | } |
| 646 | preempt_enable(); | ||
| 640 | 647 | ||
| 641 | p->nmissed = 0; | 648 | p->nmissed = 0; |
| 642 | INIT_LIST_HEAD(&p->list); | 649 | INIT_LIST_HEAD(&p->list); |
| @@ -718,6 +725,10 @@ static void __kprobes __unregister_kprobe_bottom(struct kprobe *p) | |||
| 718 | struct kprobe *old_p; | 725 | struct kprobe *old_p; |
| 719 | 726 | ||
| 720 | if (p->mod_refcounted) { | 727 | if (p->mod_refcounted) { |
| 728 | /* | ||
| 729 | * Since we've already incremented refcount, | ||
| 730 | * we don't need to disable preemption. | ||
| 731 | */ | ||
| 721 | mod = module_text_address((unsigned long)p->addr); | 732 | mod = module_text_address((unsigned long)p->addr); |
| 722 | if (mod) | 733 | if (mod) |
| 723 | module_put(mod); | 734 | module_put(mod); |
diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c index 153dcb2639c3..895337b16a24 100644 --- a/kernel/posix-cpu-timers.c +++ b/kernel/posix-cpu-timers.c | |||
| @@ -1308,9 +1308,10 @@ static inline int task_cputime_expired(const struct task_cputime *sample, | |||
| 1308 | */ | 1308 | */ |
| 1309 | static inline int fastpath_timer_check(struct task_struct *tsk) | 1309 | static inline int fastpath_timer_check(struct task_struct *tsk) |
| 1310 | { | 1310 | { |
| 1311 | struct signal_struct *sig = tsk->signal; | 1311 | struct signal_struct *sig; |
| 1312 | 1312 | ||
| 1313 | if (unlikely(!sig)) | 1313 | /* tsk == current, ensure it is safe to use ->signal/sighand */ |
| 1314 | if (unlikely(tsk->exit_state)) | ||
| 1314 | return 0; | 1315 | return 0; |
| 1315 | 1316 | ||
| 1316 | if (!task_cputime_zero(&tsk->cputime_expires)) { | 1317 | if (!task_cputime_zero(&tsk->cputime_expires)) { |
| @@ -1323,6 +1324,8 @@ static inline int fastpath_timer_check(struct task_struct *tsk) | |||
| 1323 | if (task_cputime_expired(&task_sample, &tsk->cputime_expires)) | 1324 | if (task_cputime_expired(&task_sample, &tsk->cputime_expires)) |
| 1324 | return 1; | 1325 | return 1; |
| 1325 | } | 1326 | } |
| 1327 | |||
| 1328 | sig = tsk->signal; | ||
| 1326 | if (!task_cputime_zero(&sig->cputime_expires)) { | 1329 | if (!task_cputime_zero(&sig->cputime_expires)) { |
| 1327 | struct task_cputime group_sample; | 1330 | struct task_cputime group_sample; |
| 1328 | 1331 | ||
diff --git a/kernel/power/main.c b/kernel/power/main.c index 19122cf6d827..b8f7ce9473e8 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c | |||
| @@ -174,7 +174,7 @@ static void suspend_test_finish(const char *label) | |||
| 174 | * has some performance issues. The stack dump of a WARN_ON | 174 | * has some performance issues. The stack dump of a WARN_ON |
| 175 | * is more likely to get the right attention than a printk... | 175 | * is more likely to get the right attention than a printk... |
| 176 | */ | 176 | */ |
| 177 | WARN_ON(msec > (TEST_SUSPEND_SECONDS * 1000)); | 177 | WARN(msec > (TEST_SUSPEND_SECONDS * 1000), "Component: %s\n", label); |
| 178 | } | 178 | } |
| 179 | 179 | ||
| 180 | #else | 180 | #else |
diff --git a/kernel/profile.c b/kernel/profile.c index 9830a037d8db..5b7d1ac7124c 100644 --- a/kernel/profile.c +++ b/kernel/profile.c | |||
| @@ -544,7 +544,7 @@ static const struct file_operations proc_profile_operations = { | |||
| 544 | }; | 544 | }; |
| 545 | 545 | ||
| 546 | #ifdef CONFIG_SMP | 546 | #ifdef CONFIG_SMP |
| 547 | static void __init profile_nop(void *unused) | 547 | static inline void profile_nop(void *unused) |
| 548 | { | 548 | { |
| 549 | } | 549 | } |
| 550 | 550 | ||
diff --git a/kernel/relay.c b/kernel/relay.c index 8d13a7855c08..32b0befdcb6a 100644 --- a/kernel/relay.c +++ b/kernel/relay.c | |||
| @@ -400,7 +400,7 @@ void relay_reset(struct rchan *chan) | |||
| 400 | } | 400 | } |
| 401 | 401 | ||
| 402 | mutex_lock(&relay_channels_mutex); | 402 | mutex_lock(&relay_channels_mutex); |
| 403 | for_each_online_cpu(i) | 403 | for_each_possible_cpu(i) |
| 404 | if (chan->buf[i]) | 404 | if (chan->buf[i]) |
| 405 | __relay_reset(chan->buf[i], 0); | 405 | __relay_reset(chan->buf[i], 0); |
| 406 | mutex_unlock(&relay_channels_mutex); | 406 | mutex_unlock(&relay_channels_mutex); |
| @@ -611,10 +611,9 @@ struct rchan *relay_open(const char *base_filename, | |||
| 611 | return chan; | 611 | return chan; |
| 612 | 612 | ||
| 613 | free_bufs: | 613 | free_bufs: |
| 614 | for_each_online_cpu(i) { | 614 | for_each_possible_cpu(i) { |
| 615 | if (!chan->buf[i]) | 615 | if (chan->buf[i]) |
| 616 | break; | 616 | relay_close_buf(chan->buf[i]); |
| 617 | relay_close_buf(chan->buf[i]); | ||
| 618 | } | 617 | } |
| 619 | 618 | ||
| 620 | kref_put(&chan->kref, relay_destroy_channel); | 619 | kref_put(&chan->kref, relay_destroy_channel); |
diff --git a/kernel/sched.c b/kernel/sched.c index ad10d0aae1d7..338340a3fb89 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
| @@ -397,9 +397,9 @@ struct cfs_rq { | |||
| 397 | * 'curr' points to currently running entity on this cfs_rq. | 397 | * 'curr' points to currently running entity on this cfs_rq. |
| 398 | * It is set to NULL otherwise (i.e when none are currently running). | 398 | * It is set to NULL otherwise (i.e when none are currently running). |
| 399 | */ | 399 | */ |
| 400 | struct sched_entity *curr, *next; | 400 | struct sched_entity *curr, *next, *last; |
| 401 | 401 | ||
| 402 | unsigned long nr_spread_over; | 402 | unsigned int nr_spread_over; |
| 403 | 403 | ||
| 404 | #ifdef CONFIG_FAIR_GROUP_SCHED | 404 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 405 | struct rq *rq; /* cpu runqueue to which this cfs_rq is attached */ | 405 | struct rq *rq; /* cpu runqueue to which this cfs_rq is attached */ |
| @@ -703,45 +703,18 @@ static __read_mostly char *sched_feat_names[] = { | |||
| 703 | 703 | ||
| 704 | #undef SCHED_FEAT | 704 | #undef SCHED_FEAT |
| 705 | 705 | ||
| 706 | static int sched_feat_open(struct inode *inode, struct file *filp) | 706 | static int sched_feat_show(struct seq_file *m, void *v) |
| 707 | { | ||
| 708 | filp->private_data = inode->i_private; | ||
| 709 | return 0; | ||
| 710 | } | ||
| 711 | |||
| 712 | static ssize_t | ||
| 713 | sched_feat_read(struct file *filp, char __user *ubuf, | ||
| 714 | size_t cnt, loff_t *ppos) | ||
| 715 | { | 707 | { |
| 716 | char *buf; | ||
| 717 | int r = 0; | ||
| 718 | int len = 0; | ||
| 719 | int i; | 708 | int i; |
| 720 | 709 | ||
| 721 | for (i = 0; sched_feat_names[i]; i++) { | 710 | for (i = 0; sched_feat_names[i]; i++) { |
| 722 | len += strlen(sched_feat_names[i]); | 711 | if (!(sysctl_sched_features & (1UL << i))) |
| 723 | len += 4; | 712 | seq_puts(m, "NO_"); |
| 713 | seq_printf(m, "%s ", sched_feat_names[i]); | ||
| 724 | } | 714 | } |
| 715 | seq_puts(m, "\n"); | ||
| 725 | 716 | ||
| 726 | buf = kmalloc(len + 2, GFP_KERNEL); | 717 | return 0; |
| 727 | if (!buf) | ||
| 728 | return -ENOMEM; | ||
| 729 | |||
| 730 | for (i = 0; sched_feat_names[i]; i++) { | ||
| 731 | if (sysctl_sched_features & (1UL << i)) | ||
| 732 | r += sprintf(buf + r, "%s ", sched_feat_names[i]); | ||
| 733 | else | ||
| 734 | r += sprintf(buf + r, "NO_%s ", sched_feat_names[i]); | ||
| 735 | } | ||
| 736 | |||
| 737 | r += sprintf(buf + r, "\n"); | ||
| 738 | WARN_ON(r >= len + 2); | ||
| 739 | |||
| 740 | r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); | ||
| 741 | |||
| 742 | kfree(buf); | ||
| 743 | |||
| 744 | return r; | ||
| 745 | } | 718 | } |
| 746 | 719 | ||
| 747 | static ssize_t | 720 | static ssize_t |
| @@ -786,10 +759,17 @@ sched_feat_write(struct file *filp, const char __user *ubuf, | |||
| 786 | return cnt; | 759 | return cnt; |
| 787 | } | 760 | } |
| 788 | 761 | ||
| 762 | static int sched_feat_open(struct inode *inode, struct file *filp) | ||
| 763 | { | ||
| 764 | return single_open(filp, sched_feat_show, NULL); | ||
| 765 | } | ||
| 766 | |||
| 789 | static struct file_operations sched_feat_fops = { | 767 | static struct file_operations sched_feat_fops = { |
| 790 | .open = sched_feat_open, | 768 | .open = sched_feat_open, |
| 791 | .read = sched_feat_read, | 769 | .write = sched_feat_write, |
| 792 | .write = sched_feat_write, | 770 | .read = seq_read, |
| 771 | .llseek = seq_lseek, | ||
| 772 | .release = single_release, | ||
| 793 | }; | 773 | }; |
| 794 | 774 | ||
| 795 | static __init int sched_init_debug(void) | 775 | static __init int sched_init_debug(void) |
| @@ -969,6 +949,14 @@ static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags) | |||
| 969 | } | 949 | } |
| 970 | } | 950 | } |
| 971 | 951 | ||
| 952 | void task_rq_unlock_wait(struct task_struct *p) | ||
| 953 | { | ||
| 954 | struct rq *rq = task_rq(p); | ||
| 955 | |||
| 956 | smp_mb(); /* spin-unlock-wait is not a full memory barrier */ | ||
| 957 | spin_unlock_wait(&rq->lock); | ||
| 958 | } | ||
| 959 | |||
| 972 | static void __task_rq_unlock(struct rq *rq) | 960 | static void __task_rq_unlock(struct rq *rq) |
| 973 | __releases(rq->lock) | 961 | __releases(rq->lock) |
| 974 | { | 962 | { |
| @@ -1448,6 +1436,8 @@ static unsigned long cpu_avg_load_per_task(int cpu) | |||
| 1448 | 1436 | ||
| 1449 | if (rq->nr_running) | 1437 | if (rq->nr_running) |
| 1450 | rq->avg_load_per_task = rq->load.weight / rq->nr_running; | 1438 | rq->avg_load_per_task = rq->load.weight / rq->nr_running; |
| 1439 | else | ||
| 1440 | rq->avg_load_per_task = 0; | ||
| 1451 | 1441 | ||
| 1452 | return rq->avg_load_per_task; | 1442 | return rq->avg_load_per_task; |
| 1453 | } | 1443 | } |
| @@ -1463,27 +1453,13 @@ static void | |||
| 1463 | update_group_shares_cpu(struct task_group *tg, int cpu, | 1453 | update_group_shares_cpu(struct task_group *tg, int cpu, |
| 1464 | unsigned long sd_shares, unsigned long sd_rq_weight) | 1454 | unsigned long sd_shares, unsigned long sd_rq_weight) |
| 1465 | { | 1455 | { |
| 1466 | int boost = 0; | ||
| 1467 | unsigned long shares; | 1456 | unsigned long shares; |
| 1468 | unsigned long rq_weight; | 1457 | unsigned long rq_weight; |
| 1469 | 1458 | ||
| 1470 | if (!tg->se[cpu]) | 1459 | if (!tg->se[cpu]) |
| 1471 | return; | 1460 | return; |
| 1472 | 1461 | ||
| 1473 | rq_weight = tg->cfs_rq[cpu]->load.weight; | 1462 | rq_weight = tg->cfs_rq[cpu]->rq_weight; |
| 1474 | |||
| 1475 | /* | ||
| 1476 | * If there are currently no tasks on the cpu pretend there is one of | ||
| 1477 | * average load so that when a new task gets to run here it will not | ||
| 1478 | * get delayed by group starvation. | ||
| 1479 | */ | ||
| 1480 | if (!rq_weight) { | ||
| 1481 | boost = 1; | ||
| 1482 | rq_weight = NICE_0_LOAD; | ||
| 1483 | } | ||
| 1484 | |||
| 1485 | if (unlikely(rq_weight > sd_rq_weight)) | ||
| 1486 | rq_weight = sd_rq_weight; | ||
| 1487 | 1463 | ||
| 1488 | /* | 1464 | /* |
| 1489 | * \Sum shares * rq_weight | 1465 | * \Sum shares * rq_weight |
| @@ -1491,7 +1467,7 @@ update_group_shares_cpu(struct task_group *tg, int cpu, | |||
| 1491 | * \Sum rq_weight | 1467 | * \Sum rq_weight |
| 1492 | * | 1468 | * |
| 1493 | */ | 1469 | */ |
| 1494 | shares = (sd_shares * rq_weight) / (sd_rq_weight + 1); | 1470 | shares = (sd_shares * rq_weight) / sd_rq_weight; |
| 1495 | shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES); | 1471 | shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES); |
| 1496 | 1472 | ||
| 1497 | if (abs(shares - tg->se[cpu]->load.weight) > | 1473 | if (abs(shares - tg->se[cpu]->load.weight) > |
| @@ -1500,11 +1476,7 @@ update_group_shares_cpu(struct task_group *tg, int cpu, | |||
| 1500 | unsigned long flags; | 1476 | unsigned long flags; |
| 1501 | 1477 | ||
| 1502 | spin_lock_irqsave(&rq->lock, flags); | 1478 | spin_lock_irqsave(&rq->lock, flags); |
| 1503 | /* | 1479 | tg->cfs_rq[cpu]->shares = shares; |
| 1504 | * record the actual number of shares, not the boosted amount. | ||
| 1505 | */ | ||
| 1506 | tg->cfs_rq[cpu]->shares = boost ? 0 : shares; | ||
| 1507 | tg->cfs_rq[cpu]->rq_weight = rq_weight; | ||
| 1508 | 1480 | ||
| 1509 | __set_se_shares(tg->se[cpu], shares); | 1481 | __set_se_shares(tg->se[cpu], shares); |
| 1510 | spin_unlock_irqrestore(&rq->lock, flags); | 1482 | spin_unlock_irqrestore(&rq->lock, flags); |
| @@ -1518,13 +1490,23 @@ update_group_shares_cpu(struct task_group *tg, int cpu, | |||
| 1518 | */ | 1490 | */ |
| 1519 | static int tg_shares_up(struct task_group *tg, void *data) | 1491 | static int tg_shares_up(struct task_group *tg, void *data) |
| 1520 | { | 1492 | { |
| 1521 | unsigned long rq_weight = 0; | 1493 | unsigned long weight, rq_weight = 0; |
| 1522 | unsigned long shares = 0; | 1494 | unsigned long shares = 0; |
| 1523 | struct sched_domain *sd = data; | 1495 | struct sched_domain *sd = data; |
| 1524 | int i; | 1496 | int i; |
| 1525 | 1497 | ||
| 1526 | for_each_cpu_mask(i, sd->span) { | 1498 | for_each_cpu_mask(i, sd->span) { |
| 1527 | rq_weight += tg->cfs_rq[i]->load.weight; | 1499 | /* |
| 1500 | * If there are currently no tasks on the cpu pretend there | ||
| 1501 | * is one of average load so that when a new task gets to | ||
| 1502 | * run here it will not get delayed by group starvation. | ||
| 1503 | */ | ||
| 1504 | weight = tg->cfs_rq[i]->load.weight; | ||
| 1505 | if (!weight) | ||
| 1506 | weight = NICE_0_LOAD; | ||
| 1507 | |||
| 1508 | tg->cfs_rq[i]->rq_weight = weight; | ||
| 1509 | rq_weight += weight; | ||
| 1528 | shares += tg->cfs_rq[i]->shares; | 1510 | shares += tg->cfs_rq[i]->shares; |
| 1529 | } | 1511 | } |
| 1530 | 1512 | ||
| @@ -1534,9 +1516,6 @@ static int tg_shares_up(struct task_group *tg, void *data) | |||
| 1534 | if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE)) | 1516 | if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE)) |
| 1535 | shares = tg->shares; | 1517 | shares = tg->shares; |
| 1536 | 1518 | ||
| 1537 | if (!rq_weight) | ||
| 1538 | rq_weight = cpus_weight(sd->span) * NICE_0_LOAD; | ||
| 1539 | |||
| 1540 | for_each_cpu_mask(i, sd->span) | 1519 | for_each_cpu_mask(i, sd->span) |
| 1541 | update_group_shares_cpu(tg, i, shares, rq_weight); | 1520 | update_group_shares_cpu(tg, i, shares, rq_weight); |
| 1542 | 1521 | ||
| @@ -1805,7 +1784,9 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) | |||
| 1805 | /* | 1784 | /* |
| 1806 | * Buddy candidates are cache hot: | 1785 | * Buddy candidates are cache hot: |
| 1807 | */ | 1786 | */ |
| 1808 | if (sched_feat(CACHE_HOT_BUDDY) && (&p->se == cfs_rq_of(&p->se)->next)) | 1787 | if (sched_feat(CACHE_HOT_BUDDY) && |
| 1788 | (&p->se == cfs_rq_of(&p->se)->next || | ||
| 1789 | &p->se == cfs_rq_of(&p->se)->last)) | ||
| 1809 | return 1; | 1790 | return 1; |
| 1810 | 1791 | ||
| 1811 | if (p->sched_class != &fair_sched_class) | 1792 | if (p->sched_class != &fair_sched_class) |
| @@ -5858,6 +5839,8 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu) | |||
| 5858 | struct rq *rq = cpu_rq(cpu); | 5839 | struct rq *rq = cpu_rq(cpu); |
| 5859 | unsigned long flags; | 5840 | unsigned long flags; |
| 5860 | 5841 | ||
| 5842 | spin_lock_irqsave(&rq->lock, flags); | ||
| 5843 | |||
| 5861 | __sched_fork(idle); | 5844 | __sched_fork(idle); |
| 5862 | idle->se.exec_start = sched_clock(); | 5845 | idle->se.exec_start = sched_clock(); |
| 5863 | 5846 | ||
| @@ -5865,7 +5848,6 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu) | |||
| 5865 | idle->cpus_allowed = cpumask_of_cpu(cpu); | 5848 | idle->cpus_allowed = cpumask_of_cpu(cpu); |
| 5866 | __set_task_cpu(idle, cpu); | 5849 | __set_task_cpu(idle, cpu); |
| 5867 | 5850 | ||
| 5868 | spin_lock_irqsave(&rq->lock, flags); | ||
| 5869 | rq->curr = rq->idle = idle; | 5851 | rq->curr = rq->idle = idle; |
| 5870 | #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW) | 5852 | #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW) |
| 5871 | idle->oncpu = 1; | 5853 | idle->oncpu = 1; |
| @@ -6112,7 +6094,6 @@ static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu) | |||
| 6112 | 6094 | ||
| 6113 | /* | 6095 | /* |
| 6114 | * Figure out where task on dead CPU should go, use force if necessary. | 6096 | * Figure out where task on dead CPU should go, use force if necessary. |
| 6115 | * NOTE: interrupts should be disabled by the caller | ||
| 6116 | */ | 6097 | */ |
| 6117 | static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p) | 6098 | static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p) |
| 6118 | { | 6099 | { |
| @@ -6622,28 +6603,6 @@ early_initcall(migration_init); | |||
| 6622 | 6603 | ||
| 6623 | #ifdef CONFIG_SCHED_DEBUG | 6604 | #ifdef CONFIG_SCHED_DEBUG |
| 6624 | 6605 | ||
| 6625 | static inline const char *sd_level_to_string(enum sched_domain_level lvl) | ||
| 6626 | { | ||
| 6627 | switch (lvl) { | ||
| 6628 | case SD_LV_NONE: | ||
| 6629 | return "NONE"; | ||
| 6630 | case SD_LV_SIBLING: | ||
| 6631 | return "SIBLING"; | ||
| 6632 | case SD_LV_MC: | ||
| 6633 | return "MC"; | ||
| 6634 | case SD_LV_CPU: | ||
| 6635 | return "CPU"; | ||
| 6636 | case SD_LV_NODE: | ||
| 6637 | return "NODE"; | ||
| 6638 | case SD_LV_ALLNODES: | ||
| 6639 | return "ALLNODES"; | ||
| 6640 | case SD_LV_MAX: | ||
| 6641 | return "MAX"; | ||
| 6642 | |||
| 6643 | } | ||
| 6644 | return "MAX"; | ||
| 6645 | } | ||
| 6646 | |||
| 6647 | static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, | 6606 | static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, |
| 6648 | cpumask_t *groupmask) | 6607 | cpumask_t *groupmask) |
| 6649 | { | 6608 | { |
| @@ -6663,8 +6622,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, | |||
| 6663 | return -1; | 6622 | return -1; |
| 6664 | } | 6623 | } |
| 6665 | 6624 | ||
| 6666 | printk(KERN_CONT "span %s level %s\n", | 6625 | printk(KERN_CONT "span %s level %s\n", str, sd->name); |
| 6667 | str, sd_level_to_string(sd->level)); | ||
| 6668 | 6626 | ||
| 6669 | if (!cpu_isset(cpu, sd->span)) { | 6627 | if (!cpu_isset(cpu, sd->span)) { |
| 6670 | printk(KERN_ERR "ERROR: domain->span does not contain " | 6628 | printk(KERN_ERR "ERROR: domain->span does not contain " |
| @@ -6875,15 +6833,17 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) | |||
| 6875 | struct sched_domain *tmp; | 6833 | struct sched_domain *tmp; |
| 6876 | 6834 | ||
| 6877 | /* Remove the sched domains which do not contribute to scheduling. */ | 6835 | /* Remove the sched domains which do not contribute to scheduling. */ |
| 6878 | for (tmp = sd; tmp; tmp = tmp->parent) { | 6836 | for (tmp = sd; tmp; ) { |
| 6879 | struct sched_domain *parent = tmp->parent; | 6837 | struct sched_domain *parent = tmp->parent; |
| 6880 | if (!parent) | 6838 | if (!parent) |
| 6881 | break; | 6839 | break; |
| 6840 | |||
| 6882 | if (sd_parent_degenerate(tmp, parent)) { | 6841 | if (sd_parent_degenerate(tmp, parent)) { |
| 6883 | tmp->parent = parent->parent; | 6842 | tmp->parent = parent->parent; |
| 6884 | if (parent->parent) | 6843 | if (parent->parent) |
| 6885 | parent->parent->child = tmp; | 6844 | parent->parent->child = tmp; |
| 6886 | } | 6845 | } else |
| 6846 | tmp = tmp->parent; | ||
| 6887 | } | 6847 | } |
| 6888 | 6848 | ||
| 6889 | if (sd && sd_degenerate(sd)) { | 6849 | if (sd && sd_degenerate(sd)) { |
| @@ -7318,13 +7278,21 @@ struct allmasks { | |||
| 7318 | }; | 7278 | }; |
| 7319 | 7279 | ||
| 7320 | #if NR_CPUS > 128 | 7280 | #if NR_CPUS > 128 |
| 7321 | #define SCHED_CPUMASK_ALLOC 1 | 7281 | #define SCHED_CPUMASK_DECLARE(v) struct allmasks *v |
| 7322 | #define SCHED_CPUMASK_FREE(v) kfree(v) | 7282 | static inline void sched_cpumask_alloc(struct allmasks **masks) |
| 7323 | #define SCHED_CPUMASK_DECLARE(v) struct allmasks *v | 7283 | { |
| 7284 | *masks = kmalloc(sizeof(**masks), GFP_KERNEL); | ||
| 7285 | } | ||
| 7286 | static inline void sched_cpumask_free(struct allmasks *masks) | ||
| 7287 | { | ||
| 7288 | kfree(masks); | ||
| 7289 | } | ||
| 7324 | #else | 7290 | #else |
| 7325 | #define SCHED_CPUMASK_ALLOC 0 | 7291 | #define SCHED_CPUMASK_DECLARE(v) struct allmasks _v, *v = &_v |
| 7326 | #define SCHED_CPUMASK_FREE(v) | 7292 | static inline void sched_cpumask_alloc(struct allmasks **masks) |
| 7327 | #define SCHED_CPUMASK_DECLARE(v) struct allmasks _v, *v = &_v | 7293 | { } |
| 7294 | static inline void sched_cpumask_free(struct allmasks *masks) | ||
| 7295 | { } | ||
| 7328 | #endif | 7296 | #endif |
| 7329 | 7297 | ||
| 7330 | #define SCHED_CPUMASK_VAR(v, a) cpumask_t *v = (cpumask_t *) \ | 7298 | #define SCHED_CPUMASK_VAR(v, a) cpumask_t *v = (cpumask_t *) \ |
| @@ -7400,9 +7368,8 @@ static int __build_sched_domains(const cpumask_t *cpu_map, | |||
| 7400 | return -ENOMEM; | 7368 | return -ENOMEM; |
| 7401 | } | 7369 | } |
| 7402 | 7370 | ||
| 7403 | #if SCHED_CPUMASK_ALLOC | ||
| 7404 | /* get space for all scratch cpumask variables */ | 7371 | /* get space for all scratch cpumask variables */ |
| 7405 | allmasks = kmalloc(sizeof(*allmasks), GFP_KERNEL); | 7372 | sched_cpumask_alloc(&allmasks); |
| 7406 | if (!allmasks) { | 7373 | if (!allmasks) { |
| 7407 | printk(KERN_WARNING "Cannot alloc cpumask array\n"); | 7374 | printk(KERN_WARNING "Cannot alloc cpumask array\n"); |
| 7408 | kfree(rd); | 7375 | kfree(rd); |
| @@ -7411,7 +7378,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map, | |||
| 7411 | #endif | 7378 | #endif |
| 7412 | return -ENOMEM; | 7379 | return -ENOMEM; |
| 7413 | } | 7380 | } |
| 7414 | #endif | 7381 | |
| 7415 | tmpmask = (cpumask_t *)allmasks; | 7382 | tmpmask = (cpumask_t *)allmasks; |
| 7416 | 7383 | ||
| 7417 | 7384 | ||
| @@ -7665,13 +7632,14 @@ static int __build_sched_domains(const cpumask_t *cpu_map, | |||
| 7665 | cpu_attach_domain(sd, rd, i); | 7632 | cpu_attach_domain(sd, rd, i); |
| 7666 | } | 7633 | } |
| 7667 | 7634 | ||
| 7668 | SCHED_CPUMASK_FREE((void *)allmasks); | 7635 | sched_cpumask_free(allmasks); |
| 7669 | return 0; | 7636 | return 0; |
| 7670 | 7637 | ||
| 7671 | #ifdef CONFIG_NUMA | 7638 | #ifdef CONFIG_NUMA |
| 7672 | error: | 7639 | error: |
| 7673 | free_sched_groups(cpu_map, tmpmask); | 7640 | free_sched_groups(cpu_map, tmpmask); |
| 7674 | SCHED_CPUMASK_FREE((void *)allmasks); | 7641 | sched_cpumask_free(allmasks); |
| 7642 | kfree(rd); | ||
| 7675 | return -ENOMEM; | 7643 | return -ENOMEM; |
| 7676 | #endif | 7644 | #endif |
| 7677 | } | 7645 | } |
| @@ -7734,8 +7702,6 @@ static void detach_destroy_domains(const cpumask_t *cpu_map) | |||
| 7734 | cpumask_t tmpmask; | 7702 | cpumask_t tmpmask; |
| 7735 | int i; | 7703 | int i; |
| 7736 | 7704 | ||
| 7737 | unregister_sched_domain_sysctl(); | ||
| 7738 | |||
| 7739 | for_each_cpu_mask_nr(i, *cpu_map) | 7705 | for_each_cpu_mask_nr(i, *cpu_map) |
| 7740 | cpu_attach_domain(NULL, &def_root_domain, i); | 7706 | cpu_attach_domain(NULL, &def_root_domain, i); |
| 7741 | synchronize_sched(); | 7707 | synchronize_sched(); |
| @@ -7773,13 +7739,14 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur, | |||
| 7773 | * | 7739 | * |
| 7774 | * The passed in 'doms_new' should be kmalloc'd. This routine takes | 7740 | * The passed in 'doms_new' should be kmalloc'd. This routine takes |
| 7775 | * ownership of it and will kfree it when done with it. If the caller | 7741 | * ownership of it and will kfree it when done with it. If the caller |
| 7776 | * failed the kmalloc call, then it can pass in doms_new == NULL, | 7742 | * failed the kmalloc call, then it can pass in doms_new == NULL && |
| 7777 | * and partition_sched_domains() will fallback to the single partition | 7743 | * ndoms_new == 1, and partition_sched_domains() will fallback to |
| 7778 | * 'fallback_doms', it also forces the domains to be rebuilt. | 7744 | * the single partition 'fallback_doms', it also forces the domains |
| 7745 | * to be rebuilt. | ||
| 7779 | * | 7746 | * |
| 7780 | * If doms_new==NULL it will be replaced with cpu_online_map. | 7747 | * If doms_new == NULL it will be replaced with cpu_online_map. |
| 7781 | * ndoms_new==0 is a special case for destroying existing domains. | 7748 | * ndoms_new == 0 is a special case for destroying existing domains, |
| 7782 | * It will not create the default domain. | 7749 | * and it will not create the default domain. |
| 7783 | * | 7750 | * |
| 7784 | * Call with hotplug lock held | 7751 | * Call with hotplug lock held |
| 7785 | */ | 7752 | */ |
| @@ -7812,7 +7779,7 @@ match1: | |||
| 7812 | ndoms_cur = 0; | 7779 | ndoms_cur = 0; |
| 7813 | doms_new = &fallback_doms; | 7780 | doms_new = &fallback_doms; |
| 7814 | cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map); | 7781 | cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map); |
| 7815 | dattr_new = NULL; | 7782 | WARN_ON_ONCE(dattr_new); |
| 7816 | } | 7783 | } |
| 7817 | 7784 | ||
| 7818 | /* Build new domains */ | 7785 | /* Build new domains */ |
| @@ -8472,7 +8439,7 @@ static | |||
| 8472 | int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) | 8439 | int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) |
| 8473 | { | 8440 | { |
| 8474 | struct cfs_rq *cfs_rq; | 8441 | struct cfs_rq *cfs_rq; |
| 8475 | struct sched_entity *se, *parent_se; | 8442 | struct sched_entity *se; |
| 8476 | struct rq *rq; | 8443 | struct rq *rq; |
| 8477 | int i; | 8444 | int i; |
| 8478 | 8445 | ||
| @@ -8488,18 +8455,17 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) | |||
| 8488 | for_each_possible_cpu(i) { | 8455 | for_each_possible_cpu(i) { |
| 8489 | rq = cpu_rq(i); | 8456 | rq = cpu_rq(i); |
| 8490 | 8457 | ||
| 8491 | cfs_rq = kmalloc_node(sizeof(struct cfs_rq), | 8458 | cfs_rq = kzalloc_node(sizeof(struct cfs_rq), |
| 8492 | GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); | 8459 | GFP_KERNEL, cpu_to_node(i)); |
| 8493 | if (!cfs_rq) | 8460 | if (!cfs_rq) |
| 8494 | goto err; | 8461 | goto err; |
| 8495 | 8462 | ||
| 8496 | se = kmalloc_node(sizeof(struct sched_entity), | 8463 | se = kzalloc_node(sizeof(struct sched_entity), |
| 8497 | GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); | 8464 | GFP_KERNEL, cpu_to_node(i)); |
| 8498 | if (!se) | 8465 | if (!se) |
| 8499 | goto err; | 8466 | goto err; |
| 8500 | 8467 | ||
| 8501 | parent_se = parent ? parent->se[i] : NULL; | 8468 | init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]); |
| 8502 | init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent_se); | ||
| 8503 | } | 8469 | } |
| 8504 | 8470 | ||
| 8505 | return 1; | 8471 | return 1; |
| @@ -8560,7 +8526,7 @@ static | |||
| 8560 | int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) | 8526 | int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) |
| 8561 | { | 8527 | { |
| 8562 | struct rt_rq *rt_rq; | 8528 | struct rt_rq *rt_rq; |
| 8563 | struct sched_rt_entity *rt_se, *parent_se; | 8529 | struct sched_rt_entity *rt_se; |
| 8564 | struct rq *rq; | 8530 | struct rq *rq; |
| 8565 | int i; | 8531 | int i; |
| 8566 | 8532 | ||
| @@ -8577,18 +8543,17 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) | |||
| 8577 | for_each_possible_cpu(i) { | 8543 | for_each_possible_cpu(i) { |
| 8578 | rq = cpu_rq(i); | 8544 | rq = cpu_rq(i); |
| 8579 | 8545 | ||
| 8580 | rt_rq = kmalloc_node(sizeof(struct rt_rq), | 8546 | rt_rq = kzalloc_node(sizeof(struct rt_rq), |
| 8581 | GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); | 8547 | GFP_KERNEL, cpu_to_node(i)); |
| 8582 | if (!rt_rq) | 8548 | if (!rt_rq) |
| 8583 | goto err; | 8549 | goto err; |
| 8584 | 8550 | ||
| 8585 | rt_se = kmalloc_node(sizeof(struct sched_rt_entity), | 8551 | rt_se = kzalloc_node(sizeof(struct sched_rt_entity), |
| 8586 | GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); | 8552 | GFP_KERNEL, cpu_to_node(i)); |
| 8587 | if (!rt_se) | 8553 | if (!rt_se) |
| 8588 | goto err; | 8554 | goto err; |
| 8589 | 8555 | ||
| 8590 | parent_se = parent ? parent->rt_se[i] : NULL; | 8556 | init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]); |
| 8591 | init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent_se); | ||
| 8592 | } | 8557 | } |
| 8593 | 8558 | ||
| 8594 | return 1; | 8559 | return 1; |
| @@ -9231,11 +9196,12 @@ struct cgroup_subsys cpu_cgroup_subsys = { | |||
| 9231 | * (balbir@in.ibm.com). | 9196 | * (balbir@in.ibm.com). |
| 9232 | */ | 9197 | */ |
| 9233 | 9198 | ||
| 9234 | /* track cpu usage of a group of tasks */ | 9199 | /* track cpu usage of a group of tasks and its child groups */ |
| 9235 | struct cpuacct { | 9200 | struct cpuacct { |
| 9236 | struct cgroup_subsys_state css; | 9201 | struct cgroup_subsys_state css; |
| 9237 | /* cpuusage holds pointer to a u64-type object on every cpu */ | 9202 | /* cpuusage holds pointer to a u64-type object on every cpu */ |
| 9238 | u64 *cpuusage; | 9203 | u64 *cpuusage; |
| 9204 | struct cpuacct *parent; | ||
| 9239 | }; | 9205 | }; |
| 9240 | 9206 | ||
| 9241 | struct cgroup_subsys cpuacct_subsys; | 9207 | struct cgroup_subsys cpuacct_subsys; |
| @@ -9269,6 +9235,9 @@ static struct cgroup_subsys_state *cpuacct_create( | |||
| 9269 | return ERR_PTR(-ENOMEM); | 9235 | return ERR_PTR(-ENOMEM); |
| 9270 | } | 9236 | } |
| 9271 | 9237 | ||
| 9238 | if (cgrp->parent) | ||
| 9239 | ca->parent = cgroup_ca(cgrp->parent); | ||
| 9240 | |||
| 9272 | return &ca->css; | 9241 | return &ca->css; |
| 9273 | } | 9242 | } |
| 9274 | 9243 | ||
| @@ -9348,14 +9317,16 @@ static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp) | |||
| 9348 | static void cpuacct_charge(struct task_struct *tsk, u64 cputime) | 9317 | static void cpuacct_charge(struct task_struct *tsk, u64 cputime) |
| 9349 | { | 9318 | { |
| 9350 | struct cpuacct *ca; | 9319 | struct cpuacct *ca; |
| 9320 | int cpu; | ||
| 9351 | 9321 | ||
| 9352 | if (!cpuacct_subsys.active) | 9322 | if (!cpuacct_subsys.active) |
| 9353 | return; | 9323 | return; |
| 9354 | 9324 | ||
| 9325 | cpu = task_cpu(tsk); | ||
| 9355 | ca = task_ca(tsk); | 9326 | ca = task_ca(tsk); |
| 9356 | if (ca) { | ||
| 9357 | u64 *cpuusage = percpu_ptr(ca->cpuusage, task_cpu(tsk)); | ||
| 9358 | 9327 | ||
| 9328 | for (; ca; ca = ca->parent) { | ||
| 9329 | u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu); | ||
| 9359 | *cpuusage += cputime; | 9330 | *cpuusage += cputime; |
| 9360 | } | 9331 | } |
| 9361 | } | 9332 | } |
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c index 5ae17762ec32..baf2f17af462 100644 --- a/kernel/sched_debug.c +++ b/kernel/sched_debug.c | |||
| @@ -53,6 +53,40 @@ static unsigned long nsec_low(unsigned long long nsec) | |||
| 53 | 53 | ||
| 54 | #define SPLIT_NS(x) nsec_high(x), nsec_low(x) | 54 | #define SPLIT_NS(x) nsec_high(x), nsec_low(x) |
| 55 | 55 | ||
| 56 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
| 57 | static void print_cfs_group_stats(struct seq_file *m, int cpu, | ||
| 58 | struct task_group *tg) | ||
| 59 | { | ||
| 60 | struct sched_entity *se = tg->se[cpu]; | ||
| 61 | if (!se) | ||
| 62 | return; | ||
| 63 | |||
| 64 | #define P(F) \ | ||
| 65 | SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F) | ||
| 66 | #define PN(F) \ | ||
| 67 | SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)F)) | ||
| 68 | |||
| 69 | PN(se->exec_start); | ||
| 70 | PN(se->vruntime); | ||
| 71 | PN(se->sum_exec_runtime); | ||
| 72 | #ifdef CONFIG_SCHEDSTATS | ||
| 73 | PN(se->wait_start); | ||
| 74 | PN(se->sleep_start); | ||
| 75 | PN(se->block_start); | ||
| 76 | PN(se->sleep_max); | ||
| 77 | PN(se->block_max); | ||
| 78 | PN(se->exec_max); | ||
| 79 | PN(se->slice_max); | ||
| 80 | PN(se->wait_max); | ||
| 81 | PN(se->wait_sum); | ||
| 82 | P(se->wait_count); | ||
| 83 | #endif | ||
| 84 | P(se->load.weight); | ||
| 85 | #undef PN | ||
| 86 | #undef P | ||
| 87 | } | ||
| 88 | #endif | ||
| 89 | |||
| 56 | static void | 90 | static void |
| 57 | print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) | 91 | print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) |
| 58 | { | 92 | { |
| @@ -121,14 +155,9 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) | |||
| 121 | 155 | ||
| 122 | #if defined(CONFIG_CGROUP_SCHED) && defined(CONFIG_FAIR_GROUP_SCHED) | 156 | #if defined(CONFIG_CGROUP_SCHED) && defined(CONFIG_FAIR_GROUP_SCHED) |
| 123 | char path[128] = ""; | 157 | char path[128] = ""; |
| 124 | struct cgroup *cgroup = NULL; | ||
| 125 | struct task_group *tg = cfs_rq->tg; | 158 | struct task_group *tg = cfs_rq->tg; |
| 126 | 159 | ||
| 127 | if (tg) | 160 | cgroup_path(tg->css.cgroup, path, sizeof(path)); |
| 128 | cgroup = tg->css.cgroup; | ||
| 129 | |||
| 130 | if (cgroup) | ||
| 131 | cgroup_path(cgroup, path, sizeof(path)); | ||
| 132 | 161 | ||
| 133 | SEQ_printf(m, "\ncfs_rq[%d]:%s\n", cpu, path); | 162 | SEQ_printf(m, "\ncfs_rq[%d]:%s\n", cpu, path); |
| 134 | #else | 163 | #else |
| @@ -144,7 +173,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) | |||
| 144 | last = __pick_last_entity(cfs_rq); | 173 | last = __pick_last_entity(cfs_rq); |
| 145 | if (last) | 174 | if (last) |
| 146 | max_vruntime = last->vruntime; | 175 | max_vruntime = last->vruntime; |
| 147 | min_vruntime = rq->cfs.min_vruntime; | 176 | min_vruntime = cfs_rq->min_vruntime; |
| 148 | rq0_min_vruntime = per_cpu(runqueues, 0).cfs.min_vruntime; | 177 | rq0_min_vruntime = per_cpu(runqueues, 0).cfs.min_vruntime; |
| 149 | spin_unlock_irqrestore(&rq->lock, flags); | 178 | spin_unlock_irqrestore(&rq->lock, flags); |
| 150 | SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "MIN_vruntime", | 179 | SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "MIN_vruntime", |
| @@ -161,31 +190,14 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) | |||
| 161 | SPLIT_NS(spread0)); | 190 | SPLIT_NS(spread0)); |
| 162 | SEQ_printf(m, " .%-30s: %ld\n", "nr_running", cfs_rq->nr_running); | 191 | SEQ_printf(m, " .%-30s: %ld\n", "nr_running", cfs_rq->nr_running); |
| 163 | SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight); | 192 | SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight); |
| 164 | #ifdef CONFIG_SCHEDSTATS | ||
| 165 | #define P(n) SEQ_printf(m, " .%-30s: %d\n", #n, rq->n); | ||
| 166 | |||
| 167 | P(yld_exp_empty); | ||
| 168 | P(yld_act_empty); | ||
| 169 | P(yld_both_empty); | ||
| 170 | P(yld_count); | ||
| 171 | |||
| 172 | P(sched_switch); | ||
| 173 | P(sched_count); | ||
| 174 | P(sched_goidle); | ||
| 175 | |||
| 176 | P(ttwu_count); | ||
| 177 | P(ttwu_local); | ||
| 178 | |||
| 179 | P(bkl_count); | ||
| 180 | 193 | ||
| 181 | #undef P | 194 | SEQ_printf(m, " .%-30s: %d\n", "nr_spread_over", |
| 182 | #endif | ||
| 183 | SEQ_printf(m, " .%-30s: %ld\n", "nr_spread_over", | ||
| 184 | cfs_rq->nr_spread_over); | 195 | cfs_rq->nr_spread_over); |
| 185 | #ifdef CONFIG_FAIR_GROUP_SCHED | 196 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 186 | #ifdef CONFIG_SMP | 197 | #ifdef CONFIG_SMP |
| 187 | SEQ_printf(m, " .%-30s: %lu\n", "shares", cfs_rq->shares); | 198 | SEQ_printf(m, " .%-30s: %lu\n", "shares", cfs_rq->shares); |
| 188 | #endif | 199 | #endif |
| 200 | print_cfs_group_stats(m, cpu, cfs_rq->tg); | ||
| 189 | #endif | 201 | #endif |
| 190 | } | 202 | } |
| 191 | 203 | ||
| @@ -193,14 +205,9 @@ void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq) | |||
| 193 | { | 205 | { |
| 194 | #if defined(CONFIG_CGROUP_SCHED) && defined(CONFIG_RT_GROUP_SCHED) | 206 | #if defined(CONFIG_CGROUP_SCHED) && defined(CONFIG_RT_GROUP_SCHED) |
| 195 | char path[128] = ""; | 207 | char path[128] = ""; |
| 196 | struct cgroup *cgroup = NULL; | ||
| 197 | struct task_group *tg = rt_rq->tg; | 208 | struct task_group *tg = rt_rq->tg; |
| 198 | 209 | ||
| 199 | if (tg) | 210 | cgroup_path(tg->css.cgroup, path, sizeof(path)); |
| 200 | cgroup = tg->css.cgroup; | ||
| 201 | |||
| 202 | if (cgroup) | ||
| 203 | cgroup_path(cgroup, path, sizeof(path)); | ||
| 204 | 211 | ||
| 205 | SEQ_printf(m, "\nrt_rq[%d]:%s\n", cpu, path); | 212 | SEQ_printf(m, "\nrt_rq[%d]:%s\n", cpu, path); |
| 206 | #else | 213 | #else |
| @@ -260,6 +267,25 @@ static void print_cpu(struct seq_file *m, int cpu) | |||
| 260 | #undef P | 267 | #undef P |
| 261 | #undef PN | 268 | #undef PN |
| 262 | 269 | ||
| 270 | #ifdef CONFIG_SCHEDSTATS | ||
| 271 | #define P(n) SEQ_printf(m, " .%-30s: %d\n", #n, rq->n); | ||
| 272 | |||
| 273 | P(yld_exp_empty); | ||
| 274 | P(yld_act_empty); | ||
| 275 | P(yld_both_empty); | ||
| 276 | P(yld_count); | ||
| 277 | |||
| 278 | P(sched_switch); | ||
| 279 | P(sched_count); | ||
| 280 | P(sched_goidle); | ||
| 281 | |||
| 282 | P(ttwu_count); | ||
| 283 | P(ttwu_local); | ||
| 284 | |||
| 285 | P(bkl_count); | ||
| 286 | |||
| 287 | #undef P | ||
| 288 | #endif | ||
| 263 | print_cfs_stats(m, cpu); | 289 | print_cfs_stats(m, cpu); |
| 264 | print_rt_stats(m, cpu); | 290 | print_rt_stats(m, cpu); |
| 265 | 291 | ||
| @@ -271,7 +297,7 @@ static int sched_debug_show(struct seq_file *m, void *v) | |||
| 271 | u64 now = ktime_to_ns(ktime_get()); | 297 | u64 now = ktime_to_ns(ktime_get()); |
| 272 | int cpu; | 298 | int cpu; |
| 273 | 299 | ||
| 274 | SEQ_printf(m, "Sched Debug Version: v0.07, %s %.*s\n", | 300 | SEQ_printf(m, "Sched Debug Version: v0.08, %s %.*s\n", |
| 275 | init_utsname()->release, | 301 | init_utsname()->release, |
| 276 | (int)strcspn(init_utsname()->version, " "), | 302 | (int)strcspn(init_utsname()->version, " "), |
| 277 | init_utsname()->version); | 303 | init_utsname()->version); |
| @@ -422,10 +448,11 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) | |||
| 422 | #undef __P | 448 | #undef __P |
| 423 | 449 | ||
| 424 | { | 450 | { |
| 451 | unsigned int this_cpu = raw_smp_processor_id(); | ||
| 425 | u64 t0, t1; | 452 | u64 t0, t1; |
| 426 | 453 | ||
| 427 | t0 = sched_clock(); | 454 | t0 = cpu_clock(this_cpu); |
| 428 | t1 = sched_clock(); | 455 | t1 = cpu_clock(this_cpu); |
| 429 | SEQ_printf(m, "%-35s:%21Ld\n", | 456 | SEQ_printf(m, "%-35s:%21Ld\n", |
| 430 | "clock-delta", (long long)(t1-t0)); | 457 | "clock-delta", (long long)(t1-t0)); |
| 431 | } | 458 | } |
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index ce514afd78ff..98345e45b059 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c | |||
| @@ -341,23 +341,20 @@ static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) | |||
| 341 | cfs_rq->rb_leftmost = next_node; | 341 | cfs_rq->rb_leftmost = next_node; |
| 342 | } | 342 | } |
| 343 | 343 | ||
| 344 | if (cfs_rq->next == se) | ||
| 345 | cfs_rq->next = NULL; | ||
| 346 | |||
| 347 | rb_erase(&se->run_node, &cfs_rq->tasks_timeline); | 344 | rb_erase(&se->run_node, &cfs_rq->tasks_timeline); |
| 348 | } | 345 | } |
| 349 | 346 | ||
| 350 | static inline struct rb_node *first_fair(struct cfs_rq *cfs_rq) | ||
| 351 | { | ||
| 352 | return cfs_rq->rb_leftmost; | ||
| 353 | } | ||
| 354 | |||
| 355 | static struct sched_entity *__pick_next_entity(struct cfs_rq *cfs_rq) | 347 | static struct sched_entity *__pick_next_entity(struct cfs_rq *cfs_rq) |
| 356 | { | 348 | { |
| 357 | return rb_entry(first_fair(cfs_rq), struct sched_entity, run_node); | 349 | struct rb_node *left = cfs_rq->rb_leftmost; |
| 350 | |||
| 351 | if (!left) | ||
| 352 | return NULL; | ||
| 353 | |||
| 354 | return rb_entry(left, struct sched_entity, run_node); | ||
| 358 | } | 355 | } |
| 359 | 356 | ||
| 360 | static inline struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) | 357 | static struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) |
| 361 | { | 358 | { |
| 362 | struct rb_node *last = rb_last(&cfs_rq->tasks_timeline); | 359 | struct rb_node *last = rb_last(&cfs_rq->tasks_timeline); |
| 363 | 360 | ||
| @@ -719,6 +716,15 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int wakeup) | |||
| 719 | __enqueue_entity(cfs_rq, se); | 716 | __enqueue_entity(cfs_rq, se); |
| 720 | } | 717 | } |
| 721 | 718 | ||
| 719 | static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) | ||
| 720 | { | ||
| 721 | if (cfs_rq->last == se) | ||
| 722 | cfs_rq->last = NULL; | ||
| 723 | |||
| 724 | if (cfs_rq->next == se) | ||
| 725 | cfs_rq->next = NULL; | ||
| 726 | } | ||
| 727 | |||
| 722 | static void | 728 | static void |
| 723 | dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) | 729 | dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) |
| 724 | { | 730 | { |
| @@ -741,6 +747,8 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) | |||
| 741 | #endif | 747 | #endif |
| 742 | } | 748 | } |
| 743 | 749 | ||
| 750 | clear_buddies(cfs_rq, se); | ||
| 751 | |||
| 744 | if (se != cfs_rq->curr) | 752 | if (se != cfs_rq->curr) |
| 745 | __dequeue_entity(cfs_rq, se); | 753 | __dequeue_entity(cfs_rq, se); |
| 746 | account_entity_dequeue(cfs_rq, se); | 754 | account_entity_dequeue(cfs_rq, se); |
| @@ -794,24 +802,15 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) | |||
| 794 | static int | 802 | static int |
| 795 | wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se); | 803 | wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se); |
| 796 | 804 | ||
| 797 | static struct sched_entity * | ||
| 798 | pick_next(struct cfs_rq *cfs_rq, struct sched_entity *se) | ||
| 799 | { | ||
| 800 | if (!cfs_rq->next || wakeup_preempt_entity(cfs_rq->next, se) == 1) | ||
| 801 | return se; | ||
| 802 | |||
| 803 | return cfs_rq->next; | ||
| 804 | } | ||
| 805 | |||
| 806 | static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) | 805 | static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) |
| 807 | { | 806 | { |
| 808 | struct sched_entity *se = NULL; | 807 | struct sched_entity *se = __pick_next_entity(cfs_rq); |
| 809 | 808 | ||
| 810 | if (first_fair(cfs_rq)) { | 809 | if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, se) < 1) |
| 811 | se = __pick_next_entity(cfs_rq); | 810 | return cfs_rq->next; |
| 812 | se = pick_next(cfs_rq, se); | 811 | |
| 813 | set_next_entity(cfs_rq, se); | 812 | if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, se) < 1) |
| 814 | } | 813 | return cfs_rq->last; |
| 815 | 814 | ||
| 816 | return se; | 815 | return se; |
| 817 | } | 816 | } |
| @@ -983,6 +982,8 @@ static void yield_task_fair(struct rq *rq) | |||
| 983 | if (unlikely(cfs_rq->nr_running == 1)) | 982 | if (unlikely(cfs_rq->nr_running == 1)) |
| 984 | return; | 983 | return; |
| 985 | 984 | ||
| 985 | clear_buddies(cfs_rq, se); | ||
| 986 | |||
| 986 | if (likely(!sysctl_sched_compat_yield) && curr->policy != SCHED_BATCH) { | 987 | if (likely(!sysctl_sched_compat_yield) && curr->policy != SCHED_BATCH) { |
| 987 | update_rq_clock(rq); | 988 | update_rq_clock(rq); |
| 988 | /* | 989 | /* |
| @@ -1325,26 +1326,53 @@ wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se) | |||
| 1325 | return 0; | 1326 | return 0; |
| 1326 | } | 1327 | } |
| 1327 | 1328 | ||
| 1329 | static void set_last_buddy(struct sched_entity *se) | ||
| 1330 | { | ||
| 1331 | for_each_sched_entity(se) | ||
| 1332 | cfs_rq_of(se)->last = se; | ||
| 1333 | } | ||
| 1334 | |||
| 1335 | static void set_next_buddy(struct sched_entity *se) | ||
| 1336 | { | ||
| 1337 | for_each_sched_entity(se) | ||
| 1338 | cfs_rq_of(se)->next = se; | ||
| 1339 | } | ||
| 1340 | |||
| 1328 | /* | 1341 | /* |
| 1329 | * Preempt the current task with a newly woken task if needed: | 1342 | * Preempt the current task with a newly woken task if needed: |
| 1330 | */ | 1343 | */ |
| 1331 | static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync) | 1344 | static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync) |
| 1332 | { | 1345 | { |
| 1333 | struct task_struct *curr = rq->curr; | 1346 | struct task_struct *curr = rq->curr; |
| 1334 | struct cfs_rq *cfs_rq = task_cfs_rq(curr); | ||
| 1335 | struct sched_entity *se = &curr->se, *pse = &p->se; | 1347 | struct sched_entity *se = &curr->se, *pse = &p->se; |
| 1336 | 1348 | ||
| 1337 | if (unlikely(rt_prio(p->prio))) { | 1349 | if (unlikely(rt_prio(p->prio))) { |
| 1350 | struct cfs_rq *cfs_rq = task_cfs_rq(curr); | ||
| 1351 | |||
| 1338 | update_rq_clock(rq); | 1352 | update_rq_clock(rq); |
| 1339 | update_curr(cfs_rq); | 1353 | update_curr(cfs_rq); |
| 1340 | resched_task(curr); | 1354 | resched_task(curr); |
| 1341 | return; | 1355 | return; |
| 1342 | } | 1356 | } |
| 1343 | 1357 | ||
| 1358 | if (unlikely(p->sched_class != &fair_sched_class)) | ||
| 1359 | return; | ||
| 1360 | |||
| 1344 | if (unlikely(se == pse)) | 1361 | if (unlikely(se == pse)) |
| 1345 | return; | 1362 | return; |
| 1346 | 1363 | ||
| 1347 | cfs_rq_of(pse)->next = pse; | 1364 | /* |
| 1365 | * Only set the backward buddy when the current task is still on the | ||
| 1366 | * rq. This can happen when a wakeup gets interleaved with schedule on | ||
| 1367 | * the ->pre_schedule() or idle_balance() point, either of which can | ||
| 1368 | * drop the rq lock. | ||
| 1369 | * | ||
| 1370 | * Also, during early boot the idle thread is in the fair class, for | ||
| 1371 | * obvious reasons its a bad idea to schedule back to the idle thread. | ||
| 1372 | */ | ||
| 1373 | if (sched_feat(LAST_BUDDY) && likely(se->on_rq && curr != rq->idle)) | ||
| 1374 | set_last_buddy(se); | ||
| 1375 | set_next_buddy(pse); | ||
| 1348 | 1376 | ||
| 1349 | /* | 1377 | /* |
| 1350 | * We can come here with TIF_NEED_RESCHED already set from new task | 1378 | * We can come here with TIF_NEED_RESCHED already set from new task |
| @@ -1396,6 +1424,7 @@ static struct task_struct *pick_next_task_fair(struct rq *rq) | |||
| 1396 | 1424 | ||
| 1397 | do { | 1425 | do { |
| 1398 | se = pick_next_entity(cfs_rq); | 1426 | se = pick_next_entity(cfs_rq); |
| 1427 | set_next_entity(cfs_rq, se); | ||
| 1399 | cfs_rq = group_cfs_rq(se); | 1428 | cfs_rq = group_cfs_rq(se); |
| 1400 | } while (cfs_rq); | 1429 | } while (cfs_rq); |
| 1401 | 1430 | ||
diff --git a/kernel/sched_features.h b/kernel/sched_features.h index fda016218296..da5d93b5d2c6 100644 --- a/kernel/sched_features.h +++ b/kernel/sched_features.h | |||
| @@ -12,3 +12,4 @@ SCHED_FEAT(LB_BIAS, 1) | |||
| 12 | SCHED_FEAT(LB_WAKEUP_UPDATE, 1) | 12 | SCHED_FEAT(LB_WAKEUP_UPDATE, 1) |
| 13 | SCHED_FEAT(ASYM_EFF_LOAD, 1) | 13 | SCHED_FEAT(ASYM_EFF_LOAD, 1) |
| 14 | SCHED_FEAT(WAKEUP_OVERLAP, 0) | 14 | SCHED_FEAT(WAKEUP_OVERLAP, 0) |
| 15 | SCHED_FEAT(LAST_BUDDY, 1) | ||
diff --git a/kernel/sched_stats.h b/kernel/sched_stats.h index ee71bec1da66..7dbf72a2b02c 100644 --- a/kernel/sched_stats.h +++ b/kernel/sched_stats.h | |||
| @@ -298,9 +298,11 @@ static inline void account_group_user_time(struct task_struct *tsk, | |||
| 298 | { | 298 | { |
| 299 | struct signal_struct *sig; | 299 | struct signal_struct *sig; |
| 300 | 300 | ||
| 301 | sig = tsk->signal; | 301 | /* tsk == current, ensure it is safe to use ->signal */ |
| 302 | if (unlikely(!sig)) | 302 | if (unlikely(tsk->exit_state)) |
| 303 | return; | 303 | return; |
| 304 | |||
| 305 | sig = tsk->signal; | ||
| 304 | if (sig->cputime.totals) { | 306 | if (sig->cputime.totals) { |
| 305 | struct task_cputime *times; | 307 | struct task_cputime *times; |
| 306 | 308 | ||
| @@ -325,9 +327,11 @@ static inline void account_group_system_time(struct task_struct *tsk, | |||
| 325 | { | 327 | { |
| 326 | struct signal_struct *sig; | 328 | struct signal_struct *sig; |
| 327 | 329 | ||
| 328 | sig = tsk->signal; | 330 | /* tsk == current, ensure it is safe to use ->signal */ |
| 329 | if (unlikely(!sig)) | 331 | if (unlikely(tsk->exit_state)) |
| 330 | return; | 332 | return; |
| 333 | |||
| 334 | sig = tsk->signal; | ||
| 331 | if (sig->cputime.totals) { | 335 | if (sig->cputime.totals) { |
| 332 | struct task_cputime *times; | 336 | struct task_cputime *times; |
| 333 | 337 | ||
| @@ -353,8 +357,11 @@ static inline void account_group_exec_runtime(struct task_struct *tsk, | |||
| 353 | struct signal_struct *sig; | 357 | struct signal_struct *sig; |
| 354 | 358 | ||
| 355 | sig = tsk->signal; | 359 | sig = tsk->signal; |
| 360 | /* see __exit_signal()->task_rq_unlock_wait() */ | ||
| 361 | barrier(); | ||
| 356 | if (unlikely(!sig)) | 362 | if (unlikely(!sig)) |
| 357 | return; | 363 | return; |
| 364 | |||
| 358 | if (sig->cputime.totals) { | 365 | if (sig->cputime.totals) { |
| 359 | struct task_cputime *times; | 366 | struct task_cputime *times; |
| 360 | 367 | ||
diff --git a/kernel/smp.c b/kernel/smp.c index f362a8553777..75c8dde58c55 100644 --- a/kernel/smp.c +++ b/kernel/smp.c | |||
| @@ -51,10 +51,6 @@ static void csd_flag_wait(struct call_single_data *data) | |||
| 51 | { | 51 | { |
| 52 | /* Wait for response */ | 52 | /* Wait for response */ |
| 53 | do { | 53 | do { |
| 54 | /* | ||
| 55 | * We need to see the flags store in the IPI handler | ||
| 56 | */ | ||
| 57 | smp_mb(); | ||
| 58 | if (!(data->flags & CSD_FLAG_WAIT)) | 54 | if (!(data->flags & CSD_FLAG_WAIT)) |
| 59 | break; | 55 | break; |
| 60 | cpu_relax(); | 56 | cpu_relax(); |
| @@ -76,6 +72,11 @@ static void generic_exec_single(int cpu, struct call_single_data *data) | |||
| 76 | list_add_tail(&data->list, &dst->list); | 72 | list_add_tail(&data->list, &dst->list); |
| 77 | spin_unlock_irqrestore(&dst->lock, flags); | 73 | spin_unlock_irqrestore(&dst->lock, flags); |
| 78 | 74 | ||
| 75 | /* | ||
| 76 | * Make the list addition visible before sending the ipi. | ||
| 77 | */ | ||
| 78 | smp_mb(); | ||
| 79 | |||
| 79 | if (ipi) | 80 | if (ipi) |
| 80 | arch_send_call_function_single_ipi(cpu); | 81 | arch_send_call_function_single_ipi(cpu); |
| 81 | 82 | ||
| @@ -157,7 +158,7 @@ void generic_smp_call_function_single_interrupt(void) | |||
| 157 | * Need to see other stores to list head for checking whether | 158 | * Need to see other stores to list head for checking whether |
| 158 | * list is empty without holding q->lock | 159 | * list is empty without holding q->lock |
| 159 | */ | 160 | */ |
| 160 | smp_mb(); | 161 | smp_read_barrier_depends(); |
| 161 | while (!list_empty(&q->list)) { | 162 | while (!list_empty(&q->list)) { |
| 162 | unsigned int data_flags; | 163 | unsigned int data_flags; |
| 163 | 164 | ||
| @@ -191,7 +192,7 @@ void generic_smp_call_function_single_interrupt(void) | |||
| 191 | /* | 192 | /* |
| 192 | * See comment on outer loop | 193 | * See comment on outer loop |
| 193 | */ | 194 | */ |
| 194 | smp_mb(); | 195 | smp_read_barrier_depends(); |
| 195 | } | 196 | } |
| 196 | } | 197 | } |
| 197 | 198 | ||
| @@ -370,6 +371,11 @@ int smp_call_function_mask(cpumask_t mask, void (*func)(void *), void *info, | |||
| 370 | list_add_tail_rcu(&data->csd.list, &call_function_queue); | 371 | list_add_tail_rcu(&data->csd.list, &call_function_queue); |
| 371 | spin_unlock_irqrestore(&call_function_lock, flags); | 372 | spin_unlock_irqrestore(&call_function_lock, flags); |
| 372 | 373 | ||
| 374 | /* | ||
| 375 | * Make the list addition visible before sending the ipi. | ||
| 376 | */ | ||
| 377 | smp_mb(); | ||
| 378 | |||
| 373 | /* Send a message to all CPUs in the map */ | 379 | /* Send a message to all CPUs in the map */ |
| 374 | arch_send_call_function_ipi(mask); | 380 | arch_send_call_function_ipi(mask); |
| 375 | 381 | ||
diff --git a/kernel/softirq.c b/kernel/softirq.c index 7110daeb9a90..e7c69a720d69 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c | |||
| @@ -269,10 +269,11 @@ void irq_enter(void) | |||
| 269 | { | 269 | { |
| 270 | int cpu = smp_processor_id(); | 270 | int cpu = smp_processor_id(); |
| 271 | 271 | ||
| 272 | if (idle_cpu(cpu) && !in_interrupt()) | 272 | if (idle_cpu(cpu) && !in_interrupt()) { |
| 273 | __irq_enter(); | ||
| 273 | tick_check_idle(cpu); | 274 | tick_check_idle(cpu); |
| 274 | 275 | } else | |
| 275 | __irq_enter(); | 276 | __irq_enter(); |
| 276 | } | 277 | } |
| 277 | 278 | ||
| 278 | #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED | 279 | #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED |
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c index 9bc4c00872c9..24e8ceacc388 100644 --- a/kernel/stop_machine.c +++ b/kernel/stop_machine.c | |||
| @@ -112,7 +112,7 @@ static int chill(void *unused) | |||
| 112 | int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) | 112 | int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) |
| 113 | { | 113 | { |
| 114 | struct work_struct *sm_work; | 114 | struct work_struct *sm_work; |
| 115 | int i; | 115 | int i, ret; |
| 116 | 116 | ||
| 117 | /* Set up initial state. */ | 117 | /* Set up initial state. */ |
| 118 | mutex_lock(&lock); | 118 | mutex_lock(&lock); |
| @@ -137,8 +137,9 @@ int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) | |||
| 137 | /* This will release the thread on our CPU. */ | 137 | /* This will release the thread on our CPU. */ |
| 138 | put_cpu(); | 138 | put_cpu(); |
| 139 | flush_workqueue(stop_machine_wq); | 139 | flush_workqueue(stop_machine_wq); |
| 140 | ret = active.fnret; | ||
| 140 | mutex_unlock(&lock); | 141 | mutex_unlock(&lock); |
| 141 | return active.fnret; | 142 | return ret; |
| 142 | } | 143 | } |
| 143 | 144 | ||
| 144 | int stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) | 145 | int stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) |
diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index a77b27b11b04..e14a23281707 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c | |||
| @@ -31,7 +31,7 @@ cond_syscall(sys_socketpair); | |||
| 31 | cond_syscall(sys_bind); | 31 | cond_syscall(sys_bind); |
| 32 | cond_syscall(sys_listen); | 32 | cond_syscall(sys_listen); |
| 33 | cond_syscall(sys_accept); | 33 | cond_syscall(sys_accept); |
| 34 | cond_syscall(sys_paccept); | 34 | cond_syscall(sys_accept4); |
| 35 | cond_syscall(sys_connect); | 35 | cond_syscall(sys_connect); |
| 36 | cond_syscall(sys_getsockname); | 36 | cond_syscall(sys_getsockname); |
| 37 | cond_syscall(sys_getpeername); | 37 | cond_syscall(sys_getpeername); |
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 5bbb1044f847..342fc9ccab46 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c | |||
| @@ -568,6 +568,9 @@ static void tick_nohz_switch_to_nohz(void) | |||
| 568 | */ | 568 | */ |
| 569 | static void tick_nohz_kick_tick(int cpu) | 569 | static void tick_nohz_kick_tick(int cpu) |
| 570 | { | 570 | { |
| 571 | #if 0 | ||
| 572 | /* Switch back to 2.6.27 behaviour */ | ||
| 573 | |||
| 571 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); | 574 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); |
| 572 | ktime_t delta, now; | 575 | ktime_t delta, now; |
| 573 | 576 | ||
| @@ -584,6 +587,7 @@ static void tick_nohz_kick_tick(int cpu) | |||
| 584 | return; | 587 | return; |
| 585 | 588 | ||
| 586 | tick_nohz_restart(ts, now); | 589 | tick_nohz_restart(ts, now); |
| 590 | #endif | ||
| 587 | } | 591 | } |
| 588 | 592 | ||
| 589 | #else | 593 | #else |
diff --git a/kernel/timer.c b/kernel/timer.c index 56becf373c58..dbd50fabe4c7 100644 --- a/kernel/timer.c +++ b/kernel/timer.c | |||
| @@ -112,27 +112,8 @@ timer_set_base(struct timer_list *timer, struct tvec_base *new_base) | |||
| 112 | tbase_get_deferrable(timer->base)); | 112 | tbase_get_deferrable(timer->base)); |
| 113 | } | 113 | } |
| 114 | 114 | ||
| 115 | /** | 115 | static unsigned long round_jiffies_common(unsigned long j, int cpu, |
| 116 | * __round_jiffies - function to round jiffies to a full second | 116 | bool force_up) |
| 117 | * @j: the time in (absolute) jiffies that should be rounded | ||
| 118 | * @cpu: the processor number on which the timeout will happen | ||
| 119 | * | ||
| 120 | * __round_jiffies() rounds an absolute time in the future (in jiffies) | ||
| 121 | * up or down to (approximately) full seconds. This is useful for timers | ||
| 122 | * for which the exact time they fire does not matter too much, as long as | ||
| 123 | * they fire approximately every X seconds. | ||
| 124 | * | ||
| 125 | * By rounding these timers to whole seconds, all such timers will fire | ||
| 126 | * at the same time, rather than at various times spread out. The goal | ||
| 127 | * of this is to have the CPU wake up less, which saves power. | ||
| 128 | * | ||
| 129 | * The exact rounding is skewed for each processor to avoid all | ||
| 130 | * processors firing at the exact same time, which could lead | ||
| 131 | * to lock contention or spurious cache line bouncing. | ||
| 132 | * | ||
| 133 | * The return value is the rounded version of the @j parameter. | ||
| 134 | */ | ||
| 135 | unsigned long __round_jiffies(unsigned long j, int cpu) | ||
| 136 | { | 117 | { |
| 137 | int rem; | 118 | int rem; |
| 138 | unsigned long original = j; | 119 | unsigned long original = j; |
| @@ -154,8 +135,9 @@ unsigned long __round_jiffies(unsigned long j, int cpu) | |||
| 154 | * due to delays of the timer irq, long irq off times etc etc) then | 135 | * due to delays of the timer irq, long irq off times etc etc) then |
| 155 | * we should round down to the whole second, not up. Use 1/4th second | 136 | * we should round down to the whole second, not up. Use 1/4th second |
| 156 | * as cutoff for this rounding as an extreme upper bound for this. | 137 | * as cutoff for this rounding as an extreme upper bound for this. |
| 138 | * But never round down if @force_up is set. | ||
| 157 | */ | 139 | */ |
| 158 | if (rem < HZ/4) /* round down */ | 140 | if (rem < HZ/4 && !force_up) /* round down */ |
| 159 | j = j - rem; | 141 | j = j - rem; |
| 160 | else /* round up */ | 142 | else /* round up */ |
| 161 | j = j - rem + HZ; | 143 | j = j - rem + HZ; |
| @@ -167,6 +149,31 @@ unsigned long __round_jiffies(unsigned long j, int cpu) | |||
| 167 | return original; | 149 | return original; |
| 168 | return j; | 150 | return j; |
| 169 | } | 151 | } |
| 152 | |||
| 153 | /** | ||
| 154 | * __round_jiffies - function to round jiffies to a full second | ||
| 155 | * @j: the time in (absolute) jiffies that should be rounded | ||
| 156 | * @cpu: the processor number on which the timeout will happen | ||
| 157 | * | ||
| 158 | * __round_jiffies() rounds an absolute time in the future (in jiffies) | ||
| 159 | * up or down to (approximately) full seconds. This is useful for timers | ||
| 160 | * for which the exact time they fire does not matter too much, as long as | ||
| 161 | * they fire approximately every X seconds. | ||
| 162 | * | ||
| 163 | * By rounding these timers to whole seconds, all such timers will fire | ||
| 164 | * at the same time, rather than at various times spread out. The goal | ||
| 165 | * of this is to have the CPU wake up less, which saves power. | ||
| 166 | * | ||
| 167 | * The exact rounding is skewed for each processor to avoid all | ||
| 168 | * processors firing at the exact same time, which could lead | ||
| 169 | * to lock contention or spurious cache line bouncing. | ||
| 170 | * | ||
| 171 | * The return value is the rounded version of the @j parameter. | ||
| 172 | */ | ||
| 173 | unsigned long __round_jiffies(unsigned long j, int cpu) | ||
| 174 | { | ||
| 175 | return round_jiffies_common(j, cpu, false); | ||
| 176 | } | ||
| 170 | EXPORT_SYMBOL_GPL(__round_jiffies); | 177 | EXPORT_SYMBOL_GPL(__round_jiffies); |
| 171 | 178 | ||
| 172 | /** | 179 | /** |
| @@ -191,13 +198,10 @@ EXPORT_SYMBOL_GPL(__round_jiffies); | |||
| 191 | */ | 198 | */ |
| 192 | unsigned long __round_jiffies_relative(unsigned long j, int cpu) | 199 | unsigned long __round_jiffies_relative(unsigned long j, int cpu) |
| 193 | { | 200 | { |
| 194 | /* | 201 | unsigned long j0 = jiffies; |
| 195 | * In theory the following code can skip a jiffy in case jiffies | 202 | |
| 196 | * increments right between the addition and the later subtraction. | 203 | /* Use j0 because jiffies might change while we run */ |
| 197 | * However since the entire point of this function is to use approximate | 204 | return round_jiffies_common(j + j0, cpu, false) - j0; |
| 198 | * timeouts, it's entirely ok to not handle that. | ||
| 199 | */ | ||
| 200 | return __round_jiffies(j + jiffies, cpu) - jiffies; | ||
| 201 | } | 205 | } |
| 202 | EXPORT_SYMBOL_GPL(__round_jiffies_relative); | 206 | EXPORT_SYMBOL_GPL(__round_jiffies_relative); |
| 203 | 207 | ||
| @@ -218,7 +222,7 @@ EXPORT_SYMBOL_GPL(__round_jiffies_relative); | |||
| 218 | */ | 222 | */ |
| 219 | unsigned long round_jiffies(unsigned long j) | 223 | unsigned long round_jiffies(unsigned long j) |
| 220 | { | 224 | { |
| 221 | return __round_jiffies(j, raw_smp_processor_id()); | 225 | return round_jiffies_common(j, raw_smp_processor_id(), false); |
| 222 | } | 226 | } |
| 223 | EXPORT_SYMBOL_GPL(round_jiffies); | 227 | EXPORT_SYMBOL_GPL(round_jiffies); |
| 224 | 228 | ||
| @@ -243,6 +247,71 @@ unsigned long round_jiffies_relative(unsigned long j) | |||
| 243 | } | 247 | } |
| 244 | EXPORT_SYMBOL_GPL(round_jiffies_relative); | 248 | EXPORT_SYMBOL_GPL(round_jiffies_relative); |
| 245 | 249 | ||
| 250 | /** | ||
| 251 | * __round_jiffies_up - function to round jiffies up to a full second | ||
| 252 | * @j: the time in (absolute) jiffies that should be rounded | ||
| 253 | * @cpu: the processor number on which the timeout will happen | ||
| 254 | * | ||
| 255 | * This is the same as __round_jiffies() except that it will never | ||
| 256 | * round down. This is useful for timeouts for which the exact time | ||
| 257 | * of firing does not matter too much, as long as they don't fire too | ||
| 258 | * early. | ||
| 259 | */ | ||
| 260 | unsigned long __round_jiffies_up(unsigned long j, int cpu) | ||
| 261 | { | ||
| 262 | return round_jiffies_common(j, cpu, true); | ||
| 263 | } | ||
| 264 | EXPORT_SYMBOL_GPL(__round_jiffies_up); | ||
| 265 | |||
| 266 | /** | ||
| 267 | * __round_jiffies_up_relative - function to round jiffies up to a full second | ||
| 268 | * @j: the time in (relative) jiffies that should be rounded | ||
| 269 | * @cpu: the processor number on which the timeout will happen | ||
| 270 | * | ||
| 271 | * This is the same as __round_jiffies_relative() except that it will never | ||
| 272 | * round down. This is useful for timeouts for which the exact time | ||
| 273 | * of firing does not matter too much, as long as they don't fire too | ||
| 274 | * early. | ||
| 275 | */ | ||
| 276 | unsigned long __round_jiffies_up_relative(unsigned long j, int cpu) | ||
| 277 | { | ||
| 278 | unsigned long j0 = jiffies; | ||
| 279 | |||
| 280 | /* Use j0 because jiffies might change while we run */ | ||
| 281 | return round_jiffies_common(j + j0, cpu, true) - j0; | ||
| 282 | } | ||
| 283 | EXPORT_SYMBOL_GPL(__round_jiffies_up_relative); | ||
| 284 | |||
| 285 | /** | ||
| 286 | * round_jiffies_up - function to round jiffies up to a full second | ||
| 287 | * @j: the time in (absolute) jiffies that should be rounded | ||
| 288 | * | ||
| 289 | * This is the same as round_jiffies() except that it will never | ||
| 290 | * round down. This is useful for timeouts for which the exact time | ||
| 291 | * of firing does not matter too much, as long as they don't fire too | ||
| 292 | * early. | ||
| 293 | */ | ||
| 294 | unsigned long round_jiffies_up(unsigned long j) | ||
| 295 | { | ||
| 296 | return round_jiffies_common(j, raw_smp_processor_id(), true); | ||
| 297 | } | ||
| 298 | EXPORT_SYMBOL_GPL(round_jiffies_up); | ||
| 299 | |||
| 300 | /** | ||
| 301 | * round_jiffies_up_relative - function to round jiffies up to a full second | ||
| 302 | * @j: the time in (relative) jiffies that should be rounded | ||
| 303 | * | ||
| 304 | * This is the same as round_jiffies_relative() except that it will never | ||
| 305 | * round down. This is useful for timeouts for which the exact time | ||
| 306 | * of firing does not matter too much, as long as they don't fire too | ||
| 307 | * early. | ||
| 308 | */ | ||
| 309 | unsigned long round_jiffies_up_relative(unsigned long j) | ||
| 310 | { | ||
| 311 | return __round_jiffies_up_relative(j, raw_smp_processor_id()); | ||
| 312 | } | ||
| 313 | EXPORT_SYMBOL_GPL(round_jiffies_up_relative); | ||
| 314 | |||
| 246 | 315 | ||
| 247 | static inline void set_running_timer(struct tvec_base *base, | 316 | static inline void set_running_timer(struct tvec_base *base, |
| 248 | struct timer_list *timer) | 317 | struct timer_list *timer) |
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index b58f43bec363..33dbefd471e8 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig | |||
| @@ -25,7 +25,7 @@ config TRACING | |||
| 25 | bool | 25 | bool |
| 26 | select DEBUG_FS | 26 | select DEBUG_FS |
| 27 | select RING_BUFFER | 27 | select RING_BUFFER |
| 28 | select STACKTRACE | 28 | select STACKTRACE if STACKTRACE_SUPPORT |
| 29 | select TRACEPOINTS | 29 | select TRACEPOINTS |
| 30 | select NOP_TRACER | 30 | select NOP_TRACER |
| 31 | 31 | ||
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 4a39d24568c8..78db083390f0 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
| @@ -185,7 +185,6 @@ enum { | |||
| 185 | }; | 185 | }; |
| 186 | 186 | ||
| 187 | static int ftrace_filtered; | 187 | static int ftrace_filtered; |
| 188 | static int tracing_on; | ||
| 189 | 188 | ||
| 190 | static LIST_HEAD(ftrace_new_addrs); | 189 | static LIST_HEAD(ftrace_new_addrs); |
| 191 | 190 | ||
| @@ -327,96 +326,89 @@ ftrace_record_ip(unsigned long ip) | |||
| 327 | 326 | ||
| 328 | static int | 327 | static int |
| 329 | __ftrace_replace_code(struct dyn_ftrace *rec, | 328 | __ftrace_replace_code(struct dyn_ftrace *rec, |
| 330 | unsigned char *old, unsigned char *new, int enable) | 329 | unsigned char *nop, int enable) |
| 331 | { | 330 | { |
| 332 | unsigned long ip, fl; | 331 | unsigned long ip, fl; |
| 332 | unsigned char *call, *old, *new; | ||
| 333 | 333 | ||
| 334 | ip = rec->ip; | 334 | ip = rec->ip; |
| 335 | 335 | ||
| 336 | if (ftrace_filtered && enable) { | 336 | /* |
| 337 | * If this record is not to be traced and | ||
| 338 | * it is not enabled then do nothing. | ||
| 339 | * | ||
| 340 | * If this record is not to be traced and | ||
| 341 | * it is enabled then disabled it. | ||
| 342 | * | ||
| 343 | */ | ||
| 344 | if (rec->flags & FTRACE_FL_NOTRACE) { | ||
| 345 | if (rec->flags & FTRACE_FL_ENABLED) | ||
| 346 | rec->flags &= ~FTRACE_FL_ENABLED; | ||
| 347 | else | ||
| 348 | return 0; | ||
| 349 | |||
| 350 | } else if (ftrace_filtered && enable) { | ||
| 337 | /* | 351 | /* |
| 338 | * If filtering is on: | 352 | * Filtering is on: |
| 339 | * | ||
| 340 | * If this record is set to be filtered and | ||
| 341 | * is enabled then do nothing. | ||
| 342 | * | ||
| 343 | * If this record is set to be filtered and | ||
| 344 | * it is not enabled, enable it. | ||
| 345 | * | ||
| 346 | * If this record is not set to be filtered | ||
| 347 | * and it is not enabled do nothing. | ||
| 348 | * | ||
| 349 | * If this record is set not to trace then | ||
| 350 | * do nothing. | ||
| 351 | * | ||
| 352 | * If this record is set not to trace and | ||
| 353 | * it is enabled then disable it. | ||
| 354 | * | ||
| 355 | * If this record is not set to be filtered and | ||
| 356 | * it is enabled, disable it. | ||
| 357 | */ | 353 | */ |
| 358 | 354 | ||
| 359 | fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE | | 355 | fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED); |
| 360 | FTRACE_FL_ENABLED); | ||
| 361 | 356 | ||
| 362 | if ((fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) || | 357 | /* Record is filtered and enabled, do nothing */ |
| 363 | (fl == (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE)) || | 358 | if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) |
| 364 | !fl || (fl == FTRACE_FL_NOTRACE)) | ||
| 365 | return 0; | 359 | return 0; |
| 366 | 360 | ||
| 367 | /* | 361 | /* Record is not filtered and is not enabled do nothing */ |
| 368 | * If it is enabled disable it, | 362 | if (!fl) |
| 369 | * otherwise enable it! | 363 | return 0; |
| 370 | */ | 364 | |
| 371 | if (fl & FTRACE_FL_ENABLED) { | 365 | /* Record is not filtered but enabled, disable it */ |
| 372 | /* swap new and old */ | 366 | if (fl == FTRACE_FL_ENABLED) |
| 373 | new = old; | ||
| 374 | old = ftrace_call_replace(ip, FTRACE_ADDR); | ||
| 375 | rec->flags &= ~FTRACE_FL_ENABLED; | 367 | rec->flags &= ~FTRACE_FL_ENABLED; |
| 376 | } else { | 368 | else |
| 377 | new = ftrace_call_replace(ip, FTRACE_ADDR); | 369 | /* Otherwise record is filtered but not enabled, enable it */ |
| 378 | rec->flags |= FTRACE_FL_ENABLED; | 370 | rec->flags |= FTRACE_FL_ENABLED; |
| 379 | } | ||
| 380 | } else { | 371 | } else { |
| 372 | /* Disable or not filtered */ | ||
| 381 | 373 | ||
| 382 | if (enable) { | 374 | if (enable) { |
| 383 | /* | 375 | /* if record is enabled, do nothing */ |
| 384 | * If this record is set not to trace and is | ||
| 385 | * not enabled, do nothing. | ||
| 386 | */ | ||
| 387 | fl = rec->flags & (FTRACE_FL_NOTRACE | FTRACE_FL_ENABLED); | ||
| 388 | if (fl == FTRACE_FL_NOTRACE) | ||
| 389 | return 0; | ||
| 390 | |||
| 391 | new = ftrace_call_replace(ip, FTRACE_ADDR); | ||
| 392 | } else | ||
| 393 | old = ftrace_call_replace(ip, FTRACE_ADDR); | ||
| 394 | |||
| 395 | if (enable) { | ||
| 396 | if (rec->flags & FTRACE_FL_ENABLED) | 376 | if (rec->flags & FTRACE_FL_ENABLED) |
| 397 | return 0; | 377 | return 0; |
| 378 | |||
| 398 | rec->flags |= FTRACE_FL_ENABLED; | 379 | rec->flags |= FTRACE_FL_ENABLED; |
| 380 | |||
| 399 | } else { | 381 | } else { |
| 382 | |||
| 383 | /* if record is not enabled do nothing */ | ||
| 400 | if (!(rec->flags & FTRACE_FL_ENABLED)) | 384 | if (!(rec->flags & FTRACE_FL_ENABLED)) |
| 401 | return 0; | 385 | return 0; |
| 386 | |||
| 402 | rec->flags &= ~FTRACE_FL_ENABLED; | 387 | rec->flags &= ~FTRACE_FL_ENABLED; |
| 403 | } | 388 | } |
| 404 | } | 389 | } |
| 405 | 390 | ||
| 391 | call = ftrace_call_replace(ip, FTRACE_ADDR); | ||
| 392 | |||
| 393 | if (rec->flags & FTRACE_FL_ENABLED) { | ||
| 394 | old = nop; | ||
| 395 | new = call; | ||
| 396 | } else { | ||
| 397 | old = call; | ||
| 398 | new = nop; | ||
| 399 | } | ||
| 400 | |||
| 406 | return ftrace_modify_code(ip, old, new); | 401 | return ftrace_modify_code(ip, old, new); |
| 407 | } | 402 | } |
| 408 | 403 | ||
| 409 | static void ftrace_replace_code(int enable) | 404 | static void ftrace_replace_code(int enable) |
| 410 | { | 405 | { |
| 411 | int i, failed; | 406 | int i, failed; |
| 412 | unsigned char *new = NULL, *old = NULL; | 407 | unsigned char *nop = NULL; |
| 413 | struct dyn_ftrace *rec; | 408 | struct dyn_ftrace *rec; |
| 414 | struct ftrace_page *pg; | 409 | struct ftrace_page *pg; |
| 415 | 410 | ||
| 416 | if (enable) | 411 | nop = ftrace_nop_replace(); |
| 417 | old = ftrace_nop_replace(); | ||
| 418 | else | ||
| 419 | new = ftrace_nop_replace(); | ||
| 420 | 412 | ||
| 421 | for (pg = ftrace_pages_start; pg; pg = pg->next) { | 413 | for (pg = ftrace_pages_start; pg; pg = pg->next) { |
| 422 | for (i = 0; i < pg->index; i++) { | 414 | for (i = 0; i < pg->index; i++) { |
| @@ -434,7 +426,7 @@ static void ftrace_replace_code(int enable) | |||
| 434 | unfreeze_record(rec); | 426 | unfreeze_record(rec); |
| 435 | } | 427 | } |
| 436 | 428 | ||
| 437 | failed = __ftrace_replace_code(rec, old, new, enable); | 429 | failed = __ftrace_replace_code(rec, nop, enable); |
| 438 | if (failed && (rec->flags & FTRACE_FL_CONVERTED)) { | 430 | if (failed && (rec->flags & FTRACE_FL_CONVERTED)) { |
| 439 | rec->flags |= FTRACE_FL_FAILED; | 431 | rec->flags |= FTRACE_FL_FAILED; |
| 440 | if ((system_state == SYSTEM_BOOTING) || | 432 | if ((system_state == SYSTEM_BOOTING) || |
| @@ -506,13 +498,10 @@ static int __ftrace_modify_code(void *data) | |||
| 506 | { | 498 | { |
| 507 | int *command = data; | 499 | int *command = data; |
| 508 | 500 | ||
| 509 | if (*command & FTRACE_ENABLE_CALLS) { | 501 | if (*command & FTRACE_ENABLE_CALLS) |
| 510 | ftrace_replace_code(1); | 502 | ftrace_replace_code(1); |
| 511 | tracing_on = 1; | 503 | else if (*command & FTRACE_DISABLE_CALLS) |
| 512 | } else if (*command & FTRACE_DISABLE_CALLS) { | ||
| 513 | ftrace_replace_code(0); | 504 | ftrace_replace_code(0); |
| 514 | tracing_on = 0; | ||
| 515 | } | ||
| 516 | 505 | ||
| 517 | if (*command & FTRACE_UPDATE_TRACE_FUNC) | 506 | if (*command & FTRACE_UPDATE_TRACE_FUNC) |
| 518 | ftrace_update_ftrace_func(ftrace_trace_function); | 507 | ftrace_update_ftrace_func(ftrace_trace_function); |
| @@ -538,8 +527,7 @@ static void ftrace_startup(void) | |||
| 538 | 527 | ||
| 539 | mutex_lock(&ftrace_start_lock); | 528 | mutex_lock(&ftrace_start_lock); |
| 540 | ftrace_start++; | 529 | ftrace_start++; |
| 541 | if (ftrace_start == 1) | 530 | command |= FTRACE_ENABLE_CALLS; |
| 542 | command |= FTRACE_ENABLE_CALLS; | ||
| 543 | 531 | ||
| 544 | if (saved_ftrace_func != ftrace_trace_function) { | 532 | if (saved_ftrace_func != ftrace_trace_function) { |
| 545 | saved_ftrace_func = ftrace_trace_function; | 533 | saved_ftrace_func = ftrace_trace_function; |
| @@ -677,7 +665,7 @@ static int __init ftrace_dyn_table_alloc(unsigned long num_to_init) | |||
| 677 | 665 | ||
| 678 | cnt = num_to_init / ENTRIES_PER_PAGE; | 666 | cnt = num_to_init / ENTRIES_PER_PAGE; |
| 679 | pr_info("ftrace: allocating %ld entries in %d pages\n", | 667 | pr_info("ftrace: allocating %ld entries in %d pages\n", |
| 680 | num_to_init, cnt); | 668 | num_to_init, cnt + 1); |
| 681 | 669 | ||
| 682 | for (i = 0; i < cnt; i++) { | 670 | for (i = 0; i < cnt; i++) { |
| 683 | pg->next = (void *)get_zeroed_page(GFP_KERNEL); | 671 | pg->next = (void *)get_zeroed_page(GFP_KERNEL); |
| @@ -738,6 +726,9 @@ t_next(struct seq_file *m, void *v, loff_t *pos) | |||
| 738 | ((iter->flags & FTRACE_ITER_FAILURES) && | 726 | ((iter->flags & FTRACE_ITER_FAILURES) && |
| 739 | !(rec->flags & FTRACE_FL_FAILED)) || | 727 | !(rec->flags & FTRACE_FL_FAILED)) || |
| 740 | 728 | ||
| 729 | ((iter->flags & FTRACE_ITER_FILTER) && | ||
| 730 | !(rec->flags & FTRACE_FL_FILTER)) || | ||
| 731 | |||
| 741 | ((iter->flags & FTRACE_ITER_NOTRACE) && | 732 | ((iter->flags & FTRACE_ITER_NOTRACE) && |
| 742 | !(rec->flags & FTRACE_FL_NOTRACE))) { | 733 | !(rec->flags & FTRACE_FL_NOTRACE))) { |
| 743 | rec = NULL; | 734 | rec = NULL; |
| @@ -757,13 +748,11 @@ static void *t_start(struct seq_file *m, loff_t *pos) | |||
| 757 | void *p = NULL; | 748 | void *p = NULL; |
| 758 | loff_t l = -1; | 749 | loff_t l = -1; |
| 759 | 750 | ||
| 760 | if (*pos != iter->pos) { | 751 | if (*pos > iter->pos) |
| 761 | for (p = t_next(m, p, &l); p && l < *pos; p = t_next(m, p, &l)) | 752 | *pos = iter->pos; |
| 762 | ; | 753 | |
| 763 | } else { | 754 | l = *pos; |
| 764 | l = *pos; | 755 | p = t_next(m, p, &l); |
| 765 | p = t_next(m, p, &l); | ||
| 766 | } | ||
| 767 | 756 | ||
| 768 | return p; | 757 | return p; |
| 769 | } | 758 | } |
| @@ -774,15 +763,21 @@ static void t_stop(struct seq_file *m, void *p) | |||
| 774 | 763 | ||
| 775 | static int t_show(struct seq_file *m, void *v) | 764 | static int t_show(struct seq_file *m, void *v) |
| 776 | { | 765 | { |
| 766 | struct ftrace_iterator *iter = m->private; | ||
| 777 | struct dyn_ftrace *rec = v; | 767 | struct dyn_ftrace *rec = v; |
| 778 | char str[KSYM_SYMBOL_LEN]; | 768 | char str[KSYM_SYMBOL_LEN]; |
| 769 | int ret = 0; | ||
| 779 | 770 | ||
| 780 | if (!rec) | 771 | if (!rec) |
| 781 | return 0; | 772 | return 0; |
| 782 | 773 | ||
| 783 | kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); | 774 | kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); |
| 784 | 775 | ||
| 785 | seq_printf(m, "%s\n", str); | 776 | ret = seq_printf(m, "%s\n", str); |
| 777 | if (ret < 0) { | ||
| 778 | iter->pos--; | ||
| 779 | iter->idx--; | ||
| 780 | } | ||
| 786 | 781 | ||
| 787 | return 0; | 782 | return 0; |
| 788 | } | 783 | } |
| @@ -808,7 +803,7 @@ ftrace_avail_open(struct inode *inode, struct file *file) | |||
| 808 | return -ENOMEM; | 803 | return -ENOMEM; |
| 809 | 804 | ||
| 810 | iter->pg = ftrace_pages_start; | 805 | iter->pg = ftrace_pages_start; |
| 811 | iter->pos = -1; | 806 | iter->pos = 0; |
| 812 | 807 | ||
| 813 | ret = seq_open(file, &show_ftrace_seq_ops); | 808 | ret = seq_open(file, &show_ftrace_seq_ops); |
| 814 | if (!ret) { | 809 | if (!ret) { |
| @@ -895,7 +890,7 @@ ftrace_regex_open(struct inode *inode, struct file *file, int enable) | |||
| 895 | 890 | ||
| 896 | if (file->f_mode & FMODE_READ) { | 891 | if (file->f_mode & FMODE_READ) { |
| 897 | iter->pg = ftrace_pages_start; | 892 | iter->pg = ftrace_pages_start; |
| 898 | iter->pos = -1; | 893 | iter->pos = 0; |
| 899 | iter->flags = enable ? FTRACE_ITER_FILTER : | 894 | iter->flags = enable ? FTRACE_ITER_FILTER : |
| 900 | FTRACE_ITER_NOTRACE; | 895 | FTRACE_ITER_NOTRACE; |
| 901 | 896 | ||
| @@ -1186,7 +1181,7 @@ ftrace_regex_release(struct inode *inode, struct file *file, int enable) | |||
| 1186 | 1181 | ||
| 1187 | mutex_lock(&ftrace_sysctl_lock); | 1182 | mutex_lock(&ftrace_sysctl_lock); |
| 1188 | mutex_lock(&ftrace_start_lock); | 1183 | mutex_lock(&ftrace_start_lock); |
| 1189 | if (iter->filtered && ftrace_start && ftrace_enabled) | 1184 | if (ftrace_start && ftrace_enabled) |
| 1190 | ftrace_run_update_code(FTRACE_ENABLE_CALLS); | 1185 | ftrace_run_update_code(FTRACE_ENABLE_CALLS); |
| 1191 | mutex_unlock(&ftrace_start_lock); | 1186 | mutex_unlock(&ftrace_start_lock); |
| 1192 | mutex_unlock(&ftrace_sysctl_lock); | 1187 | mutex_unlock(&ftrace_sysctl_lock); |
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index cedf4e268285..f780e9552f91 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c | |||
| @@ -16,14 +16,49 @@ | |||
| 16 | #include <linux/list.h> | 16 | #include <linux/list.h> |
| 17 | #include <linux/fs.h> | 17 | #include <linux/fs.h> |
| 18 | 18 | ||
| 19 | #include "trace.h" | ||
| 20 | |||
| 21 | /* Global flag to disable all recording to ring buffers */ | ||
| 22 | static int ring_buffers_off __read_mostly; | ||
| 23 | |||
| 24 | /** | ||
| 25 | * tracing_on - enable all tracing buffers | ||
| 26 | * | ||
| 27 | * This function enables all tracing buffers that may have been | ||
| 28 | * disabled with tracing_off. | ||
| 29 | */ | ||
| 30 | void tracing_on(void) | ||
| 31 | { | ||
| 32 | ring_buffers_off = 0; | ||
| 33 | } | ||
| 34 | |||
| 35 | /** | ||
| 36 | * tracing_off - turn off all tracing buffers | ||
| 37 | * | ||
| 38 | * This function stops all tracing buffers from recording data. | ||
| 39 | * It does not disable any overhead the tracers themselves may | ||
| 40 | * be causing. This function simply causes all recording to | ||
| 41 | * the ring buffers to fail. | ||
| 42 | */ | ||
| 43 | void tracing_off(void) | ||
| 44 | { | ||
| 45 | ring_buffers_off = 1; | ||
| 46 | } | ||
| 47 | |||
| 19 | /* Up this if you want to test the TIME_EXTENTS and normalization */ | 48 | /* Up this if you want to test the TIME_EXTENTS and normalization */ |
| 20 | #define DEBUG_SHIFT 0 | 49 | #define DEBUG_SHIFT 0 |
| 21 | 50 | ||
| 22 | /* FIXME!!! */ | 51 | /* FIXME!!! */ |
| 23 | u64 ring_buffer_time_stamp(int cpu) | 52 | u64 ring_buffer_time_stamp(int cpu) |
| 24 | { | 53 | { |
| 54 | u64 time; | ||
| 55 | |||
| 56 | preempt_disable_notrace(); | ||
| 25 | /* shift to debug/test normalization and TIME_EXTENTS */ | 57 | /* shift to debug/test normalization and TIME_EXTENTS */ |
| 26 | return sched_clock() << DEBUG_SHIFT; | 58 | time = sched_clock() << DEBUG_SHIFT; |
| 59 | preempt_enable_notrace(); | ||
| 60 | |||
| 61 | return time; | ||
| 27 | } | 62 | } |
| 28 | 63 | ||
| 29 | void ring_buffer_normalize_time_stamp(int cpu, u64 *ts) | 64 | void ring_buffer_normalize_time_stamp(int cpu, u64 *ts) |
| @@ -503,6 +538,12 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) | |||
| 503 | LIST_HEAD(pages); | 538 | LIST_HEAD(pages); |
| 504 | int i, cpu; | 539 | int i, cpu; |
| 505 | 540 | ||
| 541 | /* | ||
| 542 | * Always succeed at resizing a non-existent buffer: | ||
| 543 | */ | ||
| 544 | if (!buffer) | ||
| 545 | return size; | ||
| 546 | |||
| 506 | size = DIV_ROUND_UP(size, BUF_PAGE_SIZE); | 547 | size = DIV_ROUND_UP(size, BUF_PAGE_SIZE); |
| 507 | size *= BUF_PAGE_SIZE; | 548 | size *= BUF_PAGE_SIZE; |
| 508 | buffer_size = buffer->pages * BUF_PAGE_SIZE; | 549 | buffer_size = buffer->pages * BUF_PAGE_SIZE; |
| @@ -576,6 +617,7 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) | |||
| 576 | list_del_init(&page->list); | 617 | list_del_init(&page->list); |
| 577 | free_buffer_page(page); | 618 | free_buffer_page(page); |
| 578 | } | 619 | } |
| 620 | mutex_unlock(&buffer->mutex); | ||
| 579 | return -ENOMEM; | 621 | return -ENOMEM; |
| 580 | } | 622 | } |
| 581 | 623 | ||
| @@ -1022,8 +1064,23 @@ rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer, | |||
| 1022 | struct ring_buffer_event *event; | 1064 | struct ring_buffer_event *event; |
| 1023 | u64 ts, delta; | 1065 | u64 ts, delta; |
| 1024 | int commit = 0; | 1066 | int commit = 0; |
| 1067 | int nr_loops = 0; | ||
| 1025 | 1068 | ||
| 1026 | again: | 1069 | again: |
| 1070 | /* | ||
| 1071 | * We allow for interrupts to reenter here and do a trace. | ||
| 1072 | * If one does, it will cause this original code to loop | ||
| 1073 | * back here. Even with heavy interrupts happening, this | ||
| 1074 | * should only happen a few times in a row. If this happens | ||
| 1075 | * 1000 times in a row, there must be either an interrupt | ||
| 1076 | * storm or we have something buggy. | ||
| 1077 | * Bail! | ||
| 1078 | */ | ||
| 1079 | if (unlikely(++nr_loops > 1000)) { | ||
| 1080 | RB_WARN_ON(cpu_buffer, 1); | ||
| 1081 | return NULL; | ||
| 1082 | } | ||
| 1083 | |||
| 1027 | ts = ring_buffer_time_stamp(cpu_buffer->cpu); | 1084 | ts = ring_buffer_time_stamp(cpu_buffer->cpu); |
| 1028 | 1085 | ||
| 1029 | /* | 1086 | /* |
| @@ -1045,7 +1102,7 @@ rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer, | |||
| 1045 | 1102 | ||
| 1046 | /* Did the write stamp get updated already? */ | 1103 | /* Did the write stamp get updated already? */ |
| 1047 | if (unlikely(ts < cpu_buffer->write_stamp)) | 1104 | if (unlikely(ts < cpu_buffer->write_stamp)) |
| 1048 | goto again; | 1105 | delta = 0; |
| 1049 | 1106 | ||
| 1050 | if (test_time_stamp(delta)) { | 1107 | if (test_time_stamp(delta)) { |
| 1051 | 1108 | ||
| @@ -1118,6 +1175,9 @@ ring_buffer_lock_reserve(struct ring_buffer *buffer, | |||
| 1118 | struct ring_buffer_event *event; | 1175 | struct ring_buffer_event *event; |
| 1119 | int cpu, resched; | 1176 | int cpu, resched; |
| 1120 | 1177 | ||
| 1178 | if (ring_buffers_off) | ||
| 1179 | return NULL; | ||
| 1180 | |||
| 1121 | if (atomic_read(&buffer->record_disabled)) | 1181 | if (atomic_read(&buffer->record_disabled)) |
| 1122 | return NULL; | 1182 | return NULL; |
| 1123 | 1183 | ||
| @@ -1234,6 +1294,9 @@ int ring_buffer_write(struct ring_buffer *buffer, | |||
| 1234 | int ret = -EBUSY; | 1294 | int ret = -EBUSY; |
| 1235 | int cpu, resched; | 1295 | int cpu, resched; |
| 1236 | 1296 | ||
| 1297 | if (ring_buffers_off) | ||
| 1298 | return -EBUSY; | ||
| 1299 | |||
| 1237 | if (atomic_read(&buffer->record_disabled)) | 1300 | if (atomic_read(&buffer->record_disabled)) |
| 1238 | return -EBUSY; | 1301 | return -EBUSY; |
| 1239 | 1302 | ||
| @@ -1532,10 +1595,23 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) | |||
| 1532 | { | 1595 | { |
| 1533 | struct buffer_page *reader = NULL; | 1596 | struct buffer_page *reader = NULL; |
| 1534 | unsigned long flags; | 1597 | unsigned long flags; |
| 1598 | int nr_loops = 0; | ||
| 1535 | 1599 | ||
| 1536 | spin_lock_irqsave(&cpu_buffer->lock, flags); | 1600 | spin_lock_irqsave(&cpu_buffer->lock, flags); |
| 1537 | 1601 | ||
| 1538 | again: | 1602 | again: |
| 1603 | /* | ||
| 1604 | * This should normally only loop twice. But because the | ||
| 1605 | * start of the reader inserts an empty page, it causes | ||
| 1606 | * a case where we will loop three times. There should be no | ||
| 1607 | * reason to loop four times (that I know of). | ||
| 1608 | */ | ||
| 1609 | if (unlikely(++nr_loops > 3)) { | ||
| 1610 | RB_WARN_ON(cpu_buffer, 1); | ||
| 1611 | reader = NULL; | ||
| 1612 | goto out; | ||
| 1613 | } | ||
| 1614 | |||
| 1539 | reader = cpu_buffer->reader_page; | 1615 | reader = cpu_buffer->reader_page; |
| 1540 | 1616 | ||
| 1541 | /* If there's more to read, return this page */ | 1617 | /* If there's more to read, return this page */ |
| @@ -1665,6 +1741,7 @@ ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts) | |||
| 1665 | struct ring_buffer_per_cpu *cpu_buffer; | 1741 | struct ring_buffer_per_cpu *cpu_buffer; |
| 1666 | struct ring_buffer_event *event; | 1742 | struct ring_buffer_event *event; |
| 1667 | struct buffer_page *reader; | 1743 | struct buffer_page *reader; |
| 1744 | int nr_loops = 0; | ||
| 1668 | 1745 | ||
| 1669 | if (!cpu_isset(cpu, buffer->cpumask)) | 1746 | if (!cpu_isset(cpu, buffer->cpumask)) |
| 1670 | return NULL; | 1747 | return NULL; |
| @@ -1672,6 +1749,19 @@ ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts) | |||
| 1672 | cpu_buffer = buffer->buffers[cpu]; | 1749 | cpu_buffer = buffer->buffers[cpu]; |
| 1673 | 1750 | ||
| 1674 | again: | 1751 | again: |
| 1752 | /* | ||
| 1753 | * We repeat when a timestamp is encountered. It is possible | ||
| 1754 | * to get multiple timestamps from an interrupt entering just | ||
| 1755 | * as one timestamp is about to be written. The max times | ||
| 1756 | * that this can happen is the number of nested interrupts we | ||
| 1757 | * can have. Nesting 10 deep of interrupts is clearly | ||
| 1758 | * an anomaly. | ||
| 1759 | */ | ||
| 1760 | if (unlikely(++nr_loops > 10)) { | ||
| 1761 | RB_WARN_ON(cpu_buffer, 1); | ||
| 1762 | return NULL; | ||
| 1763 | } | ||
| 1764 | |||
| 1675 | reader = rb_get_reader_page(cpu_buffer); | 1765 | reader = rb_get_reader_page(cpu_buffer); |
| 1676 | if (!reader) | 1766 | if (!reader) |
| 1677 | return NULL; | 1767 | return NULL; |
| @@ -1722,6 +1812,7 @@ ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts) | |||
| 1722 | struct ring_buffer *buffer; | 1812 | struct ring_buffer *buffer; |
| 1723 | struct ring_buffer_per_cpu *cpu_buffer; | 1813 | struct ring_buffer_per_cpu *cpu_buffer; |
| 1724 | struct ring_buffer_event *event; | 1814 | struct ring_buffer_event *event; |
| 1815 | int nr_loops = 0; | ||
| 1725 | 1816 | ||
| 1726 | if (ring_buffer_iter_empty(iter)) | 1817 | if (ring_buffer_iter_empty(iter)) |
| 1727 | return NULL; | 1818 | return NULL; |
| @@ -1730,6 +1821,19 @@ ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts) | |||
| 1730 | buffer = cpu_buffer->buffer; | 1821 | buffer = cpu_buffer->buffer; |
| 1731 | 1822 | ||
| 1732 | again: | 1823 | again: |
| 1824 | /* | ||
| 1825 | * We repeat when a timestamp is encountered. It is possible | ||
| 1826 | * to get multiple timestamps from an interrupt entering just | ||
| 1827 | * as one timestamp is about to be written. The max times | ||
| 1828 | * that this can happen is the number of nested interrupts we | ||
| 1829 | * can have. Nesting 10 deep of interrupts is clearly | ||
| 1830 | * an anomaly. | ||
| 1831 | */ | ||
| 1832 | if (unlikely(++nr_loops > 10)) { | ||
| 1833 | RB_WARN_ON(cpu_buffer, 1); | ||
| 1834 | return NULL; | ||
| 1835 | } | ||
| 1836 | |||
| 1733 | if (rb_per_cpu_empty(cpu_buffer)) | 1837 | if (rb_per_cpu_empty(cpu_buffer)) |
| 1734 | return NULL; | 1838 | return NULL; |
| 1735 | 1839 | ||
| @@ -2014,3 +2118,69 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a, | |||
| 2014 | return 0; | 2118 | return 0; |
| 2015 | } | 2119 | } |
| 2016 | 2120 | ||
| 2121 | static ssize_t | ||
| 2122 | rb_simple_read(struct file *filp, char __user *ubuf, | ||
| 2123 | size_t cnt, loff_t *ppos) | ||
| 2124 | { | ||
| 2125 | int *p = filp->private_data; | ||
| 2126 | char buf[64]; | ||
| 2127 | int r; | ||
| 2128 | |||
| 2129 | /* !ring_buffers_off == tracing_on */ | ||
| 2130 | r = sprintf(buf, "%d\n", !*p); | ||
| 2131 | |||
| 2132 | return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); | ||
| 2133 | } | ||
| 2134 | |||
| 2135 | static ssize_t | ||
| 2136 | rb_simple_write(struct file *filp, const char __user *ubuf, | ||
| 2137 | size_t cnt, loff_t *ppos) | ||
| 2138 | { | ||
| 2139 | int *p = filp->private_data; | ||
| 2140 | char buf[64]; | ||
| 2141 | long val; | ||
| 2142 | int ret; | ||
| 2143 | |||
| 2144 | if (cnt >= sizeof(buf)) | ||
| 2145 | return -EINVAL; | ||
| 2146 | |||
| 2147 | if (copy_from_user(&buf, ubuf, cnt)) | ||
| 2148 | return -EFAULT; | ||
| 2149 | |||
| 2150 | buf[cnt] = 0; | ||
| 2151 | |||
| 2152 | ret = strict_strtoul(buf, 10, &val); | ||
| 2153 | if (ret < 0) | ||
| 2154 | return ret; | ||
| 2155 | |||
| 2156 | /* !ring_buffers_off == tracing_on */ | ||
| 2157 | *p = !val; | ||
| 2158 | |||
| 2159 | (*ppos)++; | ||
| 2160 | |||
| 2161 | return cnt; | ||
| 2162 | } | ||
| 2163 | |||
| 2164 | static struct file_operations rb_simple_fops = { | ||
| 2165 | .open = tracing_open_generic, | ||
| 2166 | .read = rb_simple_read, | ||
| 2167 | .write = rb_simple_write, | ||
| 2168 | }; | ||
| 2169 | |||
| 2170 | |||
| 2171 | static __init int rb_init_debugfs(void) | ||
| 2172 | { | ||
| 2173 | struct dentry *d_tracer; | ||
| 2174 | struct dentry *entry; | ||
| 2175 | |||
| 2176 | d_tracer = tracing_init_dentry(); | ||
| 2177 | |||
| 2178 | entry = debugfs_create_file("tracing_on", 0644, d_tracer, | ||
| 2179 | &ring_buffers_off, &rb_simple_fops); | ||
| 2180 | if (!entry) | ||
| 2181 | pr_warning("Could not create debugfs 'tracing_on' entry\n"); | ||
| 2182 | |||
| 2183 | return 0; | ||
| 2184 | } | ||
| 2185 | |||
| 2186 | fs_initcall(rb_init_debugfs); | ||
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 8a499e2adaec..d86e3252f300 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
| @@ -705,6 +705,7 @@ static void ftrace_trace_stack(struct trace_array *tr, | |||
| 705 | unsigned long flags, | 705 | unsigned long flags, |
| 706 | int skip, int pc) | 706 | int skip, int pc) |
| 707 | { | 707 | { |
| 708 | #ifdef CONFIG_STACKTRACE | ||
| 708 | struct ring_buffer_event *event; | 709 | struct ring_buffer_event *event; |
| 709 | struct stack_entry *entry; | 710 | struct stack_entry *entry; |
| 710 | struct stack_trace trace; | 711 | struct stack_trace trace; |
| @@ -730,6 +731,7 @@ static void ftrace_trace_stack(struct trace_array *tr, | |||
| 730 | 731 | ||
| 731 | save_stack_trace(&trace); | 732 | save_stack_trace(&trace); |
| 732 | ring_buffer_unlock_commit(tr->buffer, event, irq_flags); | 733 | ring_buffer_unlock_commit(tr->buffer, event, irq_flags); |
| 734 | #endif | ||
| 733 | } | 735 | } |
| 734 | 736 | ||
| 735 | void __trace_stack(struct trace_array *tr, | 737 | void __trace_stack(struct trace_array *tr, |
| @@ -1086,17 +1088,20 @@ static void s_stop(struct seq_file *m, void *p) | |||
| 1086 | mutex_unlock(&trace_types_lock); | 1088 | mutex_unlock(&trace_types_lock); |
| 1087 | } | 1089 | } |
| 1088 | 1090 | ||
| 1089 | #define KRETPROBE_MSG "[unknown/kretprobe'd]" | ||
| 1090 | |||
| 1091 | #ifdef CONFIG_KRETPROBES | 1091 | #ifdef CONFIG_KRETPROBES |
| 1092 | static inline int kretprobed(unsigned long addr) | 1092 | static inline const char *kretprobed(const char *name) |
| 1093 | { | 1093 | { |
| 1094 | return addr == (unsigned long)kretprobe_trampoline; | 1094 | static const char tramp_name[] = "kretprobe_trampoline"; |
| 1095 | int size = sizeof(tramp_name); | ||
| 1096 | |||
| 1097 | if (strncmp(tramp_name, name, size) == 0) | ||
| 1098 | return "[unknown/kretprobe'd]"; | ||
| 1099 | return name; | ||
| 1095 | } | 1100 | } |
| 1096 | #else | 1101 | #else |
| 1097 | static inline int kretprobed(unsigned long addr) | 1102 | static inline const char *kretprobed(const char *name) |
| 1098 | { | 1103 | { |
| 1099 | return 0; | 1104 | return name; |
| 1100 | } | 1105 | } |
| 1101 | #endif /* CONFIG_KRETPROBES */ | 1106 | #endif /* CONFIG_KRETPROBES */ |
| 1102 | 1107 | ||
| @@ -1105,10 +1110,13 @@ seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address) | |||
| 1105 | { | 1110 | { |
| 1106 | #ifdef CONFIG_KALLSYMS | 1111 | #ifdef CONFIG_KALLSYMS |
| 1107 | char str[KSYM_SYMBOL_LEN]; | 1112 | char str[KSYM_SYMBOL_LEN]; |
| 1113 | const char *name; | ||
| 1108 | 1114 | ||
| 1109 | kallsyms_lookup(address, NULL, NULL, NULL, str); | 1115 | kallsyms_lookup(address, NULL, NULL, NULL, str); |
| 1110 | 1116 | ||
| 1111 | return trace_seq_printf(s, fmt, str); | 1117 | name = kretprobed(str); |
| 1118 | |||
| 1119 | return trace_seq_printf(s, fmt, name); | ||
| 1112 | #endif | 1120 | #endif |
| 1113 | return 1; | 1121 | return 1; |
| 1114 | } | 1122 | } |
| @@ -1119,9 +1127,12 @@ seq_print_sym_offset(struct trace_seq *s, const char *fmt, | |||
| 1119 | { | 1127 | { |
| 1120 | #ifdef CONFIG_KALLSYMS | 1128 | #ifdef CONFIG_KALLSYMS |
| 1121 | char str[KSYM_SYMBOL_LEN]; | 1129 | char str[KSYM_SYMBOL_LEN]; |
| 1130 | const char *name; | ||
| 1122 | 1131 | ||
| 1123 | sprint_symbol(str, address); | 1132 | sprint_symbol(str, address); |
| 1124 | return trace_seq_printf(s, fmt, str); | 1133 | name = kretprobed(str); |
| 1134 | |||
| 1135 | return trace_seq_printf(s, fmt, name); | ||
| 1125 | #endif | 1136 | #endif |
| 1126 | return 1; | 1137 | return 1; |
| 1127 | } | 1138 | } |
| @@ -1375,10 +1386,7 @@ print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu) | |||
| 1375 | 1386 | ||
| 1376 | seq_print_ip_sym(s, field->ip, sym_flags); | 1387 | seq_print_ip_sym(s, field->ip, sym_flags); |
| 1377 | trace_seq_puts(s, " ("); | 1388 | trace_seq_puts(s, " ("); |
| 1378 | if (kretprobed(field->parent_ip)) | 1389 | seq_print_ip_sym(s, field->parent_ip, sym_flags); |
| 1379 | trace_seq_puts(s, KRETPROBE_MSG); | ||
| 1380 | else | ||
| 1381 | seq_print_ip_sym(s, field->parent_ip, sym_flags); | ||
| 1382 | trace_seq_puts(s, ")\n"); | 1390 | trace_seq_puts(s, ")\n"); |
| 1383 | break; | 1391 | break; |
| 1384 | } | 1392 | } |
| @@ -1494,12 +1502,9 @@ static enum print_line_t print_trace_fmt(struct trace_iterator *iter) | |||
| 1494 | ret = trace_seq_printf(s, " <-"); | 1502 | ret = trace_seq_printf(s, " <-"); |
| 1495 | if (!ret) | 1503 | if (!ret) |
| 1496 | return TRACE_TYPE_PARTIAL_LINE; | 1504 | return TRACE_TYPE_PARTIAL_LINE; |
| 1497 | if (kretprobed(field->parent_ip)) | 1505 | ret = seq_print_ip_sym(s, |
| 1498 | ret = trace_seq_puts(s, KRETPROBE_MSG); | 1506 | field->parent_ip, |
| 1499 | else | 1507 | sym_flags); |
| 1500 | ret = seq_print_ip_sym(s, | ||
| 1501 | field->parent_ip, | ||
| 1502 | sym_flags); | ||
| 1503 | if (!ret) | 1508 | if (!ret) |
| 1504 | return TRACE_TYPE_PARTIAL_LINE; | 1509 | return TRACE_TYPE_PARTIAL_LINE; |
| 1505 | } | 1510 | } |
| @@ -1750,7 +1755,7 @@ static enum print_line_t print_bin_fmt(struct trace_iterator *iter) | |||
| 1750 | return TRACE_TYPE_HANDLED; | 1755 | return TRACE_TYPE_HANDLED; |
| 1751 | 1756 | ||
| 1752 | SEQ_PUT_FIELD_RET(s, entry->pid); | 1757 | SEQ_PUT_FIELD_RET(s, entry->pid); |
| 1753 | SEQ_PUT_FIELD_RET(s, iter->cpu); | 1758 | SEQ_PUT_FIELD_RET(s, entry->cpu); |
| 1754 | SEQ_PUT_FIELD_RET(s, iter->ts); | 1759 | SEQ_PUT_FIELD_RET(s, iter->ts); |
| 1755 | 1760 | ||
| 1756 | switch (entry->type) { | 1761 | switch (entry->type) { |
| @@ -1931,6 +1936,7 @@ __tracing_open(struct inode *inode, struct file *file, int *ret) | |||
| 1931 | ring_buffer_read_finish(iter->buffer_iter[cpu]); | 1936 | ring_buffer_read_finish(iter->buffer_iter[cpu]); |
| 1932 | } | 1937 | } |
| 1933 | mutex_unlock(&trace_types_lock); | 1938 | mutex_unlock(&trace_types_lock); |
| 1939 | kfree(iter); | ||
| 1934 | 1940 | ||
| 1935 | return ERR_PTR(-ENOMEM); | 1941 | return ERR_PTR(-ENOMEM); |
| 1936 | } | 1942 | } |
| @@ -2671,7 +2677,7 @@ tracing_entries_write(struct file *filp, const char __user *ubuf, | |||
| 2671 | { | 2677 | { |
| 2672 | unsigned long val; | 2678 | unsigned long val; |
| 2673 | char buf[64]; | 2679 | char buf[64]; |
| 2674 | int ret; | 2680 | int ret, cpu; |
| 2675 | struct trace_array *tr = filp->private_data; | 2681 | struct trace_array *tr = filp->private_data; |
| 2676 | 2682 | ||
| 2677 | if (cnt >= sizeof(buf)) | 2683 | if (cnt >= sizeof(buf)) |
| @@ -2699,6 +2705,14 @@ tracing_entries_write(struct file *filp, const char __user *ubuf, | |||
| 2699 | goto out; | 2705 | goto out; |
| 2700 | } | 2706 | } |
| 2701 | 2707 | ||
| 2708 | /* disable all cpu buffers */ | ||
| 2709 | for_each_tracing_cpu(cpu) { | ||
| 2710 | if (global_trace.data[cpu]) | ||
| 2711 | atomic_inc(&global_trace.data[cpu]->disabled); | ||
| 2712 | if (max_tr.data[cpu]) | ||
| 2713 | atomic_inc(&max_tr.data[cpu]->disabled); | ||
| 2714 | } | ||
| 2715 | |||
| 2702 | if (val != global_trace.entries) { | 2716 | if (val != global_trace.entries) { |
| 2703 | ret = ring_buffer_resize(global_trace.buffer, val); | 2717 | ret = ring_buffer_resize(global_trace.buffer, val); |
| 2704 | if (ret < 0) { | 2718 | if (ret < 0) { |
| @@ -2730,6 +2744,13 @@ tracing_entries_write(struct file *filp, const char __user *ubuf, | |||
| 2730 | if (tracing_disabled) | 2744 | if (tracing_disabled) |
| 2731 | cnt = -ENOMEM; | 2745 | cnt = -ENOMEM; |
| 2732 | out: | 2746 | out: |
| 2747 | for_each_tracing_cpu(cpu) { | ||
| 2748 | if (global_trace.data[cpu]) | ||
| 2749 | atomic_dec(&global_trace.data[cpu]->disabled); | ||
| 2750 | if (max_tr.data[cpu]) | ||
| 2751 | atomic_dec(&max_tr.data[cpu]->disabled); | ||
| 2752 | } | ||
| 2753 | |||
| 2733 | max_tr.entries = global_trace.entries; | 2754 | max_tr.entries = global_trace.entries; |
| 2734 | mutex_unlock(&trace_types_lock); | 2755 | mutex_unlock(&trace_types_lock); |
| 2735 | 2756 | ||
diff --git a/kernel/workqueue.c b/kernel/workqueue.c index f928f2a87b9b..d4dc69ddebd7 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c | |||
| @@ -970,6 +970,51 @@ undo: | |||
| 970 | return ret; | 970 | return ret; |
| 971 | } | 971 | } |
| 972 | 972 | ||
| 973 | #ifdef CONFIG_SMP | ||
| 974 | struct work_for_cpu { | ||
| 975 | struct work_struct work; | ||
| 976 | long (*fn)(void *); | ||
| 977 | void *arg; | ||
| 978 | long ret; | ||
| 979 | }; | ||
| 980 | |||
| 981 | static void do_work_for_cpu(struct work_struct *w) | ||
| 982 | { | ||
| 983 | struct work_for_cpu *wfc = container_of(w, struct work_for_cpu, work); | ||
| 984 | |||
| 985 | wfc->ret = wfc->fn(wfc->arg); | ||
| 986 | } | ||
| 987 | |||
| 988 | /** | ||
| 989 | * work_on_cpu - run a function in user context on a particular cpu | ||
| 990 | * @cpu: the cpu to run on | ||
| 991 | * @fn: the function to run | ||
| 992 | * @arg: the function arg | ||
| 993 | * | ||
| 994 | * This will return -EINVAL in the cpu is not online, or the return value | ||
| 995 | * of @fn otherwise. | ||
| 996 | */ | ||
| 997 | long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg) | ||
| 998 | { | ||
| 999 | struct work_for_cpu wfc; | ||
| 1000 | |||
| 1001 | INIT_WORK(&wfc.work, do_work_for_cpu); | ||
| 1002 | wfc.fn = fn; | ||
| 1003 | wfc.arg = arg; | ||
| 1004 | get_online_cpus(); | ||
| 1005 | if (unlikely(!cpu_online(cpu))) | ||
| 1006 | wfc.ret = -EINVAL; | ||
| 1007 | else { | ||
| 1008 | schedule_work_on(cpu, &wfc.work); | ||
| 1009 | flush_work(&wfc.work); | ||
| 1010 | } | ||
| 1011 | put_online_cpus(); | ||
| 1012 | |||
| 1013 | return wfc.ret; | ||
| 1014 | } | ||
| 1015 | EXPORT_SYMBOL_GPL(work_on_cpu); | ||
| 1016 | #endif /* CONFIG_SMP */ | ||
| 1017 | |||
| 973 | void __init init_workqueues(void) | 1018 | void __init init_workqueues(void) |
| 974 | { | 1019 | { |
| 975 | cpu_populated_map = cpu_online_map; | 1020 | cpu_populated_map = cpu_online_map; |
