diff options
Diffstat (limited to 'kernel')
56 files changed, 197 insertions, 84 deletions
diff --git a/kernel/async.c b/kernel/async.c index 27235f5de198..15319d6c18fe 100644 --- a/kernel/async.c +++ b/kernel/async.c | |||
@@ -56,6 +56,7 @@ asynchronous and synchronous parts of the kernel. | |||
56 | #include <linux/init.h> | 56 | #include <linux/init.h> |
57 | #include <linux/kthread.h> | 57 | #include <linux/kthread.h> |
58 | #include <linux/delay.h> | 58 | #include <linux/delay.h> |
59 | #include <linux/slab.h> | ||
59 | #include <asm/atomic.h> | 60 | #include <asm/atomic.h> |
60 | 61 | ||
61 | static async_cookie_t next_cookie = 1; | 62 | static async_cookie_t next_cookie = 1; |
diff --git a/kernel/audit.c b/kernel/audit.c index 78f7f86aa238..c71bd26631a2 100644 --- a/kernel/audit.c +++ b/kernel/audit.c | |||
@@ -46,6 +46,7 @@ | |||
46 | #include <asm/atomic.h> | 46 | #include <asm/atomic.h> |
47 | #include <linux/mm.h> | 47 | #include <linux/mm.h> |
48 | #include <linux/module.h> | 48 | #include <linux/module.h> |
49 | #include <linux/slab.h> | ||
49 | #include <linux/err.h> | 50 | #include <linux/err.h> |
50 | #include <linux/kthread.h> | 51 | #include <linux/kthread.h> |
51 | 52 | ||
diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c index 028e85663f27..46a57b57a335 100644 --- a/kernel/audit_tree.c +++ b/kernel/audit_tree.c | |||
@@ -3,6 +3,7 @@ | |||
3 | #include <linux/namei.h> | 3 | #include <linux/namei.h> |
4 | #include <linux/mount.h> | 4 | #include <linux/mount.h> |
5 | #include <linux/kthread.h> | 5 | #include <linux/kthread.h> |
6 | #include <linux/slab.h> | ||
6 | 7 | ||
7 | struct audit_tree; | 8 | struct audit_tree; |
8 | struct audit_chunk; | 9 | struct audit_chunk; |
diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c index cc7e87936cbc..8df43696f4ba 100644 --- a/kernel/audit_watch.c +++ b/kernel/audit_watch.c | |||
@@ -27,6 +27,7 @@ | |||
27 | #include <linux/namei.h> | 27 | #include <linux/namei.h> |
28 | #include <linux/netlink.h> | 28 | #include <linux/netlink.h> |
29 | #include <linux/sched.h> | 29 | #include <linux/sched.h> |
30 | #include <linux/slab.h> | ||
30 | #include <linux/inotify.h> | 31 | #include <linux/inotify.h> |
31 | #include <linux/security.h> | 32 | #include <linux/security.h> |
32 | #include "audit.h" | 33 | #include "audit.h" |
diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c index a70604047f3c..ce08041f578d 100644 --- a/kernel/auditfilter.c +++ b/kernel/auditfilter.c | |||
@@ -27,6 +27,7 @@ | |||
27 | #include <linux/namei.h> | 27 | #include <linux/namei.h> |
28 | #include <linux/netlink.h> | 28 | #include <linux/netlink.h> |
29 | #include <linux/sched.h> | 29 | #include <linux/sched.h> |
30 | #include <linux/slab.h> | ||
30 | #include <linux/security.h> | 31 | #include <linux/security.h> |
31 | #include "audit.h" | 32 | #include "audit.h" |
32 | 33 | ||
diff --git a/kernel/auditsc.c b/kernel/auditsc.c index f3a461c0970a..3828ad5fb8f1 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c | |||
@@ -49,6 +49,7 @@ | |||
49 | #include <linux/namei.h> | 49 | #include <linux/namei.h> |
50 | #include <linux/mm.h> | 50 | #include <linux/mm.h> |
51 | #include <linux/module.h> | 51 | #include <linux/module.h> |
52 | #include <linux/slab.h> | ||
52 | #include <linux/mount.h> | 53 | #include <linux/mount.h> |
53 | #include <linux/socket.h> | 54 | #include <linux/socket.h> |
54 | #include <linux/mqueue.h> | 55 | #include <linux/mqueue.h> |
@@ -1893,7 +1894,7 @@ static int audit_inc_name_count(struct audit_context *context, | |||
1893 | { | 1894 | { |
1894 | if (context->name_count >= AUDIT_NAMES) { | 1895 | if (context->name_count >= AUDIT_NAMES) { |
1895 | if (inode) | 1896 | if (inode) |
1896 | printk(KERN_DEBUG "name_count maxed, losing inode data: " | 1897 | printk(KERN_DEBUG "audit: name_count maxed, losing inode data: " |
1897 | "dev=%02x:%02x, inode=%lu\n", | 1898 | "dev=%02x:%02x, inode=%lu\n", |
1898 | MAJOR(inode->i_sb->s_dev), | 1899 | MAJOR(inode->i_sb->s_dev), |
1899 | MINOR(inode->i_sb->s_dev), | 1900 | MINOR(inode->i_sb->s_dev), |
diff --git a/kernel/cgroup_freezer.c b/kernel/cgroup_freezer.c index eb3f34d57419..da5e13975531 100644 --- a/kernel/cgroup_freezer.c +++ b/kernel/cgroup_freezer.c | |||
@@ -15,6 +15,7 @@ | |||
15 | */ | 15 | */ |
16 | 16 | ||
17 | #include <linux/module.h> | 17 | #include <linux/module.h> |
18 | #include <linux/slab.h> | ||
18 | #include <linux/cgroup.h> | 19 | #include <linux/cgroup.h> |
19 | #include <linux/fs.h> | 20 | #include <linux/fs.h> |
20 | #include <linux/uaccess.h> | 21 | #include <linux/uaccess.h> |
diff --git a/kernel/compat.c b/kernel/compat.c index f6c204f07ea6..7f40e9275fd9 100644 --- a/kernel/compat.c +++ b/kernel/compat.c | |||
@@ -25,6 +25,7 @@ | |||
25 | #include <linux/posix-timers.h> | 25 | #include <linux/posix-timers.h> |
26 | #include <linux/times.h> | 26 | #include <linux/times.h> |
27 | #include <linux/ptrace.h> | 27 | #include <linux/ptrace.h> |
28 | #include <linux/gfp.h> | ||
28 | 29 | ||
29 | #include <asm/uaccess.h> | 30 | #include <asm/uaccess.h> |
30 | 31 | ||
diff --git a/kernel/cpu.c b/kernel/cpu.c index f8cced2692b3..25bba73b1be3 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c | |||
@@ -14,6 +14,7 @@ | |||
14 | #include <linux/kthread.h> | 14 | #include <linux/kthread.h> |
15 | #include <linux/stop_machine.h> | 15 | #include <linux/stop_machine.h> |
16 | #include <linux/mutex.h> | 16 | #include <linux/mutex.h> |
17 | #include <linux/gfp.h> | ||
17 | 18 | ||
18 | #ifdef CONFIG_SMP | 19 | #ifdef CONFIG_SMP |
19 | /* Serializes the updates to cpu_online_mask, cpu_present_mask */ | 20 | /* Serializes the updates to cpu_online_mask, cpu_present_mask */ |
diff --git a/kernel/cred.c b/kernel/cred.c index 1b1129d0cce8..e1dbe9eef800 100644 --- a/kernel/cred.c +++ b/kernel/cred.c | |||
@@ -10,6 +10,7 @@ | |||
10 | */ | 10 | */ |
11 | #include <linux/module.h> | 11 | #include <linux/module.h> |
12 | #include <linux/cred.h> | 12 | #include <linux/cred.h> |
13 | #include <linux/slab.h> | ||
13 | #include <linux/sched.h> | 14 | #include <linux/sched.h> |
14 | #include <linux/key.h> | 15 | #include <linux/key.h> |
15 | #include <linux/keyctl.h> | 16 | #include <linux/keyctl.h> |
diff --git a/kernel/exit.c b/kernel/exit.c index cce59cb5ee6a..7f2683a10ac4 100644 --- a/kernel/exit.c +++ b/kernel/exit.c | |||
@@ -953,7 +953,8 @@ NORET_TYPE void do_exit(long code) | |||
953 | 953 | ||
954 | acct_update_integrals(tsk); | 954 | acct_update_integrals(tsk); |
955 | /* sync mm's RSS info before statistics gathering */ | 955 | /* sync mm's RSS info before statistics gathering */ |
956 | sync_mm_rss(tsk, tsk->mm); | 956 | if (tsk->mm) |
957 | sync_mm_rss(tsk, tsk->mm); | ||
957 | group_dead = atomic_dec_and_test(&tsk->signal->live); | 958 | group_dead = atomic_dec_and_test(&tsk->signal->live); |
958 | if (group_dead) { | 959 | if (group_dead) { |
959 | hrtimer_cancel(&tsk->signal->real_timer); | 960 | hrtimer_cancel(&tsk->signal->real_timer); |
diff --git a/kernel/fork.c b/kernel/fork.c index 4799c5f0e6d0..44b0791b0a2e 100644 --- a/kernel/fork.c +++ b/kernel/fork.c | |||
@@ -1052,6 +1052,9 @@ static struct task_struct *copy_process(unsigned long clone_flags, | |||
1052 | p->prev_utime = cputime_zero; | 1052 | p->prev_utime = cputime_zero; |
1053 | p->prev_stime = cputime_zero; | 1053 | p->prev_stime = cputime_zero; |
1054 | #endif | 1054 | #endif |
1055 | #if defined(SPLIT_RSS_COUNTING) | ||
1056 | memset(&p->rss_stat, 0, sizeof(p->rss_stat)); | ||
1057 | #endif | ||
1055 | 1058 | ||
1056 | p->default_timer_slack_ns = current->timer_slack_ns; | 1059 | p->default_timer_slack_ns = current->timer_slack_ns; |
1057 | 1060 | ||
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 398fda155f6e..704e488730a5 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c | |||
@@ -757,6 +757,16 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new) | |||
757 | if (new->flags & IRQF_ONESHOT) | 757 | if (new->flags & IRQF_ONESHOT) |
758 | desc->status |= IRQ_ONESHOT; | 758 | desc->status |= IRQ_ONESHOT; |
759 | 759 | ||
760 | /* | ||
761 | * Force MSI interrupts to run with interrupts | ||
762 | * disabled. The multi vector cards can cause stack | ||
763 | * overflows due to nested interrupts when enough of | ||
764 | * them are directed to a core and fire at the same | ||
765 | * time. | ||
766 | */ | ||
767 | if (desc->msi_desc) | ||
768 | new->flags |= IRQF_DISABLED; | ||
769 | |||
760 | if (!(desc->status & IRQ_NOAUTOEN)) { | 770 | if (!(desc->status & IRQ_NOAUTOEN)) { |
761 | desc->depth = 0; | 771 | desc->depth = 0; |
762 | desc->status &= ~IRQ_DISABLED; | 772 | desc->status &= ~IRQ_DISABLED; |
diff --git a/kernel/irq/numa_migrate.c b/kernel/irq/numa_migrate.c index 963559dbd858..65d3845665ac 100644 --- a/kernel/irq/numa_migrate.c +++ b/kernel/irq/numa_migrate.c | |||
@@ -6,6 +6,7 @@ | |||
6 | */ | 6 | */ |
7 | 7 | ||
8 | #include <linux/irq.h> | 8 | #include <linux/irq.h> |
9 | #include <linux/slab.h> | ||
9 | #include <linux/module.h> | 10 | #include <linux/module.h> |
10 | #include <linux/random.h> | 11 | #include <linux/random.h> |
11 | #include <linux/interrupt.h> | 12 | #include <linux/interrupt.h> |
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 6f50eccc79c0..7a6eb04ef6b5 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c | |||
@@ -7,6 +7,7 @@ | |||
7 | */ | 7 | */ |
8 | 8 | ||
9 | #include <linux/irq.h> | 9 | #include <linux/irq.h> |
10 | #include <linux/gfp.h> | ||
10 | #include <linux/proc_fs.h> | 11 | #include <linux/proc_fs.h> |
11 | #include <linux/seq_file.h> | 12 | #include <linux/seq_file.h> |
12 | #include <linux/interrupt.h> | 13 | #include <linux/interrupt.h> |
diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 8e5288a8a355..13aff293f4de 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c | |||
@@ -21,6 +21,7 @@ | |||
21 | #include <linux/sched.h> /* for cond_resched */ | 21 | #include <linux/sched.h> /* for cond_resched */ |
22 | #include <linux/mm.h> | 22 | #include <linux/mm.h> |
23 | #include <linux/ctype.h> | 23 | #include <linux/ctype.h> |
24 | #include <linux/slab.h> | ||
24 | 25 | ||
25 | #include <asm/sections.h> | 26 | #include <asm/sections.h> |
26 | 27 | ||
diff --git a/kernel/latencytop.c b/kernel/latencytop.c index ca07c5c0c914..877fb306d415 100644 --- a/kernel/latencytop.c +++ b/kernel/latencytop.c | |||
@@ -56,7 +56,6 @@ | |||
56 | #include <linux/module.h> | 56 | #include <linux/module.h> |
57 | #include <linux/sched.h> | 57 | #include <linux/sched.h> |
58 | #include <linux/list.h> | 58 | #include <linux/list.h> |
59 | #include <linux/slab.h> | ||
60 | #include <linux/stacktrace.h> | 59 | #include <linux/stacktrace.h> |
61 | 60 | ||
62 | static DEFINE_SPINLOCK(latency_lock); | 61 | static DEFINE_SPINLOCK(latency_lock); |
diff --git a/kernel/lockdep.c b/kernel/lockdep.c index c927a549db2c..2594e1ce41cb 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c | |||
@@ -43,6 +43,7 @@ | |||
43 | #include <linux/ftrace.h> | 43 | #include <linux/ftrace.h> |
44 | #include <linux/stringify.h> | 44 | #include <linux/stringify.h> |
45 | #include <linux/bitops.h> | 45 | #include <linux/bitops.h> |
46 | #include <linux/gfp.h> | ||
46 | 47 | ||
47 | #include <asm/sections.h> | 48 | #include <asm/sections.h> |
48 | 49 | ||
@@ -582,9 +583,6 @@ static int static_obj(void *obj) | |||
582 | unsigned long start = (unsigned long) &_stext, | 583 | unsigned long start = (unsigned long) &_stext, |
583 | end = (unsigned long) &_end, | 584 | end = (unsigned long) &_end, |
584 | addr = (unsigned long) obj; | 585 | addr = (unsigned long) obj; |
585 | #ifdef CONFIG_SMP | ||
586 | int i; | ||
587 | #endif | ||
588 | 586 | ||
589 | /* | 587 | /* |
590 | * static variable? | 588 | * static variable? |
@@ -595,24 +593,16 @@ static int static_obj(void *obj) | |||
595 | if (arch_is_kernel_data(addr)) | 593 | if (arch_is_kernel_data(addr)) |
596 | return 1; | 594 | return 1; |
597 | 595 | ||
598 | #ifdef CONFIG_SMP | ||
599 | /* | 596 | /* |
600 | * percpu var? | 597 | * in-kernel percpu var? |
601 | */ | 598 | */ |
602 | for_each_possible_cpu(i) { | 599 | if (is_kernel_percpu_address(addr)) |
603 | start = (unsigned long) &__per_cpu_start + per_cpu_offset(i); | 600 | return 1; |
604 | end = (unsigned long) &__per_cpu_start + PERCPU_ENOUGH_ROOM | ||
605 | + per_cpu_offset(i); | ||
606 | |||
607 | if ((addr >= start) && (addr < end)) | ||
608 | return 1; | ||
609 | } | ||
610 | #endif | ||
611 | 601 | ||
612 | /* | 602 | /* |
613 | * module var? | 603 | * module static or percpu var? |
614 | */ | 604 | */ |
615 | return is_module_address(addr); | 605 | return is_module_address(addr) || is_module_percpu_address(addr); |
616 | } | 606 | } |
617 | 607 | ||
618 | /* | 608 | /* |
diff --git a/kernel/module.c b/kernel/module.c index c968d3606dca..1016b75b026a 100644 --- a/kernel/module.c +++ b/kernel/module.c | |||
@@ -370,27 +370,33 @@ EXPORT_SYMBOL_GPL(find_module); | |||
370 | 370 | ||
371 | #ifdef CONFIG_SMP | 371 | #ifdef CONFIG_SMP |
372 | 372 | ||
373 | static void *percpu_modalloc(unsigned long size, unsigned long align, | 373 | static inline void __percpu *mod_percpu(struct module *mod) |
374 | const char *name) | ||
375 | { | 374 | { |
376 | void *ptr; | 375 | return mod->percpu; |
376 | } | ||
377 | 377 | ||
378 | static int percpu_modalloc(struct module *mod, | ||
379 | unsigned long size, unsigned long align) | ||
380 | { | ||
378 | if (align > PAGE_SIZE) { | 381 | if (align > PAGE_SIZE) { |
379 | printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n", | 382 | printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n", |
380 | name, align, PAGE_SIZE); | 383 | mod->name, align, PAGE_SIZE); |
381 | align = PAGE_SIZE; | 384 | align = PAGE_SIZE; |
382 | } | 385 | } |
383 | 386 | ||
384 | ptr = __alloc_reserved_percpu(size, align); | 387 | mod->percpu = __alloc_reserved_percpu(size, align); |
385 | if (!ptr) | 388 | if (!mod->percpu) { |
386 | printk(KERN_WARNING | 389 | printk(KERN_WARNING |
387 | "Could not allocate %lu bytes percpu data\n", size); | 390 | "Could not allocate %lu bytes percpu data\n", size); |
388 | return ptr; | 391 | return -ENOMEM; |
392 | } | ||
393 | mod->percpu_size = size; | ||
394 | return 0; | ||
389 | } | 395 | } |
390 | 396 | ||
391 | static void percpu_modfree(void *freeme) | 397 | static void percpu_modfree(struct module *mod) |
392 | { | 398 | { |
393 | free_percpu(freeme); | 399 | free_percpu(mod->percpu); |
394 | } | 400 | } |
395 | 401 | ||
396 | static unsigned int find_pcpusec(Elf_Ehdr *hdr, | 402 | static unsigned int find_pcpusec(Elf_Ehdr *hdr, |
@@ -400,24 +406,62 @@ static unsigned int find_pcpusec(Elf_Ehdr *hdr, | |||
400 | return find_sec(hdr, sechdrs, secstrings, ".data.percpu"); | 406 | return find_sec(hdr, sechdrs, secstrings, ".data.percpu"); |
401 | } | 407 | } |
402 | 408 | ||
403 | static void percpu_modcopy(void *pcpudest, const void *from, unsigned long size) | 409 | static void percpu_modcopy(struct module *mod, |
410 | const void *from, unsigned long size) | ||
404 | { | 411 | { |
405 | int cpu; | 412 | int cpu; |
406 | 413 | ||
407 | for_each_possible_cpu(cpu) | 414 | for_each_possible_cpu(cpu) |
408 | memcpy(pcpudest + per_cpu_offset(cpu), from, size); | 415 | memcpy(per_cpu_ptr(mod->percpu, cpu), from, size); |
416 | } | ||
417 | |||
418 | /** | ||
419 | * is_module_percpu_address - test whether address is from module static percpu | ||
420 | * @addr: address to test | ||
421 | * | ||
422 | * Test whether @addr belongs to module static percpu area. | ||
423 | * | ||
424 | * RETURNS: | ||
425 | * %true if @addr is from module static percpu area | ||
426 | */ | ||
427 | bool is_module_percpu_address(unsigned long addr) | ||
428 | { | ||
429 | struct module *mod; | ||
430 | unsigned int cpu; | ||
431 | |||
432 | preempt_disable(); | ||
433 | |||
434 | list_for_each_entry_rcu(mod, &modules, list) { | ||
435 | if (!mod->percpu_size) | ||
436 | continue; | ||
437 | for_each_possible_cpu(cpu) { | ||
438 | void *start = per_cpu_ptr(mod->percpu, cpu); | ||
439 | |||
440 | if ((void *)addr >= start && | ||
441 | (void *)addr < start + mod->percpu_size) { | ||
442 | preempt_enable(); | ||
443 | return true; | ||
444 | } | ||
445 | } | ||
446 | } | ||
447 | |||
448 | preempt_enable(); | ||
449 | return false; | ||
409 | } | 450 | } |
410 | 451 | ||
411 | #else /* ... !CONFIG_SMP */ | 452 | #else /* ... !CONFIG_SMP */ |
412 | 453 | ||
413 | static inline void *percpu_modalloc(unsigned long size, unsigned long align, | 454 | static inline void __percpu *mod_percpu(struct module *mod) |
414 | const char *name) | ||
415 | { | 455 | { |
416 | return NULL; | 456 | return NULL; |
417 | } | 457 | } |
418 | static inline void percpu_modfree(void *pcpuptr) | 458 | static inline int percpu_modalloc(struct module *mod, |
459 | unsigned long size, unsigned long align) | ||
460 | { | ||
461 | return -ENOMEM; | ||
462 | } | ||
463 | static inline void percpu_modfree(struct module *mod) | ||
419 | { | 464 | { |
420 | BUG(); | ||
421 | } | 465 | } |
422 | static inline unsigned int find_pcpusec(Elf_Ehdr *hdr, | 466 | static inline unsigned int find_pcpusec(Elf_Ehdr *hdr, |
423 | Elf_Shdr *sechdrs, | 467 | Elf_Shdr *sechdrs, |
@@ -425,12 +469,16 @@ static inline unsigned int find_pcpusec(Elf_Ehdr *hdr, | |||
425 | { | 469 | { |
426 | return 0; | 470 | return 0; |
427 | } | 471 | } |
428 | static inline void percpu_modcopy(void *pcpudst, const void *src, | 472 | static inline void percpu_modcopy(struct module *mod, |
429 | unsigned long size) | 473 | const void *from, unsigned long size) |
430 | { | 474 | { |
431 | /* pcpusec should be 0, and size of that section should be 0. */ | 475 | /* pcpusec should be 0, and size of that section should be 0. */ |
432 | BUG_ON(size != 0); | 476 | BUG_ON(size != 0); |
433 | } | 477 | } |
478 | bool is_module_percpu_address(unsigned long addr) | ||
479 | { | ||
480 | return false; | ||
481 | } | ||
434 | 482 | ||
435 | #endif /* CONFIG_SMP */ | 483 | #endif /* CONFIG_SMP */ |
436 | 484 | ||
@@ -473,11 +521,13 @@ static void module_unload_init(struct module *mod) | |||
473 | int cpu; | 521 | int cpu; |
474 | 522 | ||
475 | INIT_LIST_HEAD(&mod->modules_which_use_me); | 523 | INIT_LIST_HEAD(&mod->modules_which_use_me); |
476 | for_each_possible_cpu(cpu) | 524 | for_each_possible_cpu(cpu) { |
477 | per_cpu_ptr(mod->refptr, cpu)->count = 0; | 525 | per_cpu_ptr(mod->refptr, cpu)->incs = 0; |
526 | per_cpu_ptr(mod->refptr, cpu)->decs = 0; | ||
527 | } | ||
478 | 528 | ||
479 | /* Hold reference count during initialization. */ | 529 | /* Hold reference count during initialization. */ |
480 | __this_cpu_write(mod->refptr->count, 1); | 530 | __this_cpu_write(mod->refptr->incs, 1); |
481 | /* Backwards compatibility macros put refcount during init. */ | 531 | /* Backwards compatibility macros put refcount during init. */ |
482 | mod->waiter = current; | 532 | mod->waiter = current; |
483 | } | 533 | } |
@@ -616,12 +666,28 @@ static int try_stop_module(struct module *mod, int flags, int *forced) | |||
616 | 666 | ||
617 | unsigned int module_refcount(struct module *mod) | 667 | unsigned int module_refcount(struct module *mod) |
618 | { | 668 | { |
619 | unsigned int total = 0; | 669 | unsigned int incs = 0, decs = 0; |
620 | int cpu; | 670 | int cpu; |
621 | 671 | ||
622 | for_each_possible_cpu(cpu) | 672 | for_each_possible_cpu(cpu) |
623 | total += per_cpu_ptr(mod->refptr, cpu)->count; | 673 | decs += per_cpu_ptr(mod->refptr, cpu)->decs; |
624 | return total; | 674 | /* |
675 | * ensure the incs are added up after the decs. | ||
676 | * module_put ensures incs are visible before decs with smp_wmb. | ||
677 | * | ||
678 | * This 2-count scheme avoids the situation where the refcount | ||
679 | * for CPU0 is read, then CPU0 increments the module refcount, | ||
680 | * then CPU1 drops that refcount, then the refcount for CPU1 is | ||
681 | * read. We would record a decrement but not its corresponding | ||
682 | * increment so we would see a low count (disaster). | ||
683 | * | ||
684 | * Rare situation? But module_refcount can be preempted, and we | ||
685 | * might be tallying up 4096+ CPUs. So it is not impossible. | ||
686 | */ | ||
687 | smp_rmb(); | ||
688 | for_each_possible_cpu(cpu) | ||
689 | incs += per_cpu_ptr(mod->refptr, cpu)->incs; | ||
690 | return incs - decs; | ||
625 | } | 691 | } |
626 | EXPORT_SYMBOL(module_refcount); | 692 | EXPORT_SYMBOL(module_refcount); |
627 | 693 | ||
@@ -798,10 +864,11 @@ void module_put(struct module *module) | |||
798 | { | 864 | { |
799 | if (module) { | 865 | if (module) { |
800 | preempt_disable(); | 866 | preempt_disable(); |
801 | __this_cpu_dec(module->refptr->count); | 867 | smp_wmb(); /* see comment in module_refcount */ |
868 | __this_cpu_inc(module->refptr->decs); | ||
802 | 869 | ||
803 | trace_module_put(module, _RET_IP_, | 870 | trace_module_put(module, _RET_IP_, |
804 | __this_cpu_read(module->refptr->count)); | 871 | __this_cpu_read(module->refptr->decs)); |
805 | /* Maybe they're waiting for us to drop reference? */ | 872 | /* Maybe they're waiting for us to drop reference? */ |
806 | if (unlikely(!module_is_live(module))) | 873 | if (unlikely(!module_is_live(module))) |
807 | wake_up_process(module->waiter); | 874 | wake_up_process(module->waiter); |
@@ -1400,8 +1467,7 @@ static void free_module(struct module *mod) | |||
1400 | /* This may be NULL, but that's OK */ | 1467 | /* This may be NULL, but that's OK */ |
1401 | module_free(mod, mod->module_init); | 1468 | module_free(mod, mod->module_init); |
1402 | kfree(mod->args); | 1469 | kfree(mod->args); |
1403 | if (mod->percpu) | 1470 | percpu_modfree(mod); |
1404 | percpu_modfree(mod->percpu); | ||
1405 | #if defined(CONFIG_MODULE_UNLOAD) | 1471 | #if defined(CONFIG_MODULE_UNLOAD) |
1406 | if (mod->refptr) | 1472 | if (mod->refptr) |
1407 | free_percpu(mod->refptr); | 1473 | free_percpu(mod->refptr); |
@@ -1520,7 +1586,7 @@ static int simplify_symbols(Elf_Shdr *sechdrs, | |||
1520 | default: | 1586 | default: |
1521 | /* Divert to percpu allocation if a percpu var. */ | 1587 | /* Divert to percpu allocation if a percpu var. */ |
1522 | if (sym[i].st_shndx == pcpuindex) | 1588 | if (sym[i].st_shndx == pcpuindex) |
1523 | secbase = (unsigned long)mod->percpu; | 1589 | secbase = (unsigned long)mod_percpu(mod); |
1524 | else | 1590 | else |
1525 | secbase = sechdrs[sym[i].st_shndx].sh_addr; | 1591 | secbase = sechdrs[sym[i].st_shndx].sh_addr; |
1526 | sym[i].st_value += secbase; | 1592 | sym[i].st_value += secbase; |
@@ -1954,7 +2020,7 @@ static noinline struct module *load_module(void __user *umod, | |||
1954 | unsigned int modindex, versindex, infoindex, pcpuindex; | 2020 | unsigned int modindex, versindex, infoindex, pcpuindex; |
1955 | struct module *mod; | 2021 | struct module *mod; |
1956 | long err = 0; | 2022 | long err = 0; |
1957 | void *percpu = NULL, *ptr = NULL; /* Stops spurious gcc warning */ | 2023 | void *ptr = NULL; /* Stops spurious gcc warning */ |
1958 | unsigned long symoffs, stroffs, *strmap; | 2024 | unsigned long symoffs, stroffs, *strmap; |
1959 | 2025 | ||
1960 | mm_segment_t old_fs; | 2026 | mm_segment_t old_fs; |
@@ -2094,15 +2160,11 @@ static noinline struct module *load_module(void __user *umod, | |||
2094 | 2160 | ||
2095 | if (pcpuindex) { | 2161 | if (pcpuindex) { |
2096 | /* We have a special allocation for this section. */ | 2162 | /* We have a special allocation for this section. */ |
2097 | percpu = percpu_modalloc(sechdrs[pcpuindex].sh_size, | 2163 | err = percpu_modalloc(mod, sechdrs[pcpuindex].sh_size, |
2098 | sechdrs[pcpuindex].sh_addralign, | 2164 | sechdrs[pcpuindex].sh_addralign); |
2099 | mod->name); | 2165 | if (err) |
2100 | if (!percpu) { | ||
2101 | err = -ENOMEM; | ||
2102 | goto free_mod; | 2166 | goto free_mod; |
2103 | } | ||
2104 | sechdrs[pcpuindex].sh_flags &= ~(unsigned long)SHF_ALLOC; | 2167 | sechdrs[pcpuindex].sh_flags &= ~(unsigned long)SHF_ALLOC; |
2105 | mod->percpu = percpu; | ||
2106 | } | 2168 | } |
2107 | 2169 | ||
2108 | /* Determine total sizes, and put offsets in sh_entsize. For now | 2170 | /* Determine total sizes, and put offsets in sh_entsize. For now |
@@ -2317,7 +2379,7 @@ static noinline struct module *load_module(void __user *umod, | |||
2317 | sort_extable(mod->extable, mod->extable + mod->num_exentries); | 2379 | sort_extable(mod->extable, mod->extable + mod->num_exentries); |
2318 | 2380 | ||
2319 | /* Finally, copy percpu area over. */ | 2381 | /* Finally, copy percpu area over. */ |
2320 | percpu_modcopy(mod->percpu, (void *)sechdrs[pcpuindex].sh_addr, | 2382 | percpu_modcopy(mod, (void *)sechdrs[pcpuindex].sh_addr, |
2321 | sechdrs[pcpuindex].sh_size); | 2383 | sechdrs[pcpuindex].sh_size); |
2322 | 2384 | ||
2323 | add_kallsyms(mod, sechdrs, hdr->e_shnum, symindex, strindex, | 2385 | add_kallsyms(mod, sechdrs, hdr->e_shnum, symindex, strindex, |
@@ -2409,8 +2471,7 @@ static noinline struct module *load_module(void __user *umod, | |||
2409 | module_free(mod, mod->module_core); | 2471 | module_free(mod, mod->module_core); |
2410 | /* mod will be freed with core. Don't access it beyond this line! */ | 2472 | /* mod will be freed with core. Don't access it beyond this line! */ |
2411 | free_percpu: | 2473 | free_percpu: |
2412 | if (percpu) | 2474 | percpu_modfree(mod); |
2413 | percpu_modfree(percpu); | ||
2414 | free_mod: | 2475 | free_mod: |
2415 | kfree(args); | 2476 | kfree(args); |
2416 | kfree(strmap); | 2477 | kfree(strmap); |
diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c index 2ab67233ee8f..f74e6c00e26d 100644 --- a/kernel/nsproxy.c +++ b/kernel/nsproxy.c | |||
@@ -13,6 +13,7 @@ | |||
13 | * Pavel Emelianov <xemul@openvz.org> | 13 | * Pavel Emelianov <xemul@openvz.org> |
14 | */ | 14 | */ |
15 | 15 | ||
16 | #include <linux/slab.h> | ||
16 | #include <linux/module.h> | 17 | #include <linux/module.h> |
17 | #include <linux/nsproxy.h> | 18 | #include <linux/nsproxy.h> |
18 | #include <linux/init_task.h> | 19 | #include <linux/init_task.h> |
diff --git a/kernel/padata.c b/kernel/padata.c index 93caf65ff57c..fd03513c7327 100644 --- a/kernel/padata.c +++ b/kernel/padata.c | |||
@@ -25,6 +25,7 @@ | |||
25 | #include <linux/padata.h> | 25 | #include <linux/padata.h> |
26 | #include <linux/mutex.h> | 26 | #include <linux/mutex.h> |
27 | #include <linux/sched.h> | 27 | #include <linux/sched.h> |
28 | #include <linux/slab.h> | ||
28 | #include <linux/rcupdate.h> | 29 | #include <linux/rcupdate.h> |
29 | 30 | ||
30 | #define MAX_SEQ_NR INT_MAX - NR_CPUS | 31 | #define MAX_SEQ_NR INT_MAX - NR_CPUS |
diff --git a/kernel/perf_event.c b/kernel/perf_event.c index 574ee58a3046..2f3fbf84215a 100644 --- a/kernel/perf_event.c +++ b/kernel/perf_event.c | |||
@@ -15,6 +15,7 @@ | |||
15 | #include <linux/smp.h> | 15 | #include <linux/smp.h> |
16 | #include <linux/file.h> | 16 | #include <linux/file.h> |
17 | #include <linux/poll.h> | 17 | #include <linux/poll.h> |
18 | #include <linux/slab.h> | ||
18 | #include <linux/sysfs.h> | 19 | #include <linux/sysfs.h> |
19 | #include <linux/dcache.h> | 20 | #include <linux/dcache.h> |
20 | #include <linux/percpu.h> | 21 | #include <linux/percpu.h> |
@@ -1164,11 +1165,9 @@ void perf_event_task_sched_out(struct task_struct *task, | |||
1164 | struct perf_event_context *ctx = task->perf_event_ctxp; | 1165 | struct perf_event_context *ctx = task->perf_event_ctxp; |
1165 | struct perf_event_context *next_ctx; | 1166 | struct perf_event_context *next_ctx; |
1166 | struct perf_event_context *parent; | 1167 | struct perf_event_context *parent; |
1167 | struct pt_regs *regs; | ||
1168 | int do_switch = 1; | 1168 | int do_switch = 1; |
1169 | 1169 | ||
1170 | regs = task_pt_regs(task); | 1170 | perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0); |
1171 | perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, regs, 0); | ||
1172 | 1171 | ||
1173 | if (likely(!ctx || !cpuctx->task_ctx)) | 1172 | if (likely(!ctx || !cpuctx->task_ctx)) |
1174 | return; | 1173 | return; |
@@ -2786,12 +2785,11 @@ __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs) | |||
2786 | return NULL; | 2785 | return NULL; |
2787 | } | 2786 | } |
2788 | 2787 | ||
2789 | #ifdef CONFIG_EVENT_TRACING | ||
2790 | __weak | 2788 | __weak |
2791 | void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int skip) | 2789 | void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int skip) |
2792 | { | 2790 | { |
2793 | } | 2791 | } |
2794 | #endif | 2792 | |
2795 | 2793 | ||
2796 | /* | 2794 | /* |
2797 | * Output | 2795 | * Output |
@@ -3378,15 +3376,23 @@ static void perf_event_task_output(struct perf_event *event, | |||
3378 | struct perf_task_event *task_event) | 3376 | struct perf_task_event *task_event) |
3379 | { | 3377 | { |
3380 | struct perf_output_handle handle; | 3378 | struct perf_output_handle handle; |
3381 | int size; | ||
3382 | struct task_struct *task = task_event->task; | 3379 | struct task_struct *task = task_event->task; |
3383 | int ret; | 3380 | unsigned long flags; |
3381 | int size, ret; | ||
3382 | |||
3383 | /* | ||
3384 | * If this CPU attempts to acquire an rq lock held by a CPU spinning | ||
3385 | * in perf_output_lock() from interrupt context, it's game over. | ||
3386 | */ | ||
3387 | local_irq_save(flags); | ||
3384 | 3388 | ||
3385 | size = task_event->event_id.header.size; | 3389 | size = task_event->event_id.header.size; |
3386 | ret = perf_output_begin(&handle, event, size, 0, 0); | 3390 | ret = perf_output_begin(&handle, event, size, 0, 0); |
3387 | 3391 | ||
3388 | if (ret) | 3392 | if (ret) { |
3393 | local_irq_restore(flags); | ||
3389 | return; | 3394 | return; |
3395 | } | ||
3390 | 3396 | ||
3391 | task_event->event_id.pid = perf_event_pid(event, task); | 3397 | task_event->event_id.pid = perf_event_pid(event, task); |
3392 | task_event->event_id.ppid = perf_event_pid(event, current); | 3398 | task_event->event_id.ppid = perf_event_pid(event, current); |
@@ -3397,6 +3403,7 @@ static void perf_event_task_output(struct perf_event *event, | |||
3397 | perf_output_put(&handle, task_event->event_id); | 3403 | perf_output_put(&handle, task_event->event_id); |
3398 | 3404 | ||
3399 | perf_output_end(&handle); | 3405 | perf_output_end(&handle); |
3406 | local_irq_restore(flags); | ||
3400 | } | 3407 | } |
3401 | 3408 | ||
3402 | static int perf_event_task_match(struct perf_event *event) | 3409 | static int perf_event_task_match(struct perf_event *event) |
diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c index 79aac93acf99..a5aff94e1f0b 100644 --- a/kernel/pid_namespace.c +++ b/kernel/pid_namespace.c | |||
@@ -13,6 +13,7 @@ | |||
13 | #include <linux/syscalls.h> | 13 | #include <linux/syscalls.h> |
14 | #include <linux/err.h> | 14 | #include <linux/err.h> |
15 | #include <linux/acct.h> | 15 | #include <linux/acct.h> |
16 | #include <linux/slab.h> | ||
16 | 17 | ||
17 | #define BITS_PER_PAGE (PAGE_SIZE*8) | 18 | #define BITS_PER_PAGE (PAGE_SIZE*8) |
18 | 19 | ||
diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index da5288ec2392..aa9e916da4d5 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c | |||
@@ -22,6 +22,7 @@ | |||
22 | #include <linux/console.h> | 22 | #include <linux/console.h> |
23 | #include <linux/cpu.h> | 23 | #include <linux/cpu.h> |
24 | #include <linux/freezer.h> | 24 | #include <linux/freezer.h> |
25 | #include <linux/gfp.h> | ||
25 | #include <scsi/scsi_scan.h> | 26 | #include <scsi/scsi_scan.h> |
26 | #include <asm/suspend.h> | 27 | #include <asm/suspend.h> |
27 | 28 | ||
diff --git a/kernel/power/hibernate_nvs.c b/kernel/power/hibernate_nvs.c index 39ac698ef836..fdcad9ed5a7b 100644 --- a/kernel/power/hibernate_nvs.c +++ b/kernel/power/hibernate_nvs.c | |||
@@ -10,6 +10,7 @@ | |||
10 | #include <linux/kernel.h> | 10 | #include <linux/kernel.h> |
11 | #include <linux/list.h> | 11 | #include <linux/list.h> |
12 | #include <linux/mm.h> | 12 | #include <linux/mm.h> |
13 | #include <linux/slab.h> | ||
13 | #include <linux/suspend.h> | 14 | #include <linux/suspend.h> |
14 | 15 | ||
15 | /* | 16 | /* |
diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index 830cadecbdfc..be861c26dda7 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c | |||
@@ -26,6 +26,7 @@ | |||
26 | #include <linux/console.h> | 26 | #include <linux/console.h> |
27 | #include <linux/highmem.h> | 27 | #include <linux/highmem.h> |
28 | #include <linux/list.h> | 28 | #include <linux/list.h> |
29 | #include <linux/slab.h> | ||
29 | 30 | ||
30 | #include <asm/uaccess.h> | 31 | #include <asm/uaccess.h> |
31 | #include <asm/mmu_context.h> | 32 | #include <asm/mmu_context.h> |
diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 44cce10b582d..56e7dbb8b996 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c | |||
@@ -15,6 +15,7 @@ | |||
15 | #include <linux/console.h> | 15 | #include <linux/console.h> |
16 | #include <linux/cpu.h> | 16 | #include <linux/cpu.h> |
17 | #include <linux/syscalls.h> | 17 | #include <linux/syscalls.h> |
18 | #include <linux/gfp.h> | ||
18 | 19 | ||
19 | #include "power.h" | 20 | #include "power.h" |
20 | 21 | ||
diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 1d575733d4e1..66824d71983a 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c | |||
@@ -23,6 +23,7 @@ | |||
23 | #include <linux/swap.h> | 23 | #include <linux/swap.h> |
24 | #include <linux/swapops.h> | 24 | #include <linux/swapops.h> |
25 | #include <linux/pm.h> | 25 | #include <linux/pm.h> |
26 | #include <linux/slab.h> | ||
26 | 27 | ||
27 | #include "power.h" | 28 | #include "power.h" |
28 | 29 | ||
diff --git a/kernel/res_counter.c b/kernel/res_counter.c index bcdabf37c40b..c7eaa37a768b 100644 --- a/kernel/res_counter.c +++ b/kernel/res_counter.c | |||
@@ -10,7 +10,6 @@ | |||
10 | #include <linux/types.h> | 10 | #include <linux/types.h> |
11 | #include <linux/parser.h> | 11 | #include <linux/parser.h> |
12 | #include <linux/fs.h> | 12 | #include <linux/fs.h> |
13 | #include <linux/slab.h> | ||
14 | #include <linux/res_counter.h> | 13 | #include <linux/res_counter.h> |
15 | #include <linux/uaccess.h> | 14 | #include <linux/uaccess.h> |
16 | #include <linux/mm.h> | 15 | #include <linux/mm.h> |
diff --git a/kernel/sched.c b/kernel/sched.c index 49d2fa7b687a..6af210a7de70 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
@@ -71,6 +71,7 @@ | |||
71 | #include <linux/debugfs.h> | 71 | #include <linux/debugfs.h> |
72 | #include <linux/ctype.h> | 72 | #include <linux/ctype.h> |
73 | #include <linux/ftrace.h> | 73 | #include <linux/ftrace.h> |
74 | #include <linux/slab.h> | ||
74 | 75 | ||
75 | #include <asm/tlb.h> | 76 | #include <asm/tlb.h> |
76 | #include <asm/irq_regs.h> | 77 | #include <asm/irq_regs.h> |
@@ -4902,7 +4903,7 @@ SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, | |||
4902 | int ret; | 4903 | int ret; |
4903 | cpumask_var_t mask; | 4904 | cpumask_var_t mask; |
4904 | 4905 | ||
4905 | if (len < nr_cpu_ids) | 4906 | if ((len * BITS_PER_BYTE) < nr_cpu_ids) |
4906 | return -EINVAL; | 4907 | return -EINVAL; |
4907 | if (len & (sizeof(unsigned long)-1)) | 4908 | if (len & (sizeof(unsigned long)-1)) |
4908 | return -EINVAL; | 4909 | return -EINVAL; |
@@ -5387,7 +5388,7 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) | |||
5387 | 5388 | ||
5388 | get_task_struct(mt); | 5389 | get_task_struct(mt); |
5389 | task_rq_unlock(rq, &flags); | 5390 | task_rq_unlock(rq, &flags); |
5390 | wake_up_process(rq->migration_thread); | 5391 | wake_up_process(mt); |
5391 | put_task_struct(mt); | 5392 | put_task_struct(mt); |
5392 | wait_for_completion(&req.done); | 5393 | wait_for_completion(&req.done); |
5393 | tlb_migrate_finish(p->mm); | 5394 | tlb_migrate_finish(p->mm); |
diff --git a/kernel/sched_cpupri.c b/kernel/sched_cpupri.c index fccf9fbb0d7b..e6871cb3fc83 100644 --- a/kernel/sched_cpupri.c +++ b/kernel/sched_cpupri.c | |||
@@ -27,6 +27,7 @@ | |||
27 | * of the License. | 27 | * of the License. |
28 | */ | 28 | */ |
29 | 29 | ||
30 | #include <linux/gfp.h> | ||
30 | #include "sched_cpupri.h" | 31 | #include "sched_cpupri.h" |
31 | 32 | ||
32 | /* Convert between a 140 based task->prio, and our 102 based cpupri */ | 33 | /* Convert between a 140 based task->prio, and our 102 based cpupri */ |
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c index 67f95aada4b9..9b49db144037 100644 --- a/kernel/sched_debug.c +++ b/kernel/sched_debug.c | |||
@@ -518,8 +518,4 @@ void proc_sched_set_task(struct task_struct *p) | |||
518 | p->se.nr_wakeups_idle = 0; | 518 | p->se.nr_wakeups_idle = 0; |
519 | p->sched_info.bkl_count = 0; | 519 | p->sched_info.bkl_count = 0; |
520 | #endif | 520 | #endif |
521 | p->se.sum_exec_runtime = 0; | ||
522 | p->se.prev_sum_exec_runtime = 0; | ||
523 | p->nvcsw = 0; | ||
524 | p->nivcsw = 0; | ||
525 | } | 521 | } |
diff --git a/kernel/smp.c b/kernel/smp.c index 9867b6bfefce..3fc697336183 100644 --- a/kernel/smp.c +++ b/kernel/smp.c | |||
@@ -9,6 +9,7 @@ | |||
9 | #include <linux/module.h> | 9 | #include <linux/module.h> |
10 | #include <linux/percpu.h> | 10 | #include <linux/percpu.h> |
11 | #include <linux/init.h> | 11 | #include <linux/init.h> |
12 | #include <linux/gfp.h> | ||
12 | #include <linux/smp.h> | 13 | #include <linux/smp.h> |
13 | #include <linux/cpu.h> | 14 | #include <linux/cpu.h> |
14 | 15 | ||
diff --git a/kernel/srcu.c b/kernel/srcu.c index bde4295774c8..2980da3fd509 100644 --- a/kernel/srcu.c +++ b/kernel/srcu.c | |||
@@ -30,7 +30,6 @@ | |||
30 | #include <linux/preempt.h> | 30 | #include <linux/preempt.h> |
31 | #include <linux/rcupdate.h> | 31 | #include <linux/rcupdate.h> |
32 | #include <linux/sched.h> | 32 | #include <linux/sched.h> |
33 | #include <linux/slab.h> | ||
34 | #include <linux/smp.h> | 33 | #include <linux/smp.h> |
35 | #include <linux/srcu.h> | 34 | #include <linux/srcu.h> |
36 | 35 | ||
diff --git a/kernel/sys.c b/kernel/sys.c index 8298878f4f71..6d1a7e0f9d5b 100644 --- a/kernel/sys.c +++ b/kernel/sys.c | |||
@@ -36,6 +36,7 @@ | |||
36 | #include <linux/personality.h> | 36 | #include <linux/personality.h> |
37 | #include <linux/ptrace.h> | 37 | #include <linux/ptrace.h> |
38 | #include <linux/fs_struct.h> | 38 | #include <linux/fs_struct.h> |
39 | #include <linux/gfp.h> | ||
39 | 40 | ||
40 | #include <linux/compat.h> | 41 | #include <linux/compat.h> |
41 | #include <linux/syscalls.h> | 42 | #include <linux/syscalls.h> |
diff --git a/kernel/sysctl_binary.c b/kernel/sysctl_binary.c index 8cd50d8f9bde..59030570f5ca 100644 --- a/kernel/sysctl_binary.c +++ b/kernel/sysctl_binary.c | |||
@@ -13,6 +13,7 @@ | |||
13 | #include <linux/file.h> | 13 | #include <linux/file.h> |
14 | #include <linux/ctype.h> | 14 | #include <linux/ctype.h> |
15 | #include <linux/netdevice.h> | 15 | #include <linux/netdevice.h> |
16 | #include <linux/slab.h> | ||
16 | 17 | ||
17 | #ifdef CONFIG_SYSCTL_SYSCALL | 18 | #ifdef CONFIG_SYSCTL_SYSCALL |
18 | 19 | ||
diff --git a/kernel/taskstats.c b/kernel/taskstats.c index 899ca51be5e8..11281d5792bd 100644 --- a/kernel/taskstats.c +++ b/kernel/taskstats.c | |||
@@ -22,6 +22,7 @@ | |||
22 | #include <linux/delayacct.h> | 22 | #include <linux/delayacct.h> |
23 | #include <linux/cpumask.h> | 23 | #include <linux/cpumask.h> |
24 | #include <linux/percpu.h> | 24 | #include <linux/percpu.h> |
25 | #include <linux/slab.h> | ||
25 | #include <linux/cgroupstats.h> | 26 | #include <linux/cgroupstats.h> |
26 | #include <linux/cgroup.h> | 27 | #include <linux/cgroup.h> |
27 | #include <linux/fs.h> | 28 | #include <linux/fs.h> |
diff --git a/kernel/time.c b/kernel/time.c index 804798005d19..656dccfe1cbb 100644 --- a/kernel/time.c +++ b/kernel/time.c | |||
@@ -35,7 +35,6 @@ | |||
35 | #include <linux/syscalls.h> | 35 | #include <linux/syscalls.h> |
36 | #include <linux/security.h> | 36 | #include <linux/security.h> |
37 | #include <linux/fs.h> | 37 | #include <linux/fs.h> |
38 | #include <linux/slab.h> | ||
39 | #include <linux/math64.h> | 38 | #include <linux/math64.h> |
40 | #include <linux/ptrace.h> | 39 | #include <linux/ptrace.h> |
41 | 40 | ||
diff --git a/kernel/time/timecompare.c b/kernel/time/timecompare.c index 12f5c55090be..ac38fbb176cc 100644 --- a/kernel/time/timecompare.c +++ b/kernel/time/timecompare.c | |||
@@ -19,6 +19,7 @@ | |||
19 | 19 | ||
20 | #include <linux/timecompare.h> | 20 | #include <linux/timecompare.h> |
21 | #include <linux/module.h> | 21 | #include <linux/module.h> |
22 | #include <linux/slab.h> | ||
22 | #include <linux/math64.h> | 23 | #include <linux/math64.h> |
23 | 24 | ||
24 | /* | 25 | /* |
diff --git a/kernel/timer.c b/kernel/timer.c index fc965eae0e87..aeb6a54f2771 100644 --- a/kernel/timer.c +++ b/kernel/timer.c | |||
@@ -39,6 +39,7 @@ | |||
39 | #include <linux/kallsyms.h> | 39 | #include <linux/kallsyms.h> |
40 | #include <linux/perf_event.h> | 40 | #include <linux/perf_event.h> |
41 | #include <linux/sched.h> | 41 | #include <linux/sched.h> |
42 | #include <linux/slab.h> | ||
42 | 43 | ||
43 | #include <asm/uaccess.h> | 44 | #include <asm/uaccess.h> |
44 | #include <asm/unistd.h> | 45 | #include <asm/unistd.h> |
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index 07f945a99430..b3bc91a3f510 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c | |||
@@ -21,6 +21,7 @@ | |||
21 | #include <linux/percpu.h> | 21 | #include <linux/percpu.h> |
22 | #include <linux/init.h> | 22 | #include <linux/init.h> |
23 | #include <linux/mutex.h> | 23 | #include <linux/mutex.h> |
24 | #include <linux/slab.h> | ||
24 | #include <linux/debugfs.h> | 25 | #include <linux/debugfs.h> |
25 | #include <linux/smp_lock.h> | 26 | #include <linux/smp_lock.h> |
26 | #include <linux/time.h> | 27 | #include <linux/time.h> |
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index d9062f5cc0c0..2404b59b3097 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
@@ -24,6 +24,7 @@ | |||
24 | #include <linux/uaccess.h> | 24 | #include <linux/uaccess.h> |
25 | #include <linux/ftrace.h> | 25 | #include <linux/ftrace.h> |
26 | #include <linux/sysctl.h> | 26 | #include <linux/sysctl.h> |
27 | #include <linux/slab.h> | ||
27 | #include <linux/ctype.h> | 28 | #include <linux/ctype.h> |
28 | #include <linux/list.h> | 29 | #include <linux/list.h> |
29 | #include <linux/hash.h> | 30 | #include <linux/hash.h> |
diff --git a/kernel/trace/power-traces.c b/kernel/trace/power-traces.c index 9f4f565b01e6..a22582a06161 100644 --- a/kernel/trace/power-traces.c +++ b/kernel/trace/power-traces.c | |||
@@ -9,7 +9,6 @@ | |||
9 | #include <linux/workqueue.h> | 9 | #include <linux/workqueue.h> |
10 | #include <linux/sched.h> | 10 | #include <linux/sched.h> |
11 | #include <linux/module.h> | 11 | #include <linux/module.h> |
12 | #include <linux/slab.h> | ||
13 | 12 | ||
14 | #define CREATE_TRACE_POINTS | 13 | #define CREATE_TRACE_POINTS |
15 | #include <trace/events/power.h> | 14 | #include <trace/events/power.h> |
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index d1187ef20caf..41ca394feb22 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c | |||
@@ -14,6 +14,7 @@ | |||
14 | #include <linux/module.h> | 14 | #include <linux/module.h> |
15 | #include <linux/percpu.h> | 15 | #include <linux/percpu.h> |
16 | #include <linux/mutex.h> | 16 | #include <linux/mutex.h> |
17 | #include <linux/slab.h> | ||
17 | #include <linux/init.h> | 18 | #include <linux/init.h> |
18 | #include <linux/hash.h> | 19 | #include <linux/hash.h> |
19 | #include <linux/list.h> | 20 | #include <linux/list.h> |
@@ -1209,18 +1210,19 @@ rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages) | |||
1209 | 1210 | ||
1210 | for (i = 0; i < nr_pages; i++) { | 1211 | for (i = 0; i < nr_pages; i++) { |
1211 | if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages))) | 1212 | if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages))) |
1212 | return; | 1213 | goto out; |
1213 | p = cpu_buffer->pages->next; | 1214 | p = cpu_buffer->pages->next; |
1214 | bpage = list_entry(p, struct buffer_page, list); | 1215 | bpage = list_entry(p, struct buffer_page, list); |
1215 | list_del_init(&bpage->list); | 1216 | list_del_init(&bpage->list); |
1216 | free_buffer_page(bpage); | 1217 | free_buffer_page(bpage); |
1217 | } | 1218 | } |
1218 | if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages))) | 1219 | if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages))) |
1219 | return; | 1220 | goto out; |
1220 | 1221 | ||
1221 | rb_reset_cpu(cpu_buffer); | 1222 | rb_reset_cpu(cpu_buffer); |
1222 | rb_check_pages(cpu_buffer); | 1223 | rb_check_pages(cpu_buffer); |
1223 | 1224 | ||
1225 | out: | ||
1224 | spin_unlock_irq(&cpu_buffer->reader_lock); | 1226 | spin_unlock_irq(&cpu_buffer->reader_lock); |
1225 | } | 1227 | } |
1226 | 1228 | ||
@@ -1237,7 +1239,7 @@ rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer, | |||
1237 | 1239 | ||
1238 | for (i = 0; i < nr_pages; i++) { | 1240 | for (i = 0; i < nr_pages; i++) { |
1239 | if (RB_WARN_ON(cpu_buffer, list_empty(pages))) | 1241 | if (RB_WARN_ON(cpu_buffer, list_empty(pages))) |
1240 | return; | 1242 | goto out; |
1241 | p = pages->next; | 1243 | p = pages->next; |
1242 | bpage = list_entry(p, struct buffer_page, list); | 1244 | bpage = list_entry(p, struct buffer_page, list); |
1243 | list_del_init(&bpage->list); | 1245 | list_del_init(&bpage->list); |
@@ -1246,6 +1248,7 @@ rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer, | |||
1246 | rb_reset_cpu(cpu_buffer); | 1248 | rb_reset_cpu(cpu_buffer); |
1247 | rb_check_pages(cpu_buffer); | 1249 | rb_check_pages(cpu_buffer); |
1248 | 1250 | ||
1251 | out: | ||
1249 | spin_unlock_irq(&cpu_buffer->reader_lock); | 1252 | spin_unlock_irq(&cpu_buffer->reader_lock); |
1250 | } | 1253 | } |
1251 | 1254 | ||
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 3ec2ee6f6560..44f916a04065 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
@@ -33,10 +33,10 @@ | |||
33 | #include <linux/kdebug.h> | 33 | #include <linux/kdebug.h> |
34 | #include <linux/string.h> | 34 | #include <linux/string.h> |
35 | #include <linux/rwsem.h> | 35 | #include <linux/rwsem.h> |
36 | #include <linux/slab.h> | ||
36 | #include <linux/ctype.h> | 37 | #include <linux/ctype.h> |
37 | #include <linux/init.h> | 38 | #include <linux/init.h> |
38 | #include <linux/poll.h> | 39 | #include <linux/poll.h> |
39 | #include <linux/gfp.h> | ||
40 | #include <linux/fs.h> | 40 | #include <linux/fs.h> |
41 | 41 | ||
42 | #include "trace.h" | 42 | #include "trace.h" |
diff --git a/kernel/trace/trace_clock.c b/kernel/trace/trace_clock.c index 6fbfb8f417b9..9d589d8dcd1a 100644 --- a/kernel/trace/trace_clock.c +++ b/kernel/trace/trace_clock.c | |||
@@ -84,7 +84,7 @@ u64 notrace trace_clock_global(void) | |||
84 | int this_cpu; | 84 | int this_cpu; |
85 | u64 now; | 85 | u64 now; |
86 | 86 | ||
87 | raw_local_irq_save(flags); | 87 | local_irq_save(flags); |
88 | 88 | ||
89 | this_cpu = raw_smp_processor_id(); | 89 | this_cpu = raw_smp_processor_id(); |
90 | now = cpu_clock(this_cpu); | 90 | now = cpu_clock(this_cpu); |
@@ -110,7 +110,7 @@ u64 notrace trace_clock_global(void) | |||
110 | arch_spin_unlock(&trace_clock_struct.lock); | 110 | arch_spin_unlock(&trace_clock_struct.lock); |
111 | 111 | ||
112 | out: | 112 | out: |
113 | raw_local_irq_restore(flags); | 113 | local_irq_restore(flags); |
114 | 114 | ||
115 | return now; | 115 | return now; |
116 | } | 116 | } |
diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c index 81f691eb3a30..0565bb42566f 100644 --- a/kernel/trace/trace_event_perf.c +++ b/kernel/trace/trace_event_perf.c | |||
@@ -17,7 +17,12 @@ EXPORT_SYMBOL_GPL(perf_arch_fetch_caller_regs); | |||
17 | static char *perf_trace_buf; | 17 | static char *perf_trace_buf; |
18 | static char *perf_trace_buf_nmi; | 18 | static char *perf_trace_buf_nmi; |
19 | 19 | ||
20 | typedef typeof(char [PERF_MAX_TRACE_SIZE]) perf_trace_t ; | 20 | /* |
21 | * Force it to be aligned to unsigned long to avoid misaligned accesses | ||
22 | * suprises | ||
23 | */ | ||
24 | typedef typeof(unsigned long [PERF_MAX_TRACE_SIZE / sizeof(unsigned long)]) | ||
25 | perf_trace_t; | ||
21 | 26 | ||
22 | /* Count the events in use (per event id, not per instance) */ | 27 | /* Count the events in use (per event id, not per instance) */ |
23 | static int total_ref_count; | 28 | static int total_ref_count; |
@@ -130,6 +135,8 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type, | |||
130 | char *trace_buf, *raw_data; | 135 | char *trace_buf, *raw_data; |
131 | int pc, cpu; | 136 | int pc, cpu; |
132 | 137 | ||
138 | BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long)); | ||
139 | |||
133 | pc = preempt_count(); | 140 | pc = preempt_count(); |
134 | 141 | ||
135 | /* Protect the per cpu buffer, begin the rcu read side */ | 142 | /* Protect the per cpu buffer, begin the rcu read side */ |
@@ -152,7 +159,7 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type, | |||
152 | raw_data = per_cpu_ptr(trace_buf, cpu); | 159 | raw_data = per_cpu_ptr(trace_buf, cpu); |
153 | 160 | ||
154 | /* zero the dead bytes from align to not leak stack to user */ | 161 | /* zero the dead bytes from align to not leak stack to user */ |
155 | *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL; | 162 | memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64)); |
156 | 163 | ||
157 | entry = (struct trace_entry *)raw_data; | 164 | entry = (struct trace_entry *)raw_data; |
158 | tracing_generic_entry_update(entry, *irq_flags, pc); | 165 | tracing_generic_entry_update(entry, *irq_flags, pc); |
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index beab8bf2f310..c697c7043349 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c | |||
@@ -15,6 +15,7 @@ | |||
15 | #include <linux/uaccess.h> | 15 | #include <linux/uaccess.h> |
16 | #include <linux/module.h> | 16 | #include <linux/module.h> |
17 | #include <linux/ctype.h> | 17 | #include <linux/ctype.h> |
18 | #include <linux/slab.h> | ||
18 | #include <linux/delay.h> | 19 | #include <linux/delay.h> |
19 | 20 | ||
20 | #include <asm/setup.h> | 21 | #include <asm/setup.h> |
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 4615f62a04f1..88c0b6dbd7fe 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c | |||
@@ -22,6 +22,7 @@ | |||
22 | #include <linux/ctype.h> | 22 | #include <linux/ctype.h> |
23 | #include <linux/mutex.h> | 23 | #include <linux/mutex.h> |
24 | #include <linux/perf_event.h> | 24 | #include <linux/perf_event.h> |
25 | #include <linux/slab.h> | ||
25 | 26 | ||
26 | #include "trace.h" | 27 | #include "trace.h" |
27 | #include "trace_output.h" | 28 | #include "trace_output.h" |
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index e6989d9b44da..9aed1a5cf553 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c | |||
@@ -9,6 +9,7 @@ | |||
9 | #include <linux/debugfs.h> | 9 | #include <linux/debugfs.h> |
10 | #include <linux/uaccess.h> | 10 | #include <linux/uaccess.h> |
11 | #include <linux/ftrace.h> | 11 | #include <linux/ftrace.h> |
12 | #include <linux/slab.h> | ||
12 | #include <linux/fs.h> | 13 | #include <linux/fs.h> |
13 | 14 | ||
14 | #include "trace.h" | 15 | #include "trace.h" |
diff --git a/kernel/trace/trace_ksym.c b/kernel/trace/trace_ksym.c index 94103cdcf9d8..d59cd6879477 100644 --- a/kernel/trace/trace_ksym.c +++ b/kernel/trace/trace_ksym.c | |||
@@ -23,6 +23,7 @@ | |||
23 | #include <linux/debugfs.h> | 23 | #include <linux/debugfs.h> |
24 | #include <linux/ftrace.h> | 24 | #include <linux/ftrace.h> |
25 | #include <linux/module.h> | 25 | #include <linux/module.h> |
26 | #include <linux/slab.h> | ||
26 | #include <linux/fs.h> | 27 | #include <linux/fs.h> |
27 | 28 | ||
28 | #include "trace_output.h" | 29 | #include "trace_output.h" |
diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c index 0acd834659ed..017fa376505d 100644 --- a/kernel/trace/trace_mmiotrace.c +++ b/kernel/trace/trace_mmiotrace.c | |||
@@ -9,6 +9,7 @@ | |||
9 | #include <linux/kernel.h> | 9 | #include <linux/kernel.h> |
10 | #include <linux/mmiotrace.h> | 10 | #include <linux/mmiotrace.h> |
11 | #include <linux/pci.h> | 11 | #include <linux/pci.h> |
12 | #include <linux/slab.h> | ||
12 | #include <linux/time.h> | 13 | #include <linux/time.h> |
13 | 14 | ||
14 | #include <asm/atomic.h> | 15 | #include <asm/atomic.h> |
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c index 280fea470d67..81003b4d617f 100644 --- a/kernel/trace/trace_selftest.c +++ b/kernel/trace/trace_selftest.c | |||
@@ -3,6 +3,7 @@ | |||
3 | #include <linux/stringify.h> | 3 | #include <linux/stringify.h> |
4 | #include <linux/kthread.h> | 4 | #include <linux/kthread.h> |
5 | #include <linux/delay.h> | 5 | #include <linux/delay.h> |
6 | #include <linux/slab.h> | ||
6 | 7 | ||
7 | static inline int trace_valid_entry(struct trace_entry *entry) | 8 | static inline int trace_valid_entry(struct trace_entry *entry) |
8 | { | 9 | { |
diff --git a/kernel/trace/trace_stat.c b/kernel/trace/trace_stat.c index a4bb239eb987..96cffb269e73 100644 --- a/kernel/trace/trace_stat.c +++ b/kernel/trace/trace_stat.c | |||
@@ -10,6 +10,7 @@ | |||
10 | 10 | ||
11 | 11 | ||
12 | #include <linux/list.h> | 12 | #include <linux/list.h> |
13 | #include <linux/slab.h> | ||
13 | #include <linux/rbtree.h> | 14 | #include <linux/rbtree.h> |
14 | #include <linux/debugfs.h> | 15 | #include <linux/debugfs.h> |
15 | #include "trace_stat.h" | 16 | #include "trace_stat.h" |
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index 33c2a5b769dc..4d6d711717f2 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c | |||
@@ -1,5 +1,6 @@ | |||
1 | #include <trace/syscall.h> | 1 | #include <trace/syscall.h> |
2 | #include <trace/events/syscalls.h> | 2 | #include <trace/events/syscalls.h> |
3 | #include <linux/slab.h> | ||
3 | #include <linux/kernel.h> | 4 | #include <linux/kernel.h> |
4 | #include <linux/ftrace.h> | 5 | #include <linux/ftrace.h> |
5 | #include <linux/perf_event.h> | 6 | #include <linux/perf_event.h> |
diff --git a/kernel/trace/trace_workqueue.c b/kernel/trace/trace_workqueue.c index 40cafb07dffd..cc2d2faa7d9e 100644 --- a/kernel/trace/trace_workqueue.c +++ b/kernel/trace/trace_workqueue.c | |||
@@ -9,6 +9,7 @@ | |||
9 | #include <trace/events/workqueue.h> | 9 | #include <trace/events/workqueue.h> |
10 | #include <linux/list.h> | 10 | #include <linux/list.h> |
11 | #include <linux/percpu.h> | 11 | #include <linux/percpu.h> |
12 | #include <linux/slab.h> | ||
12 | #include <linux/kref.h> | 13 | #include <linux/kref.h> |
13 | #include "trace_stat.h" | 14 | #include "trace_stat.h" |
14 | #include "trace.h" | 15 | #include "trace.h" |