diff options
Diffstat (limited to 'kernel')
-rw-r--r-- | kernel/cpu.c | 97 | ||||
-rw-r--r-- | kernel/cpuset.c | 4 | ||||
-rw-r--r-- | kernel/irq/chip.c | 2 | ||||
-rw-r--r-- | kernel/irq/manage.c | 22 | ||||
-rw-r--r-- | kernel/irq/migration.c | 14 | ||||
-rw-r--r-- | kernel/irq/proc.c | 29 | ||||
-rw-r--r-- | kernel/profile.c | 4 | ||||
-rw-r--r-- | kernel/sched.c | 10 | ||||
-rw-r--r-- | kernel/sched_stats.h | 2 | ||||
-rw-r--r-- | kernel/smp.c | 143 | ||||
-rw-r--r-- | kernel/taskstats.c | 2 | ||||
-rw-r--r-- | kernel/time/clockevents.c | 2 | ||||
-rw-r--r-- | kernel/time/tick-broadcast.c | 2 | ||||
-rw-r--r-- | kernel/time/tick-common.c | 12 | ||||
-rw-r--r-- | kernel/trace/trace.c | 4 |
15 files changed, 184 insertions, 165 deletions
diff --git a/kernel/cpu.c b/kernel/cpu.c index 8ea32e8d68b0..2c9f78f3a2fc 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c | |||
@@ -15,29 +15,8 @@ | |||
15 | #include <linux/stop_machine.h> | 15 | #include <linux/stop_machine.h> |
16 | #include <linux/mutex.h> | 16 | #include <linux/mutex.h> |
17 | 17 | ||
18 | /* | 18 | #ifdef CONFIG_SMP |
19 | * Represents all cpu's present in the system | 19 | /* Serializes the updates to cpu_online_mask, cpu_present_mask */ |
20 | * In systems capable of hotplug, this map could dynamically grow | ||
21 | * as new cpu's are detected in the system via any platform specific | ||
22 | * method, such as ACPI for e.g. | ||
23 | */ | ||
24 | cpumask_t cpu_present_map __read_mostly; | ||
25 | EXPORT_SYMBOL(cpu_present_map); | ||
26 | |||
27 | #ifndef CONFIG_SMP | ||
28 | |||
29 | /* | ||
30 | * Represents all cpu's that are currently online. | ||
31 | */ | ||
32 | cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL; | ||
33 | EXPORT_SYMBOL(cpu_online_map); | ||
34 | |||
35 | cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL; | ||
36 | EXPORT_SYMBOL(cpu_possible_map); | ||
37 | |||
38 | #else /* CONFIG_SMP */ | ||
39 | |||
40 | /* Serializes the updates to cpu_online_map, cpu_present_map */ | ||
41 | static DEFINE_MUTEX(cpu_add_remove_lock); | 20 | static DEFINE_MUTEX(cpu_add_remove_lock); |
42 | 21 | ||
43 | static __cpuinitdata RAW_NOTIFIER_HEAD(cpu_chain); | 22 | static __cpuinitdata RAW_NOTIFIER_HEAD(cpu_chain); |
@@ -64,8 +43,6 @@ void __init cpu_hotplug_init(void) | |||
64 | cpu_hotplug.refcount = 0; | 43 | cpu_hotplug.refcount = 0; |
65 | } | 44 | } |
66 | 45 | ||
67 | cpumask_t cpu_active_map; | ||
68 | |||
69 | #ifdef CONFIG_HOTPLUG_CPU | 46 | #ifdef CONFIG_HOTPLUG_CPU |
70 | 47 | ||
71 | void get_online_cpus(void) | 48 | void get_online_cpus(void) |
@@ -96,7 +73,7 @@ EXPORT_SYMBOL_GPL(put_online_cpus); | |||
96 | 73 | ||
97 | /* | 74 | /* |
98 | * The following two API's must be used when attempting | 75 | * The following two API's must be used when attempting |
99 | * to serialize the updates to cpu_online_map, cpu_present_map. | 76 | * to serialize the updates to cpu_online_mask, cpu_present_mask. |
100 | */ | 77 | */ |
101 | void cpu_maps_update_begin(void) | 78 | void cpu_maps_update_begin(void) |
102 | { | 79 | { |
@@ -502,3 +479,71 @@ EXPORT_SYMBOL_GPL(cpu_bit_bitmap); | |||
502 | 479 | ||
503 | const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL; | 480 | const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL; |
504 | EXPORT_SYMBOL(cpu_all_bits); | 481 | EXPORT_SYMBOL(cpu_all_bits); |
482 | |||
483 | #ifdef CONFIG_INIT_ALL_POSSIBLE | ||
484 | static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly | ||
485 | = CPU_BITS_ALL; | ||
486 | #else | ||
487 | static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly; | ||
488 | #endif | ||
489 | const struct cpumask *const cpu_possible_mask = to_cpumask(cpu_possible_bits); | ||
490 | EXPORT_SYMBOL(cpu_possible_mask); | ||
491 | |||
492 | static DECLARE_BITMAP(cpu_online_bits, CONFIG_NR_CPUS) __read_mostly; | ||
493 | const struct cpumask *const cpu_online_mask = to_cpumask(cpu_online_bits); | ||
494 | EXPORT_SYMBOL(cpu_online_mask); | ||
495 | |||
496 | static DECLARE_BITMAP(cpu_present_bits, CONFIG_NR_CPUS) __read_mostly; | ||
497 | const struct cpumask *const cpu_present_mask = to_cpumask(cpu_present_bits); | ||
498 | EXPORT_SYMBOL(cpu_present_mask); | ||
499 | |||
500 | static DECLARE_BITMAP(cpu_active_bits, CONFIG_NR_CPUS) __read_mostly; | ||
501 | const struct cpumask *const cpu_active_mask = to_cpumask(cpu_active_bits); | ||
502 | EXPORT_SYMBOL(cpu_active_mask); | ||
503 | |||
504 | void set_cpu_possible(unsigned int cpu, bool possible) | ||
505 | { | ||
506 | if (possible) | ||
507 | cpumask_set_cpu(cpu, to_cpumask(cpu_possible_bits)); | ||
508 | else | ||
509 | cpumask_clear_cpu(cpu, to_cpumask(cpu_possible_bits)); | ||
510 | } | ||
511 | |||
512 | void set_cpu_present(unsigned int cpu, bool present) | ||
513 | { | ||
514 | if (present) | ||
515 | cpumask_set_cpu(cpu, to_cpumask(cpu_present_bits)); | ||
516 | else | ||
517 | cpumask_clear_cpu(cpu, to_cpumask(cpu_present_bits)); | ||
518 | } | ||
519 | |||
520 | void set_cpu_online(unsigned int cpu, bool online) | ||
521 | { | ||
522 | if (online) | ||
523 | cpumask_set_cpu(cpu, to_cpumask(cpu_online_bits)); | ||
524 | else | ||
525 | cpumask_clear_cpu(cpu, to_cpumask(cpu_online_bits)); | ||
526 | } | ||
527 | |||
528 | void set_cpu_active(unsigned int cpu, bool active) | ||
529 | { | ||
530 | if (active) | ||
531 | cpumask_set_cpu(cpu, to_cpumask(cpu_active_bits)); | ||
532 | else | ||
533 | cpumask_clear_cpu(cpu, to_cpumask(cpu_active_bits)); | ||
534 | } | ||
535 | |||
536 | void init_cpu_present(const struct cpumask *src) | ||
537 | { | ||
538 | cpumask_copy(to_cpumask(cpu_present_bits), src); | ||
539 | } | ||
540 | |||
541 | void init_cpu_possible(const struct cpumask *src) | ||
542 | { | ||
543 | cpumask_copy(to_cpumask(cpu_possible_bits), src); | ||
544 | } | ||
545 | |||
546 | void init_cpu_online(const struct cpumask *src) | ||
547 | { | ||
548 | cpumask_copy(to_cpumask(cpu_online_bits), src); | ||
549 | } | ||
diff --git a/kernel/cpuset.c b/kernel/cpuset.c index 96c0ba13b8cd..39c1a4c1c5a9 100644 --- a/kernel/cpuset.c +++ b/kernel/cpuset.c | |||
@@ -896,7 +896,7 @@ static int update_cpumask(struct cpuset *cs, const char *buf) | |||
896 | if (!*buf) { | 896 | if (!*buf) { |
897 | cpus_clear(trialcs.cpus_allowed); | 897 | cpus_clear(trialcs.cpus_allowed); |
898 | } else { | 898 | } else { |
899 | retval = cpulist_parse(buf, trialcs.cpus_allowed); | 899 | retval = cpulist_parse(buf, &trialcs.cpus_allowed); |
900 | if (retval < 0) | 900 | if (retval < 0) |
901 | return retval; | 901 | return retval; |
902 | 902 | ||
@@ -1482,7 +1482,7 @@ static int cpuset_sprintf_cpulist(char *page, struct cpuset *cs) | |||
1482 | mask = cs->cpus_allowed; | 1482 | mask = cs->cpus_allowed; |
1483 | mutex_unlock(&callback_mutex); | 1483 | mutex_unlock(&callback_mutex); |
1484 | 1484 | ||
1485 | return cpulist_scnprintf(page, PAGE_SIZE, mask); | 1485 | return cpulist_scnprintf(page, PAGE_SIZE, &mask); |
1486 | } | 1486 | } |
1487 | 1487 | ||
1488 | static int cpuset_sprintf_memlist(char *page, struct cpuset *cs) | 1488 | static int cpuset_sprintf_memlist(char *page, struct cpuset *cs) |
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index 6eb3c7952b64..f63c706d25e1 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c | |||
@@ -46,7 +46,7 @@ void dynamic_irq_init(unsigned int irq) | |||
46 | desc->irq_count = 0; | 46 | desc->irq_count = 0; |
47 | desc->irqs_unhandled = 0; | 47 | desc->irqs_unhandled = 0; |
48 | #ifdef CONFIG_SMP | 48 | #ifdef CONFIG_SMP |
49 | cpus_setall(desc->affinity); | 49 | cpumask_setall(&desc->affinity); |
50 | #endif | 50 | #endif |
51 | spin_unlock_irqrestore(&desc->lock, flags); | 51 | spin_unlock_irqrestore(&desc->lock, flags); |
52 | } | 52 | } |
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 540f6c49f3fa..61c4a9b62165 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c | |||
@@ -79,7 +79,7 @@ int irq_can_set_affinity(unsigned int irq) | |||
79 | * @cpumask: cpumask | 79 | * @cpumask: cpumask |
80 | * | 80 | * |
81 | */ | 81 | */ |
82 | int irq_set_affinity(unsigned int irq, cpumask_t cpumask) | 82 | int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask) |
83 | { | 83 | { |
84 | struct irq_desc *desc = irq_to_desc(irq); | 84 | struct irq_desc *desc = irq_to_desc(irq); |
85 | unsigned long flags; | 85 | unsigned long flags; |
@@ -91,14 +91,14 @@ int irq_set_affinity(unsigned int irq, cpumask_t cpumask) | |||
91 | 91 | ||
92 | #ifdef CONFIG_GENERIC_PENDING_IRQ | 92 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
93 | if (desc->status & IRQ_MOVE_PCNTXT || desc->status & IRQ_DISABLED) { | 93 | if (desc->status & IRQ_MOVE_PCNTXT || desc->status & IRQ_DISABLED) { |
94 | desc->affinity = cpumask; | 94 | cpumask_copy(&desc->affinity, cpumask); |
95 | desc->chip->set_affinity(irq, cpumask); | 95 | desc->chip->set_affinity(irq, cpumask); |
96 | } else { | 96 | } else { |
97 | desc->status |= IRQ_MOVE_PENDING; | 97 | desc->status |= IRQ_MOVE_PENDING; |
98 | desc->pending_mask = cpumask; | 98 | cpumask_copy(&desc->pending_mask, cpumask); |
99 | } | 99 | } |
100 | #else | 100 | #else |
101 | desc->affinity = cpumask; | 101 | cpumask_copy(&desc->affinity, cpumask); |
102 | desc->chip->set_affinity(irq, cpumask); | 102 | desc->chip->set_affinity(irq, cpumask); |
103 | #endif | 103 | #endif |
104 | desc->status |= IRQ_AFFINITY_SET; | 104 | desc->status |= IRQ_AFFINITY_SET; |
@@ -112,26 +112,24 @@ int irq_set_affinity(unsigned int irq, cpumask_t cpumask) | |||
112 | */ | 112 | */ |
113 | int do_irq_select_affinity(unsigned int irq, struct irq_desc *desc) | 113 | int do_irq_select_affinity(unsigned int irq, struct irq_desc *desc) |
114 | { | 114 | { |
115 | cpumask_t mask; | ||
116 | |||
117 | if (!irq_can_set_affinity(irq)) | 115 | if (!irq_can_set_affinity(irq)) |
118 | return 0; | 116 | return 0; |
119 | 117 | ||
120 | cpus_and(mask, cpu_online_map, irq_default_affinity); | ||
121 | |||
122 | /* | 118 | /* |
123 | * Preserve an userspace affinity setup, but make sure that | 119 | * Preserve an userspace affinity setup, but make sure that |
124 | * one of the targets is online. | 120 | * one of the targets is online. |
125 | */ | 121 | */ |
126 | if (desc->status & (IRQ_AFFINITY_SET | IRQ_NO_BALANCING)) { | 122 | if (desc->status & (IRQ_AFFINITY_SET | IRQ_NO_BALANCING)) { |
127 | if (cpus_intersects(desc->affinity, cpu_online_map)) | 123 | if (cpumask_any_and(&desc->affinity, cpu_online_mask) |
128 | mask = desc->affinity; | 124 | < nr_cpu_ids) |
125 | goto set_affinity; | ||
129 | else | 126 | else |
130 | desc->status &= ~IRQ_AFFINITY_SET; | 127 | desc->status &= ~IRQ_AFFINITY_SET; |
131 | } | 128 | } |
132 | 129 | ||
133 | desc->affinity = mask; | 130 | cpumask_and(&desc->affinity, cpu_online_mask, &irq_default_affinity); |
134 | desc->chip->set_affinity(irq, mask); | 131 | set_affinity: |
132 | desc->chip->set_affinity(irq, &desc->affinity); | ||
135 | 133 | ||
136 | return 0; | 134 | return 0; |
137 | } | 135 | } |
diff --git a/kernel/irq/migration.c b/kernel/irq/migration.c index 9db681d95814..bd72329e630c 100644 --- a/kernel/irq/migration.c +++ b/kernel/irq/migration.c | |||
@@ -4,7 +4,6 @@ | |||
4 | void move_masked_irq(int irq) | 4 | void move_masked_irq(int irq) |
5 | { | 5 | { |
6 | struct irq_desc *desc = irq_to_desc(irq); | 6 | struct irq_desc *desc = irq_to_desc(irq); |
7 | cpumask_t tmp; | ||
8 | 7 | ||
9 | if (likely(!(desc->status & IRQ_MOVE_PENDING))) | 8 | if (likely(!(desc->status & IRQ_MOVE_PENDING))) |
10 | return; | 9 | return; |
@@ -19,7 +18,7 @@ void move_masked_irq(int irq) | |||
19 | 18 | ||
20 | desc->status &= ~IRQ_MOVE_PENDING; | 19 | desc->status &= ~IRQ_MOVE_PENDING; |
21 | 20 | ||
22 | if (unlikely(cpus_empty(desc->pending_mask))) | 21 | if (unlikely(cpumask_empty(&desc->pending_mask))) |
23 | return; | 22 | return; |
24 | 23 | ||
25 | if (!desc->chip->set_affinity) | 24 | if (!desc->chip->set_affinity) |
@@ -27,8 +26,6 @@ void move_masked_irq(int irq) | |||
27 | 26 | ||
28 | assert_spin_locked(&desc->lock); | 27 | assert_spin_locked(&desc->lock); |
29 | 28 | ||
30 | cpus_and(tmp, desc->pending_mask, cpu_online_map); | ||
31 | |||
32 | /* | 29 | /* |
33 | * If there was a valid mask to work with, please | 30 | * If there was a valid mask to work with, please |
34 | * do the disable, re-program, enable sequence. | 31 | * do the disable, re-program, enable sequence. |
@@ -41,10 +38,13 @@ void move_masked_irq(int irq) | |||
41 | * For correct operation this depends on the caller | 38 | * For correct operation this depends on the caller |
42 | * masking the irqs. | 39 | * masking the irqs. |
43 | */ | 40 | */ |
44 | if (likely(!cpus_empty(tmp))) { | 41 | if (likely(cpumask_any_and(&desc->pending_mask, cpu_online_mask) |
45 | desc->chip->set_affinity(irq,tmp); | 42 | < nr_cpu_ids)) { |
43 | cpumask_and(&desc->affinity, | ||
44 | &desc->pending_mask, cpu_online_mask); | ||
45 | desc->chip->set_affinity(irq, &desc->affinity); | ||
46 | } | 46 | } |
47 | cpus_clear(desc->pending_mask); | 47 | cpumask_clear(&desc->pending_mask); |
48 | } | 48 | } |
49 | 49 | ||
50 | void move_native_irq(int irq) | 50 | void move_native_irq(int irq) |
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index f6b3440f05bc..d2c0e5ee53c5 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c | |||
@@ -40,33 +40,42 @@ static ssize_t irq_affinity_proc_write(struct file *file, | |||
40 | const char __user *buffer, size_t count, loff_t *pos) | 40 | const char __user *buffer, size_t count, loff_t *pos) |
41 | { | 41 | { |
42 | unsigned int irq = (int)(long)PDE(file->f_path.dentry->d_inode)->data; | 42 | unsigned int irq = (int)(long)PDE(file->f_path.dentry->d_inode)->data; |
43 | cpumask_t new_value; | 43 | cpumask_var_t new_value; |
44 | int err; | 44 | int err; |
45 | 45 | ||
46 | if (!irq_to_desc(irq)->chip->set_affinity || no_irq_affinity || | 46 | if (!irq_to_desc(irq)->chip->set_affinity || no_irq_affinity || |
47 | irq_balancing_disabled(irq)) | 47 | irq_balancing_disabled(irq)) |
48 | return -EIO; | 48 | return -EIO; |
49 | 49 | ||
50 | if (!alloc_cpumask_var(&new_value, GFP_KERNEL)) | ||
51 | return -ENOMEM; | ||
52 | |||
50 | err = cpumask_parse_user(buffer, count, new_value); | 53 | err = cpumask_parse_user(buffer, count, new_value); |
51 | if (err) | 54 | if (err) |
52 | return err; | 55 | goto free_cpumask; |
53 | 56 | ||
54 | if (!is_affinity_mask_valid(new_value)) | 57 | if (!is_affinity_mask_valid(*new_value)) { |
55 | return -EINVAL; | 58 | err = -EINVAL; |
59 | goto free_cpumask; | ||
60 | } | ||
56 | 61 | ||
57 | /* | 62 | /* |
58 | * Do not allow disabling IRQs completely - it's a too easy | 63 | * Do not allow disabling IRQs completely - it's a too easy |
59 | * way to make the system unusable accidentally :-) At least | 64 | * way to make the system unusable accidentally :-) At least |
60 | * one online CPU still has to be targeted. | 65 | * one online CPU still has to be targeted. |
61 | */ | 66 | */ |
62 | if (!cpus_intersects(new_value, cpu_online_map)) | 67 | if (!cpumask_intersects(new_value, cpu_online_mask)) { |
63 | /* Special case for empty set - allow the architecture | 68 | /* Special case for empty set - allow the architecture |
64 | code to set default SMP affinity. */ | 69 | code to set default SMP affinity. */ |
65 | return irq_select_affinity_usr(irq) ? -EINVAL : count; | 70 | err = irq_select_affinity_usr(irq) ? -EINVAL : count; |
66 | 71 | } else { | |
67 | irq_set_affinity(irq, new_value); | 72 | irq_set_affinity(irq, new_value); |
73 | err = count; | ||
74 | } | ||
68 | 75 | ||
69 | return count; | 76 | free_cpumask: |
77 | free_cpumask_var(new_value); | ||
78 | return err; | ||
70 | } | 79 | } |
71 | 80 | ||
72 | static int irq_affinity_proc_open(struct inode *inode, struct file *file) | 81 | static int irq_affinity_proc_open(struct inode *inode, struct file *file) |
@@ -95,7 +104,7 @@ static ssize_t default_affinity_write(struct file *file, | |||
95 | cpumask_t new_value; | 104 | cpumask_t new_value; |
96 | int err; | 105 | int err; |
97 | 106 | ||
98 | err = cpumask_parse_user(buffer, count, new_value); | 107 | err = cpumask_parse_user(buffer, count, &new_value); |
99 | if (err) | 108 | if (err) |
100 | return err; | 109 | return err; |
101 | 110 | ||
diff --git a/kernel/profile.c b/kernel/profile.c index 60adefb59b5e..4cb7d68fed82 100644 --- a/kernel/profile.c +++ b/kernel/profile.c | |||
@@ -442,7 +442,7 @@ void profile_tick(int type) | |||
442 | static int prof_cpu_mask_read_proc(char *page, char **start, off_t off, | 442 | static int prof_cpu_mask_read_proc(char *page, char **start, off_t off, |
443 | int count, int *eof, void *data) | 443 | int count, int *eof, void *data) |
444 | { | 444 | { |
445 | int len = cpumask_scnprintf(page, count, *(cpumask_t *)data); | 445 | int len = cpumask_scnprintf(page, count, (cpumask_t *)data); |
446 | if (count - len < 2) | 446 | if (count - len < 2) |
447 | return -EINVAL; | 447 | return -EINVAL; |
448 | len += sprintf(page + len, "\n"); | 448 | len += sprintf(page + len, "\n"); |
@@ -456,7 +456,7 @@ static int prof_cpu_mask_write_proc(struct file *file, | |||
456 | unsigned long full_count = count, err; | 456 | unsigned long full_count = count, err; |
457 | cpumask_t new_value; | 457 | cpumask_t new_value; |
458 | 458 | ||
459 | err = cpumask_parse_user(buffer, count, new_value); | 459 | err = cpumask_parse_user(buffer, count, &new_value); |
460 | if (err) | 460 | if (err) |
461 | return err; | 461 | return err; |
462 | 462 | ||
diff --git a/kernel/sched.c b/kernel/sched.c index fff1c4a20b65..f2095660efec 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
@@ -6644,7 +6644,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, | |||
6644 | struct sched_group *group = sd->groups; | 6644 | struct sched_group *group = sd->groups; |
6645 | char str[256]; | 6645 | char str[256]; |
6646 | 6646 | ||
6647 | cpulist_scnprintf(str, sizeof(str), sd->span); | 6647 | cpulist_scnprintf(str, sizeof(str), &sd->span); |
6648 | cpus_clear(*groupmask); | 6648 | cpus_clear(*groupmask); |
6649 | 6649 | ||
6650 | printk(KERN_DEBUG "%*s domain %d: ", level, "", level); | 6650 | printk(KERN_DEBUG "%*s domain %d: ", level, "", level); |
@@ -6697,7 +6697,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, | |||
6697 | 6697 | ||
6698 | cpus_or(*groupmask, *groupmask, group->cpumask); | 6698 | cpus_or(*groupmask, *groupmask, group->cpumask); |
6699 | 6699 | ||
6700 | cpulist_scnprintf(str, sizeof(str), group->cpumask); | 6700 | cpulist_scnprintf(str, sizeof(str), &group->cpumask); |
6701 | printk(KERN_CONT " %s", str); | 6701 | printk(KERN_CONT " %s", str); |
6702 | 6702 | ||
6703 | group = group->next; | 6703 | group = group->next; |
@@ -7098,7 +7098,7 @@ cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, | |||
7098 | { | 7098 | { |
7099 | int group; | 7099 | int group; |
7100 | #ifdef CONFIG_SCHED_MC | 7100 | #ifdef CONFIG_SCHED_MC |
7101 | *mask = cpu_coregroup_map(cpu); | 7101 | *mask = *cpu_coregroup_mask(cpu); |
7102 | cpus_and(*mask, *mask, *cpu_map); | 7102 | cpus_and(*mask, *mask, *cpu_map); |
7103 | group = first_cpu(*mask); | 7103 | group = first_cpu(*mask); |
7104 | #elif defined(CONFIG_SCHED_SMT) | 7104 | #elif defined(CONFIG_SCHED_SMT) |
@@ -7471,7 +7471,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map, | |||
7471 | sd = &per_cpu(core_domains, i); | 7471 | sd = &per_cpu(core_domains, i); |
7472 | SD_INIT(sd, MC); | 7472 | SD_INIT(sd, MC); |
7473 | set_domain_attribute(sd, attr); | 7473 | set_domain_attribute(sd, attr); |
7474 | sd->span = cpu_coregroup_map(i); | 7474 | sd->span = *cpu_coregroup_mask(i); |
7475 | cpus_and(sd->span, sd->span, *cpu_map); | 7475 | cpus_and(sd->span, sd->span, *cpu_map); |
7476 | sd->parent = p; | 7476 | sd->parent = p; |
7477 | p->child = sd; | 7477 | p->child = sd; |
@@ -7514,7 +7514,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map, | |||
7514 | SCHED_CPUMASK_VAR(this_core_map, allmasks); | 7514 | SCHED_CPUMASK_VAR(this_core_map, allmasks); |
7515 | SCHED_CPUMASK_VAR(send_covered, allmasks); | 7515 | SCHED_CPUMASK_VAR(send_covered, allmasks); |
7516 | 7516 | ||
7517 | *this_core_map = cpu_coregroup_map(i); | 7517 | *this_core_map = *cpu_coregroup_mask(i); |
7518 | cpus_and(*this_core_map, *this_core_map, *cpu_map); | 7518 | cpus_and(*this_core_map, *this_core_map, *cpu_map); |
7519 | if (i != first_cpu(*this_core_map)) | 7519 | if (i != first_cpu(*this_core_map)) |
7520 | continue; | 7520 | continue; |
diff --git a/kernel/sched_stats.h b/kernel/sched_stats.h index 3b01098164c8..b59fd9cdc1b6 100644 --- a/kernel/sched_stats.h +++ b/kernel/sched_stats.h | |||
@@ -42,7 +42,7 @@ static int show_schedstat(struct seq_file *seq, void *v) | |||
42 | for_each_domain(cpu, sd) { | 42 | for_each_domain(cpu, sd) { |
43 | enum cpu_idle_type itype; | 43 | enum cpu_idle_type itype; |
44 | 44 | ||
45 | cpumask_scnprintf(mask_str, mask_len, sd->span); | 45 | cpumask_scnprintf(mask_str, mask_len, &sd->span); |
46 | seq_printf(seq, "domain%d %s", dcount++, mask_str); | 46 | seq_printf(seq, "domain%d %s", dcount++, mask_str); |
47 | for (itype = CPU_IDLE; itype < CPU_MAX_IDLE_TYPES; | 47 | for (itype = CPU_IDLE; itype < CPU_MAX_IDLE_TYPES; |
48 | itype++) { | 48 | itype++) { |
diff --git a/kernel/smp.c b/kernel/smp.c index 75c8dde58c55..172b18268909 100644 --- a/kernel/smp.c +++ b/kernel/smp.c | |||
@@ -24,8 +24,8 @@ struct call_function_data { | |||
24 | struct call_single_data csd; | 24 | struct call_single_data csd; |
25 | spinlock_t lock; | 25 | spinlock_t lock; |
26 | unsigned int refs; | 26 | unsigned int refs; |
27 | cpumask_t cpumask; | ||
28 | struct rcu_head rcu_head; | 27 | struct rcu_head rcu_head; |
28 | unsigned long cpumask_bits[]; | ||
29 | }; | 29 | }; |
30 | 30 | ||
31 | struct call_single_queue { | 31 | struct call_single_queue { |
@@ -110,13 +110,13 @@ void generic_smp_call_function_interrupt(void) | |||
110 | list_for_each_entry_rcu(data, &call_function_queue, csd.list) { | 110 | list_for_each_entry_rcu(data, &call_function_queue, csd.list) { |
111 | int refs; | 111 | int refs; |
112 | 112 | ||
113 | if (!cpu_isset(cpu, data->cpumask)) | 113 | if (!cpumask_test_cpu(cpu, to_cpumask(data->cpumask_bits))) |
114 | continue; | 114 | continue; |
115 | 115 | ||
116 | data->csd.func(data->csd.info); | 116 | data->csd.func(data->csd.info); |
117 | 117 | ||
118 | spin_lock(&data->lock); | 118 | spin_lock(&data->lock); |
119 | cpu_clear(cpu, data->cpumask); | 119 | cpumask_clear_cpu(cpu, to_cpumask(data->cpumask_bits)); |
120 | WARN_ON(data->refs == 0); | 120 | WARN_ON(data->refs == 0); |
121 | data->refs--; | 121 | data->refs--; |
122 | refs = data->refs; | 122 | refs = data->refs; |
@@ -266,51 +266,19 @@ void __smp_call_function_single(int cpu, struct call_single_data *data) | |||
266 | generic_exec_single(cpu, data); | 266 | generic_exec_single(cpu, data); |
267 | } | 267 | } |
268 | 268 | ||
269 | /* Dummy function */ | 269 | /* FIXME: Shim for archs using old arch_send_call_function_ipi API. */ |
270 | static void quiesce_dummy(void *unused) | 270 | #ifndef arch_send_call_function_ipi_mask |
271 | { | 271 | #define arch_send_call_function_ipi_mask(maskp) \ |
272 | } | 272 | arch_send_call_function_ipi(*(maskp)) |
273 | 273 | #endif | |
274 | /* | ||
275 | * Ensure stack based data used in call function mask is safe to free. | ||
276 | * | ||
277 | * This is needed by smp_call_function_mask when using on-stack data, because | ||
278 | * a single call function queue is shared by all CPUs, and any CPU may pick up | ||
279 | * the data item on the queue at any time before it is deleted. So we need to | ||
280 | * ensure that all CPUs have transitioned through a quiescent state after | ||
281 | * this call. | ||
282 | * | ||
283 | * This is a very slow function, implemented by sending synchronous IPIs to | ||
284 | * all possible CPUs. For this reason, we have to alloc data rather than use | ||
285 | * stack based data even in the case of synchronous calls. The stack based | ||
286 | * data is then just used for deadlock/oom fallback which will be very rare. | ||
287 | * | ||
288 | * If a faster scheme can be made, we could go back to preferring stack based | ||
289 | * data -- the data allocation/free is non-zero cost. | ||
290 | */ | ||
291 | static void smp_call_function_mask_quiesce_stack(cpumask_t mask) | ||
292 | { | ||
293 | struct call_single_data data; | ||
294 | int cpu; | ||
295 | |||
296 | data.func = quiesce_dummy; | ||
297 | data.info = NULL; | ||
298 | |||
299 | for_each_cpu_mask(cpu, mask) { | ||
300 | data.flags = CSD_FLAG_WAIT; | ||
301 | generic_exec_single(cpu, &data); | ||
302 | } | ||
303 | } | ||
304 | 274 | ||
305 | /** | 275 | /** |
306 | * smp_call_function_mask(): Run a function on a set of other CPUs. | 276 | * smp_call_function_many(): Run a function on a set of other CPUs. |
307 | * @mask: The set of cpus to run on. | 277 | * @mask: The set of cpus to run on (only runs on online subset). |
308 | * @func: The function to run. This must be fast and non-blocking. | 278 | * @func: The function to run. This must be fast and non-blocking. |
309 | * @info: An arbitrary pointer to pass to the function. | 279 | * @info: An arbitrary pointer to pass to the function. |
310 | * @wait: If true, wait (atomically) until function has completed on other CPUs. | 280 | * @wait: If true, wait (atomically) until function has completed on other CPUs. |
311 | * | 281 | * |
312 | * Returns 0 on success, else a negative status code. | ||
313 | * | ||
314 | * If @wait is true, then returns once @func has returned. Note that @wait | 282 | * If @wait is true, then returns once @func has returned. Note that @wait |
315 | * will be implicitly turned on in case of allocation failures, since | 283 | * will be implicitly turned on in case of allocation failures, since |
316 | * we fall back to on-stack allocation. | 284 | * we fall back to on-stack allocation. |
@@ -319,53 +287,57 @@ static void smp_call_function_mask_quiesce_stack(cpumask_t mask) | |||
319 | * hardware interrupt handler or from a bottom half handler. Preemption | 287 | * hardware interrupt handler or from a bottom half handler. Preemption |
320 | * must be disabled when calling this function. | 288 | * must be disabled when calling this function. |
321 | */ | 289 | */ |
322 | int smp_call_function_mask(cpumask_t mask, void (*func)(void *), void *info, | 290 | void smp_call_function_many(const struct cpumask *mask, |
323 | int wait) | 291 | void (*func)(void *), void *info, |
292 | bool wait) | ||
324 | { | 293 | { |
325 | struct call_function_data d; | 294 | struct call_function_data *data; |
326 | struct call_function_data *data = NULL; | ||
327 | cpumask_t allbutself; | ||
328 | unsigned long flags; | 295 | unsigned long flags; |
329 | int cpu, num_cpus; | 296 | int cpu, next_cpu; |
330 | int slowpath = 0; | ||
331 | 297 | ||
332 | /* Can deadlock when called with interrupts disabled */ | 298 | /* Can deadlock when called with interrupts disabled */ |
333 | WARN_ON(irqs_disabled()); | 299 | WARN_ON(irqs_disabled()); |
334 | 300 | ||
335 | cpu = smp_processor_id(); | 301 | /* So, what's a CPU they want? Ignoring this one. */ |
336 | allbutself = cpu_online_map; | 302 | cpu = cpumask_first_and(mask, cpu_online_mask); |
337 | cpu_clear(cpu, allbutself); | 303 | if (cpu == smp_processor_id()) |
338 | cpus_and(mask, mask, allbutself); | 304 | cpu = cpumask_next_and(cpu, mask, cpu_online_mask); |
339 | num_cpus = cpus_weight(mask); | 305 | /* No online cpus? We're done. */ |
340 | 306 | if (cpu >= nr_cpu_ids) | |
341 | /* | 307 | return; |
342 | * If zero CPUs, return. If just a single CPU, turn this request | 308 | |
343 | * into a targetted single call instead since it's faster. | 309 | /* Do we have another CPU which isn't us? */ |
344 | */ | 310 | next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask); |
345 | if (!num_cpus) | 311 | if (next_cpu == smp_processor_id()) |
346 | return 0; | 312 | next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask); |
347 | else if (num_cpus == 1) { | 313 | |
348 | cpu = first_cpu(mask); | 314 | /* Fastpath: do that cpu by itself. */ |
349 | return smp_call_function_single(cpu, func, info, wait); | 315 | if (next_cpu >= nr_cpu_ids) { |
316 | smp_call_function_single(cpu, func, info, wait); | ||
317 | return; | ||
350 | } | 318 | } |
351 | 319 | ||
352 | data = kmalloc(sizeof(*data), GFP_ATOMIC); | 320 | data = kmalloc(sizeof(*data) + cpumask_size(), GFP_ATOMIC); |
353 | if (data) { | 321 | if (unlikely(!data)) { |
354 | data->csd.flags = CSD_FLAG_ALLOC; | 322 | /* Slow path. */ |
355 | if (wait) | 323 | for_each_online_cpu(cpu) { |
356 | data->csd.flags |= CSD_FLAG_WAIT; | 324 | if (cpu == smp_processor_id()) |
357 | } else { | 325 | continue; |
358 | data = &d; | 326 | if (cpumask_test_cpu(cpu, mask)) |
359 | data->csd.flags = CSD_FLAG_WAIT; | 327 | smp_call_function_single(cpu, func, info, wait); |
360 | wait = 1; | 328 | } |
361 | slowpath = 1; | 329 | return; |
362 | } | 330 | } |
363 | 331 | ||
364 | spin_lock_init(&data->lock); | 332 | spin_lock_init(&data->lock); |
333 | data->csd.flags = CSD_FLAG_ALLOC; | ||
334 | if (wait) | ||
335 | data->csd.flags |= CSD_FLAG_WAIT; | ||
365 | data->csd.func = func; | 336 | data->csd.func = func; |
366 | data->csd.info = info; | 337 | data->csd.info = info; |
367 | data->refs = num_cpus; | 338 | cpumask_and(to_cpumask(data->cpumask_bits), mask, cpu_online_mask); |
368 | data->cpumask = mask; | 339 | cpumask_clear_cpu(smp_processor_id(), to_cpumask(data->cpumask_bits)); |
340 | data->refs = cpumask_weight(to_cpumask(data->cpumask_bits)); | ||
369 | 341 | ||
370 | spin_lock_irqsave(&call_function_lock, flags); | 342 | spin_lock_irqsave(&call_function_lock, flags); |
371 | list_add_tail_rcu(&data->csd.list, &call_function_queue); | 343 | list_add_tail_rcu(&data->csd.list, &call_function_queue); |
@@ -377,18 +349,13 @@ int smp_call_function_mask(cpumask_t mask, void (*func)(void *), void *info, | |||
377 | smp_mb(); | 349 | smp_mb(); |
378 | 350 | ||
379 | /* Send a message to all CPUs in the map */ | 351 | /* Send a message to all CPUs in the map */ |
380 | arch_send_call_function_ipi(mask); | 352 | arch_send_call_function_ipi_mask(to_cpumask(data->cpumask_bits)); |
381 | 353 | ||
382 | /* optionally wait for the CPUs to complete */ | 354 | /* optionally wait for the CPUs to complete */ |
383 | if (wait) { | 355 | if (wait) |
384 | csd_flag_wait(&data->csd); | 356 | csd_flag_wait(&data->csd); |
385 | if (unlikely(slowpath)) | ||
386 | smp_call_function_mask_quiesce_stack(mask); | ||
387 | } | ||
388 | |||
389 | return 0; | ||
390 | } | 357 | } |
391 | EXPORT_SYMBOL(smp_call_function_mask); | 358 | EXPORT_SYMBOL(smp_call_function_many); |
392 | 359 | ||
393 | /** | 360 | /** |
394 | * smp_call_function(): Run a function on all other CPUs. | 361 | * smp_call_function(): Run a function on all other CPUs. |
@@ -396,7 +363,7 @@ EXPORT_SYMBOL(smp_call_function_mask); | |||
396 | * @info: An arbitrary pointer to pass to the function. | 363 | * @info: An arbitrary pointer to pass to the function. |
397 | * @wait: If true, wait (atomically) until function has completed on other CPUs. | 364 | * @wait: If true, wait (atomically) until function has completed on other CPUs. |
398 | * | 365 | * |
399 | * Returns 0 on success, else a negative status code. | 366 | * Returns 0. |
400 | * | 367 | * |
401 | * If @wait is true, then returns once @func has returned; otherwise | 368 | * If @wait is true, then returns once @func has returned; otherwise |
402 | * it returns just before the target cpu calls @func. In case of allocation | 369 | * it returns just before the target cpu calls @func. In case of allocation |
@@ -407,12 +374,10 @@ EXPORT_SYMBOL(smp_call_function_mask); | |||
407 | */ | 374 | */ |
408 | int smp_call_function(void (*func)(void *), void *info, int wait) | 375 | int smp_call_function(void (*func)(void *), void *info, int wait) |
409 | { | 376 | { |
410 | int ret; | ||
411 | |||
412 | preempt_disable(); | 377 | preempt_disable(); |
413 | ret = smp_call_function_mask(cpu_online_map, func, info, wait); | 378 | smp_call_function_many(cpu_online_mask, func, info, wait); |
414 | preempt_enable(); | 379 | preempt_enable(); |
415 | return ret; | 380 | return 0; |
416 | } | 381 | } |
417 | EXPORT_SYMBOL(smp_call_function); | 382 | EXPORT_SYMBOL(smp_call_function); |
418 | 383 | ||
diff --git a/kernel/taskstats.c b/kernel/taskstats.c index bd6be76303cf..6d7dc4ec4aa5 100644 --- a/kernel/taskstats.c +++ b/kernel/taskstats.c | |||
@@ -352,7 +352,7 @@ static int parse(struct nlattr *na, cpumask_t *mask) | |||
352 | if (!data) | 352 | if (!data) |
353 | return -ENOMEM; | 353 | return -ENOMEM; |
354 | nla_strlcpy(data, na, len); | 354 | nla_strlcpy(data, na, len); |
355 | ret = cpulist_parse(data, *mask); | 355 | ret = cpulist_parse(data, mask); |
356 | kfree(data); | 356 | kfree(data); |
357 | return ret; | 357 | return ret; |
358 | } | 358 | } |
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index f8d968063cea..ea2f48af83cf 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c | |||
@@ -166,6 +166,8 @@ static void clockevents_notify_released(void) | |||
166 | void clockevents_register_device(struct clock_event_device *dev) | 166 | void clockevents_register_device(struct clock_event_device *dev) |
167 | { | 167 | { |
168 | BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED); | 168 | BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED); |
169 | BUG_ON(!dev->cpumask); | ||
170 | |||
169 | /* | 171 | /* |
170 | * A nsec2cyc multiplicator of 0 is invalid and we'd crash | 172 | * A nsec2cyc multiplicator of 0 is invalid and we'd crash |
171 | * on it, so fix it up and emit a warning: | 173 | * on it, so fix it up and emit a warning: |
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index f98a1b7b16e9..9590af2327be 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c | |||
@@ -150,7 +150,7 @@ static void tick_do_broadcast(cpumask_t mask) | |||
150 | */ | 150 | */ |
151 | cpu = first_cpu(mask); | 151 | cpu = first_cpu(mask); |
152 | td = &per_cpu(tick_cpu_device, cpu); | 152 | td = &per_cpu(tick_cpu_device, cpu); |
153 | td->evtdev->broadcast(mask); | 153 | td->evtdev->broadcast(&mask); |
154 | } | 154 | } |
155 | } | 155 | } |
156 | 156 | ||
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c index df12434b43ca..f8372be74122 100644 --- a/kernel/time/tick-common.c +++ b/kernel/time/tick-common.c | |||
@@ -136,7 +136,7 @@ void tick_setup_periodic(struct clock_event_device *dev, int broadcast) | |||
136 | */ | 136 | */ |
137 | static void tick_setup_device(struct tick_device *td, | 137 | static void tick_setup_device(struct tick_device *td, |
138 | struct clock_event_device *newdev, int cpu, | 138 | struct clock_event_device *newdev, int cpu, |
139 | const cpumask_t *cpumask) | 139 | const struct cpumask *cpumask) |
140 | { | 140 | { |
141 | ktime_t next_event; | 141 | ktime_t next_event; |
142 | void (*handler)(struct clock_event_device *) = NULL; | 142 | void (*handler)(struct clock_event_device *) = NULL; |
@@ -171,8 +171,8 @@ static void tick_setup_device(struct tick_device *td, | |||
171 | * When the device is not per cpu, pin the interrupt to the | 171 | * When the device is not per cpu, pin the interrupt to the |
172 | * current cpu: | 172 | * current cpu: |
173 | */ | 173 | */ |
174 | if (!cpus_equal(newdev->cpumask, *cpumask)) | 174 | if (!cpumask_equal(newdev->cpumask, cpumask)) |
175 | irq_set_affinity(newdev->irq, *cpumask); | 175 | irq_set_affinity(newdev->irq, cpumask); |
176 | 176 | ||
177 | /* | 177 | /* |
178 | * When global broadcasting is active, check if the current | 178 | * When global broadcasting is active, check if the current |
@@ -202,14 +202,14 @@ static int tick_check_new_device(struct clock_event_device *newdev) | |||
202 | spin_lock_irqsave(&tick_device_lock, flags); | 202 | spin_lock_irqsave(&tick_device_lock, flags); |
203 | 203 | ||
204 | cpu = smp_processor_id(); | 204 | cpu = smp_processor_id(); |
205 | if (!cpu_isset(cpu, newdev->cpumask)) | 205 | if (!cpumask_test_cpu(cpu, newdev->cpumask)) |
206 | goto out_bc; | 206 | goto out_bc; |
207 | 207 | ||
208 | td = &per_cpu(tick_cpu_device, cpu); | 208 | td = &per_cpu(tick_cpu_device, cpu); |
209 | curdev = td->evtdev; | 209 | curdev = td->evtdev; |
210 | 210 | ||
211 | /* cpu local device ? */ | 211 | /* cpu local device ? */ |
212 | if (!cpus_equal(newdev->cpumask, cpumask_of_cpu(cpu))) { | 212 | if (!cpumask_equal(newdev->cpumask, cpumask_of(cpu))) { |
213 | 213 | ||
214 | /* | 214 | /* |
215 | * If the cpu affinity of the device interrupt can not | 215 | * If the cpu affinity of the device interrupt can not |
@@ -222,7 +222,7 @@ static int tick_check_new_device(struct clock_event_device *newdev) | |||
222 | * If we have a cpu local device already, do not replace it | 222 | * If we have a cpu local device already, do not replace it |
223 | * by a non cpu local device | 223 | * by a non cpu local device |
224 | */ | 224 | */ |
225 | if (curdev && cpus_equal(curdev->cpumask, cpumask_of_cpu(cpu))) | 225 | if (curdev && cpumask_equal(curdev->cpumask, cpumask_of(cpu))) |
226 | goto out_bc; | 226 | goto out_bc; |
227 | } | 227 | } |
228 | 228 | ||
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 4185d5221633..0e91f43b6baf 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
@@ -2674,7 +2674,7 @@ tracing_cpumask_read(struct file *filp, char __user *ubuf, | |||
2674 | 2674 | ||
2675 | mutex_lock(&tracing_cpumask_update_lock); | 2675 | mutex_lock(&tracing_cpumask_update_lock); |
2676 | 2676 | ||
2677 | len = cpumask_scnprintf(mask_str, count, tracing_cpumask); | 2677 | len = cpumask_scnprintf(mask_str, count, &tracing_cpumask); |
2678 | if (count - len < 2) { | 2678 | if (count - len < 2) { |
2679 | count = -EINVAL; | 2679 | count = -EINVAL; |
2680 | goto out_err; | 2680 | goto out_err; |
@@ -2695,7 +2695,7 @@ tracing_cpumask_write(struct file *filp, const char __user *ubuf, | |||
2695 | int err, cpu; | 2695 | int err, cpu; |
2696 | 2696 | ||
2697 | mutex_lock(&tracing_cpumask_update_lock); | 2697 | mutex_lock(&tracing_cpumask_update_lock); |
2698 | err = cpumask_parse_user(ubuf, count, tracing_cpumask_new); | 2698 | err = cpumask_parse_user(ubuf, count, &tracing_cpumask_new); |
2699 | if (err) | 2699 | if (err) |
2700 | goto err_unlock; | 2700 | goto err_unlock; |
2701 | 2701 | ||