3 files changed, 130 insertions, 40 deletions

diff --git a/include/asm-x86/ipi.h b/include/asm-x86/ipi.h
index 196d63c28aa4..bb1c09f7a76c 100644
--- a/include/asm-x86/ipi.h
+++ b/include/asm-x86/ipi.h
@@ -122,7 +122,7 @@ static inline void send_IPI_mask_sequence(cpumask_t mask, int vector)
          * - mbligh
          */
         local_irq_save(flags);
-        for_each_cpu_mask(query_cpu, mask) {
+        for_each_cpu_mask_nr(query_cpu, mask) {
                 __send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, query_cpu),
                                       vector, APIC_DEST_PHYSICAL);
         }
diff --git a/include/asm-x86/processor.h b/include/asm-x86/processor.h
index 55402d2ab938..f92315ed68e1 100644
--- a/include/asm-x86/processor.h
+++ b/include/asm-x86/processor.h
@@ -134,7 +134,7 @@ extern __u32			cleared_cpu_caps[NCAPINTS];
 #ifdef CONFIG_SMP
 DECLARE_PER_CPU(struct cpuinfo_x86, cpu_info);
 #define cpu_data(cpu)           per_cpu(cpu_info, cpu)
-#define current_cpu_data        cpu_data(smp_processor_id())
+#define current_cpu_data        __get_cpu_var(cpu_info)
 #else
 #define cpu_data(cpu)           boot_cpu_data
 #define current_cpu_data        boot_cpu_data
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index c24875bd9c5b..30d59d1d0626 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -17,6 +17,20 @@
  * For details of cpus_onto(), see bitmap_onto in lib/bitmap.c.
  * For details of cpus_fold(), see bitmap_fold in lib/bitmap.c.
  *
+ * . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
+ * Note: The alternate operations with the suffix "_nr" are used
+ *       to limit the range of the loop to nr_cpu_ids instead of
+ *       NR_CPUS when NR_CPUS > 64 for performance reasons.
+ *       If NR_CPUS is <= 64 then most assembler bitmask
+ *       operators execute faster with a constant range, so
+ *       the operator will continue to use NR_CPUS.
+ *
+ *       Another consideration is that nr_cpu_ids is initialized
+ *       to NR_CPUS and isn't lowered until the possible cpus are
+ *       discovered (including any disabled cpus).  So early uses
+ *       will span the entire range of NR_CPUS.
+ * . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
+ *
  * The available cpumask operations are:
  *
  * void cpu_set(cpu, mask)              turn on bit 'cpu' in mask
@@ -38,18 +52,60 @@
  * int cpus_empty(mask)                 Is mask empty (no bits sets)?
  * int cpus_full(mask)                  Is mask full (all bits sets)?
  * int cpus_weight(mask)                Hamming weigh - number of set bits
+ * int cpus_weight_nr(mask)             Same using nr_cpu_ids instead of NR_CPUS
  *
  * void cpus_shift_right(dst, src, n)   Shift right
  * void cpus_shift_left(dst, src, n)    Shift left
  *
  * int first_cpu(mask)                  Number lowest set bit, or NR_CPUS
  * int next_cpu(cpu, mask)              Next cpu past 'cpu', or NR_CPUS
+ * int next_cpu_nr(cpu, mask)           Next cpu past 'cpu', or nr_cpu_ids
  *
  * cpumask_t cpumask_of_cpu(cpu)        Return cpumask with bit 'cpu' set
+ *ifdef CONFIG_HAS_CPUMASK_OF_CPU
+ * cpumask_of_cpu_ptr_declare(v)        Declares cpumask_t *v
+ * cpumask_of_cpu_ptr_next(v, cpu)      Sets v = &cpumask_of_cpu_map[cpu]
+ * cpumask_of_cpu_ptr(v, cpu)           Combines above two operations
+ *else
+ * cpumask_of_cpu_ptr_declare(v)        Declares cpumask_t _v and *v = &_v
+ * cpumask_of_cpu_ptr_next(v, cpu)      Sets _v = cpumask_of_cpu(cpu)
+ * cpumask_of_cpu_ptr(v, cpu)           Combines above two operations
+ *endif
  * CPU_MASK_ALL                         Initializer - all bits set
  * CPU_MASK_NONE                        Initializer - no bits set
  * unsigned long *cpus_addr(mask)       Array of unsigned long's in mask
  *
+ * CPUMASK_ALLOC kmalloc's a structure that is a composite of many cpumask_t
+ * variables, and CPUMASK_PTR provides pointers to each field.
+ *
+ * The structure should be defined something like this:
+ * struct my_cpumasks {
+ *      cpumask_t mask1;
+ *      cpumask_t mask2;
+ * };
+ *
+ * Usage is then:
+ *      CPUMASK_ALLOC(my_cpumasks);
+ *      CPUMASK_PTR(mask1, my_cpumasks);
+ *      CPUMASK_PTR(mask2, my_cpumasks);
+ *
+ *      --- DO NOT reference cpumask_t pointers until this check ---
+ *      if (my_cpumasks == NULL)
+ *              "kmalloc failed"...
+ *
+ * References are now pointers to the cpumask_t variables (*mask1, ...)
+ *
+ *if NR_CPUS > BITS_PER_LONG
+ *   CPUMASK_ALLOC(m)                   Declares and allocates struct m *m =
+ *                                              kmalloc(sizeof(*m), GFP_KERNEL)
+ *   CPUMASK_FREE(m)                    Macro for kfree(m)
+ *else
+ *   CPUMASK_ALLOC(m)                   Declares struct m _m, *m = &_m
+ *   CPUMASK_FREE(m)                    Nop
+ *endif
+ *   CPUMASK_PTR(v, m)                  Declares cpumask_t *v = &(m->v)
+ * ------------------------------------------------------------------------
+ *
  * int cpumask_scnprintf(buf, len, mask) Format cpumask for printing
  * int cpumask_parse_user(ubuf, ulen, mask)     Parse ascii string as cpumask
  * int cpulist_scnprintf(buf, len, mask) Format cpumask as list for printing
@@ -59,7 +115,8 @@
  * void cpus_onto(dst, orig, relmap)    *dst = orig relative to relmap
  * void cpus_fold(dst, orig, sz)        dst bits = orig bits mod sz
  *
- * for_each_cpu_mask(cpu, mask)         for-loop cpu over mask
+ * for_each_cpu_mask(cpu, mask)         for-loop cpu over mask using NR_CPUS
+ * for_each_cpu_mask_nr(cpu, mask)      for-loop cpu over mask using nr_cpu_ids
  *
  * int num_online_cpus()                Number of online CPUs
  * int num_possible_cpus()              Number of all possible CPUs
@@ -216,23 +273,19 @@ static inline void __cpus_shift_left(cpumask_t *dstp,
         bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
 }
 
-#ifdef CONFIG_SMP
-int __first_cpu(const cpumask_t *srcp);
-#define first_cpu(src) __first_cpu(&(src))
-int __next_cpu(int n, const cpumask_t *srcp);
-#define next_cpu(n, src) __next_cpu((n), &(src))
-#else
-#define first_cpu(src)          ({ (void)(src); 0; })
-#define next_cpu(n, src)        ({ (void)(src); 1; })
-#endif
 
 #ifdef CONFIG_HAVE_CPUMASK_OF_CPU_MAP
 extern cpumask_t *cpumask_of_cpu_map;
-#define cpumask_of_cpu(cpu)    (cpumask_of_cpu_map[cpu])
-
+#define cpumask_of_cpu(cpu)     (cpumask_of_cpu_map[cpu])
+#define cpumask_of_cpu_ptr(v, cpu)                                      \
+                const cpumask_t *v = &cpumask_of_cpu(cpu)
+#define cpumask_of_cpu_ptr_declare(v)                                   \
+                const cpumask_t *v
+#define cpumask_of_cpu_ptr_next(v, cpu)                                 \
+                                        v = &cpumask_of_cpu(cpu)
 #else
 #define cpumask_of_cpu(cpu)                                             \
-(*({                                                                    \
+({                                                                      \
         typeof(_unused_cpumask_arg_) m;                                 \
         if (sizeof(m) == sizeof(unsigned long)) {                       \
                 m.bits[0] = 1UL<<(cpu);                                 \
@@ -240,8 +293,16 @@ extern cpumask_t *cpumask_of_cpu_map;
                 cpus_clear(m);                                          \
                 cpu_set((cpu), m);                                      \
         }                                                               \
-        &m;                                                             \
-}))
+        m;                                                              \
+})
+#define cpumask_of_cpu_ptr(v, cpu)                                      \
+                cpumask_t _##v = cpumask_of_cpu(cpu);                   \
+                const cpumask_t *v = &_##v
+#define cpumask_of_cpu_ptr_declare(v)                                   \
+                cpumask_t _##v;                                         \
+                const cpumask_t *v = &_##v
+#define cpumask_of_cpu_ptr_next(v, cpu)                                 \
+                                        _##v = cpumask_of_cpu(cpu)
 #endif
 
 #define CPU_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(NR_CPUS)
@@ -281,6 +342,15 @@ extern cpumask_t cpu_mask_all;
 
 #define cpus_addr(src) ((src).bits)
 
+#if NR_CPUS > BITS_PER_LONG
+#define CPUMASK_ALLOC(m)        struct m *m = kmalloc(sizeof(*m), GFP_KERNEL)
+#define CPUMASK_FREE(m)         kfree(m)
+#else
+#define CPUMASK_ALLOC(m)        struct m _m, *m = &_m
+#define CPUMASK_FREE(m)
+#endif
+#define CPUMASK_PTR(v, m)       cpumask_t *v = &(m->v)
+
 #define cpumask_scnprintf(buf, len, src) \
                         __cpumask_scnprintf((buf), (len), &(src), NR_CPUS)
 static inline int __cpumask_scnprintf(char *buf, int len,
@@ -343,20 +413,49 @@ static inline void __cpus_fold(cpumask_t *dstp, const cpumask_t *origp,
         bitmap_fold(dstp->bits, origp->bits, sz, nbits);
 }
 
-#if NR_CPUS > 1
-#define for_each_cpu_mask(cpu, mask)            \
-        for ((cpu) = first_cpu(mask);           \
-                (cpu) < NR_CPUS;                \
-                (cpu) = next_cpu((cpu), (mask)))
-#else /* NR_CPUS == 1 */
-#define for_each_cpu_mask(cpu, mask)            \
+#if NR_CPUS == 1
+
+#define nr_cpu_ids              1
+#define first_cpu(src)          ({ (void)(src); 0; })
+#define next_cpu(n, src)        ({ (void)(src); 1; })
+#define any_online_cpu(mask)    0
+#define for_each_cpu_mask(cpu, mask)    \
         for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
-#endif /* NR_CPUS */
+
+#else /* NR_CPUS > 1 */
+
+extern int nr_cpu_ids;
+int __first_cpu(const cpumask_t *srcp);
+int __next_cpu(int n, const cpumask_t *srcp);
+int __any_online_cpu(const cpumask_t *mask);
+
+#define first_cpu(src)          __first_cpu(&(src))
+#define next_cpu(n, src)        __next_cpu((n), &(src))
+#define any_online_cpu(mask) __any_online_cpu(&(mask))
+#define for_each_cpu_mask(cpu, mask)                    \
+        for ((cpu) = -1;                                \
+                (cpu) = next_cpu((cpu), (mask)),        \
+                (cpu) < NR_CPUS; )
+#endif
+
+#if NR_CPUS <= 64
 
 #define next_cpu_nr(n, src)             next_cpu(n, src)
 #define cpus_weight_nr(cpumask)         cpus_weight(cpumask)
 #define for_each_cpu_mask_nr(cpu, mask) for_each_cpu_mask(cpu, mask)
 
+#else /* NR_CPUS > 64 */
+
+int __next_cpu_nr(int n, const cpumask_t *srcp);
+#define next_cpu_nr(n, src)     __next_cpu_nr((n), &(src))
+#define cpus_weight_nr(cpumask) __cpus_weight(&(cpumask), nr_cpu_ids)
+#define for_each_cpu_mask_nr(cpu, mask)                 \
+        for ((cpu) = -1;                                \
+                (cpu) = next_cpu_nr((cpu), (mask)),     \
+                (cpu) < nr_cpu_ids; )
+
+#endif /* NR_CPUS > 64 */
+
 /*
  * The following particular system cpumasks and operations manage
  * possible, present and online cpus.  Each of them is a fixed size
@@ -418,9 +517,9 @@ extern cpumask_t cpu_online_map;
 extern cpumask_t cpu_present_map;
 
 #if NR_CPUS > 1
-#define num_online_cpus()       cpus_weight(cpu_online_map)
-#define num_possible_cpus()     cpus_weight(cpu_possible_map)
-#define num_present_cpus()      cpus_weight(cpu_present_map)
+#define num_online_cpus()       cpus_weight_nr(cpu_online_map)
+#define num_possible_cpus()     cpus_weight_nr(cpu_possible_map)
+#define num_present_cpus()      cpus_weight_nr(cpu_present_map)
 #define cpu_online(cpu)         cpu_isset((cpu), cpu_online_map)
 #define cpu_possible(cpu)       cpu_isset((cpu), cpu_possible_map)
 #define cpu_present(cpu)        cpu_isset((cpu), cpu_present_map)
@@ -435,17 +534,8 @@ extern cpumask_t cpu_present_map;
 
 #define cpu_is_offline(cpu)     unlikely(!cpu_online(cpu))
 
-#ifdef CONFIG_SMP
-extern int nr_cpu_ids;
-#define any_online_cpu(mask) __any_online_cpu(&(mask))
-int __any_online_cpu(const cpumask_t *mask);
-#else
-#define nr_cpu_ids                      1
-#define any_online_cpu(mask)            0
-#endif
-
-#define for_each_possible_cpu(cpu)  for_each_cpu_mask((cpu), cpu_possible_map)
-#define for_each_online_cpu(cpu)  for_each_cpu_mask((cpu), cpu_online_map)
-#define for_each_present_cpu(cpu) for_each_cpu_mask((cpu), cpu_present_map)
+#define for_each_possible_cpu(cpu) for_each_cpu_mask_nr((cpu), cpu_possible_map)
+#define for_each_online_cpu(cpu)   for_each_cpu_mask_nr((cpu), cpu_online_map)
+#define for_each_present_cpu(cpu)  for_each_cpu_mask_nr((cpu), cpu_present_map)
 
 #endif /* __LINUX_CPUMASK_H */