x86: fold pda into percpu area on SMP

[ Based on original patch from Christoph Lameter and Mike Travis. ]

Currently pdas and percpu areas are allocated separately.  %gs points
to local pda and percpu area can be reached using pda->data_offset.
This patch folds pda into percpu area.

Due to strange gcc requirement, pda needs to be at the beginning of
the percpu area so that pda->stack_canary is at %gs:40.  To achieve
this, a new percpu output section macro - PERCPU_VADDR_PREALLOC() - is
added and used to reserve pda sized chunk at the start of the percpu
area.

After this change, for boot cpu, %gs first points to pda in the
data.init area and later during setup_per_cpu_areas() gets updated to
point to the actual pda.  This means that setup_per_cpu_areas() need
to reload %gs for CPU0 while clearing pda area for other cpus as cpu0
already has modified it when control reaches setup_per_cpu_areas().

This patch also removes now unnecessary get_local_pda() and its call
sites.

A lot of this patch is taken from Mike Travis' "x86_64: Fold pda into
per cpu area" patch.

Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

1 files changed, 46 insertions, 61 deletions

diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index 73ab01b297c5..63d462802272 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -13,6 +13,7 @@
 #include <asm/mpspec.h>
 #include <asm/apicdef.h>
 #include <asm/highmem.h>
+#include <asm/proto.h>
 #include <asm/cpumask.h>
 
 #ifdef CONFIG_DEBUG_PER_CPU_MAPS
@@ -65,6 +66,36 @@ static void __init setup_node_to_cpumask_map(void);
 static inline void setup_node_to_cpumask_map(void) { }
 #endif
 
+#ifdef CONFIG_X86_64
+void __cpuinit load_pda_offset(int cpu)
+{
+        /* Memory clobbers used to order pda/percpu accesses */
+        mb();
+        wrmsrl(MSR_GS_BASE, cpu_pda(cpu));
+        mb();
+}
+
+#endif /* CONFIG_SMP && CONFIG_X86_64 */
+
+#ifdef CONFIG_X86_64
+
+/* correctly size the local cpu masks */
+static void setup_cpu_local_masks(void)
+{
+        alloc_bootmem_cpumask_var(&cpu_initialized_mask);
+        alloc_bootmem_cpumask_var(&cpu_callin_mask);
+        alloc_bootmem_cpumask_var(&cpu_callout_mask);
+        alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
+}
+
+#else /* CONFIG_X86_32 */
+
+static inline void setup_cpu_local_masks(void)
+{
+}
+
+#endif /* CONFIG_X86_32 */
+
 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
 /*
  * Copy data used in early init routines from the initial arrays to the
@@ -101,63 +132,7 @@ static void __init setup_per_cpu_maps(void)
  */
 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
 EXPORT_SYMBOL(__per_cpu_offset);
-static inline void setup_cpu_pda_map(void) { }
-
-#elif !defined(CONFIG_SMP)
-static inline void setup_cpu_pda_map(void) { }
-
-#else /* CONFIG_SMP && CONFIG_X86_64 */
-
-/*
- * Allocate cpu_pda pointer table and array via alloc_bootmem.
- */
-static void __init setup_cpu_pda_map(void)
-{
-        char *pda;
-        unsigned long size;
-        int cpu;
-
-        size = roundup(sizeof(struct x8664_pda), cache_line_size());
-
-        /* allocate cpu_pda array and pointer table */
-        {
-                unsigned long asize = size * (nr_cpu_ids - 1);
-
-                pda = alloc_bootmem(asize);
-        }
-
-        /* initialize pointer table to static pda's */
-        for_each_possible_cpu(cpu) {
-                if (cpu == 0) {
-                        /* leave boot cpu pda in place */
-                        continue;
-                }
-                cpu_pda(cpu) = (struct x8664_pda *)pda;
-                cpu_pda(cpu)->in_bootmem = 1;
-                pda += size;
-        }
-}
-
-#endif /* CONFIG_SMP && CONFIG_X86_64 */
-
-#ifdef CONFIG_X86_64
-
-/* correctly size the local cpu masks */
-static void setup_cpu_local_masks(void)
-{
-        alloc_bootmem_cpumask_var(&cpu_initialized_mask);
-        alloc_bootmem_cpumask_var(&cpu_callin_mask);
-        alloc_bootmem_cpumask_var(&cpu_callout_mask);
-        alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
-}
-
-#else /* CONFIG_X86_32 */
-
-static inline void setup_cpu_local_masks(void)
-{
-}
-
-#endif /* CONFIG_X86_32 */
+#endif
 
 /*
  * Great future plan:
@@ -171,9 +146,6 @@ void __init setup_per_cpu_areas(void)
         int cpu;
         unsigned long align = 1;
 
-        /* Setup cpu_pda map */
-        setup_cpu_pda_map();
-
         /* Copy section for each CPU (we discard the original) */
         old_size = PERCPU_ENOUGH_ROOM;
         align = max_t(unsigned long, PAGE_SIZE, align);
@@ -204,8 +176,21 @@ void __init setup_per_cpu_areas(void)
                                 cpu, node, __pa(ptr));
                 }
 #endif
-                per_cpu_offset(cpu) = ptr - __per_cpu_start;
+
                 memcpy(ptr, __per_cpu_load, __per_cpu_end - __per_cpu_start);
+#ifdef CONFIG_X86_64
+                cpu_pda(cpu) = (void *)ptr;
+
+                /*
+                 * CPU0 modified pda in the init data area, reload pda
+                 * offset for CPU0 and clear the area for others.
+                 */
+                if (cpu == 0)
+                        load_pda_offset(0);
+                else
+                        memset(cpu_pda(cpu), 0, sizeof(*cpu_pda(cpu)));
+#endif
+                per_cpu_offset(cpu) = ptr - __per_cpu_start;
 
                 DBG("PERCPU: cpu %4d %p\n", cpu, ptr);
         }