[PATCH] x86_64: x86_64 version of the smp alternative patch.

Changes are largely identical to the i386 version:

 * alternative #define are moved to the new alternative.h file.
 * one new elf section with pointers to the lock prefixes which can be
   nop'ed out for non-smp.
 * two new elf sections simliar to the "classic" alternatives to
   replace SMP code with simpler UP code.
 * fixup headers to use alternative.h instead of defining their own
   LOCK / LOCK_PREFIX macros.

The patch reuses the i386 version of the alternatives code to avoid code
duplication.  The code in alternatives.c was shuffled around a bit to
reduce the number of #ifdefs needed.  It also got some tweaks needed for
x86_64 (vsyscall page handling) and new features (noreplacement option
which was x86_64 only up to now).  Debug printk's are changed from
compile-time to runtime.

Loosely based on a early version from Bastian Blank <waldi@debian.org>

Signed-off-by: Gerd Hoffmann <kraxel@suse.de>
Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

7 files changed, 152 insertions, 135 deletions

diff --git a/arch/i386/kernel/alternative.c b/arch/i386/kernel/alternative.c
index 5cbd6f99fb2a..50eb0e03777e 100644
--- a/arch/i386/kernel/alternative.c
+++ b/arch/i386/kernel/alternative.c
@@ -4,27 +4,41 @@
 #include <asm/alternative.h>
 #include <asm/sections.h>
 
-#define DEBUG 0
-#if DEBUG
-# define DPRINTK(fmt, args...) printk(fmt, args)
-#else
-# define DPRINTK(fmt, args...)
-#endif
+static int no_replacement    = 0;
+static int smp_alt_once      = 0;
+static int debug_alternative = 0;
+
+static int __init noreplacement_setup(char *s)
+{
+        no_replacement = 1;
+        return 1;
+}
+static int __init bootonly(char *str)
+{
+        smp_alt_once = 1;
+        return 1;
+}
+static int __init debug_alt(char *str)
+{
+        debug_alternative = 1;
+        return 1;
+}
 
+__setup("noreplacement", noreplacement_setup);
+__setup("smp-alt-boot", bootonly);
+__setup("debug-alternative", debug_alt);
+
+#define DPRINTK(fmt, args...) if (debug_alternative) \
+        printk(KERN_DEBUG fmt, args)
+
+#ifdef GENERIC_NOP1
 /* Use inline assembly to define this because the nops are defined
    as inline assembly strings in the include files and we cannot
    get them easily into strings. */
 asm("\t.data\nintelnops: "
         GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
         GENERIC_NOP7 GENERIC_NOP8);
-asm("\t.data\nk8nops: "
-        K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
-        K8_NOP7 K8_NOP8);
-asm("\t.data\nk7nops: "
-        K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
-        K7_NOP7 K7_NOP8);
-
-extern unsigned char intelnops[], k8nops[], k7nops[];
+extern unsigned char intelnops[];
 static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
         NULL,
         intelnops,
@@ -36,6 +50,13 @@ static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
         intelnops + 1 + 2 + 3 + 4 + 5 + 6,
         intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
 };
+#endif
+
+#ifdef K8_NOP1
+asm("\t.data\nk8nops: "
+        K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
+        K8_NOP7 K8_NOP8);
+extern unsigned char k8nops[];
 static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
         NULL,
         k8nops,
@@ -47,6 +68,13 @@ static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
         k8nops + 1 + 2 + 3 + 4 + 5 + 6,
         k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
 };
+#endif
+
+#ifdef K7_NOP1
+asm("\t.data\nk7nops: "
+        K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
+        K7_NOP7 K7_NOP8);
+extern unsigned char k7nops[];
 static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
         NULL,
         k7nops,
@@ -58,6 +86,18 @@ static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
         k7nops + 1 + 2 + 3 + 4 + 5 + 6,
         k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
 };
+#endif
+
+#ifdef CONFIG_X86_64
+
+extern char __vsyscall_0;
+static inline unsigned char** find_nop_table(void)
+{
+        return k8_nops;
+}
+
+#else /* CONFIG_X86_64 */
+
 static struct nop {
         int cpuid;
         unsigned char **noptable;
@@ -67,14 +107,6 @@ static struct nop {
         { -1, NULL }
 };
 
-
-extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
-extern struct alt_instr __smp_alt_instructions[], __smp_alt_instructions_end[];
-extern u8 *__smp_locks[], *__smp_locks_end[];
-
-extern u8 __smp_alt_begin[], __smp_alt_end[];
-
-
 static unsigned char** find_nop_table(void)
 {
         unsigned char **noptable = intel_nops;
@@ -89,6 +121,14 @@ static unsigned char** find_nop_table(void)
         return noptable;
 }
 
+#endif /* CONFIG_X86_64 */
+
+extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
+extern struct alt_instr __smp_alt_instructions[], __smp_alt_instructions_end[];
+extern u8 *__smp_locks[], *__smp_locks_end[];
+
+extern u8 __smp_alt_begin[], __smp_alt_end[];
+
 /* Replace instructions with better alternatives for this CPU type.
    This runs before SMP is initialized to avoid SMP problems with
    self modifying code. This implies that assymetric systems where
@@ -99,6 +139,7 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
 {
         unsigned char **noptable = find_nop_table();
         struct alt_instr *a;
+        u8 *instr;
         int diff, i, k;
 
         DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
@@ -106,7 +147,16 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
                 BUG_ON(a->replacementlen > a->instrlen);
                 if (!boot_cpu_has(a->cpuid))
                         continue;
-                memcpy(a->instr, a->replacement, a->replacementlen);
+                instr = a->instr;
+#ifdef CONFIG_X86_64
+                /* vsyscall code is not mapped yet. resolve it manually. */
+                if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
+                        instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
+                        DPRINTK("%s: vsyscall fixup: %p => %p\n",
+                                __FUNCTION__, a->instr, instr);
+                }
+#endif
+                memcpy(instr, a->replacement, a->replacementlen);
                 diff = a->instrlen - a->replacementlen;
                 /* Pad the rest with nops */
                 for (i = a->replacementlen; diff > 0; diff -= k, i += k) {
@@ -186,14 +236,6 @@ struct smp_alt_module {
 static LIST_HEAD(smp_alt_modules);
 static DEFINE_SPINLOCK(smp_alt);
 
-static int smp_alt_once = 0;
-static int __init bootonly(char *str)
-{
-        smp_alt_once = 1;
-        return 1;
-}
-__setup("smp-alt-boot", bootonly);
-
 void alternatives_smp_module_add(struct module *mod, char *name,
                                  void *locks, void *locks_end,
                                  void *text,  void *text_end)
@@ -201,6 +243,9 @@ void alternatives_smp_module_add(struct module *mod, char *name,
         struct smp_alt_module *smp;
         unsigned long flags;
 
+        if (no_replacement)
+                return;
+
         if (smp_alt_once) {
                 if (boot_cpu_has(X86_FEATURE_UP))
                         alternatives_smp_unlock(locks, locks_end,
@@ -235,7 +280,7 @@ void alternatives_smp_module_del(struct module *mod)
         struct smp_alt_module *item;
         unsigned long flags;
 
-        if (smp_alt_once)
+        if (no_replacement || smp_alt_once)
                 return;
 
         spin_lock_irqsave(&smp_alt, flags);
@@ -256,7 +301,7 @@ void alternatives_smp_switch(int smp)
         struct smp_alt_module *mod;
         unsigned long flags;
 
-        if (smp_alt_once)
+        if (no_replacement || smp_alt_once)
                 return;
         BUG_ON(!smp && (num_online_cpus() > 1));
 
@@ -285,6 +330,13 @@ void alternatives_smp_switch(int smp)
 
 void __init alternative_instructions(void)
 {
+        if (no_replacement) {
+                printk(KERN_INFO "(SMP-)alternatives turned off\n");
+                free_init_pages("SMP alternatives",
+                                (unsigned long)__smp_alt_begin,
+                                (unsigned long)__smp_alt_end);
+                return;
+        }
         apply_alternatives(__alt_instructions, __alt_instructions_end);
 
         /* switch to patch-once-at-boottime-only mode and free the
diff --git a/arch/x86_64/kernel/Makefile b/arch/x86_64/kernel/Makefile
index 059c88313f4e..381bc6ad743e 100644
--- a/arch/x86_64/kernel/Makefile
+++ b/arch/x86_64/kernel/Makefile
@@ -8,7 +8,7 @@ obj-y	:= process.o signal.o entry.o traps.o irq.o \
                 ptrace.o time.o ioport.o ldt.o setup.o i8259.o sys_x86_64.o \
                 x8664_ksyms.o i387.o syscall.o vsyscall.o \
                 setup64.o bootflag.o e820.o reboot.o quirks.o i8237.o \
-                pci-dma.o pci-nommu.o
+                pci-dma.o pci-nommu.o alternative.o
 
 obj-$(CONFIG_X86_MCE)         += mce.o
 obj-$(CONFIG_X86_MCE_INTEL)     += mce_intel.o
@@ -49,3 +49,5 @@ intel_cacheinfo-y		+= ../../i386/kernel/cpu/intel_cacheinfo.o
 quirks-y                        += ../../i386/kernel/quirks.o
 i8237-y                         += ../../i386/kernel/i8237.o
 msr-$(subst m,y,$(CONFIG_X86_MSR))  += ../../i386/kernel/msr.o
+alternative-y                   += ../../i386/kernel/alternative.o
+
diff --git a/arch/x86_64/kernel/module.c b/arch/x86_64/kernel/module.c
index bac195c74bcc..9d0958ff547f 100644
--- a/arch/x86_64/kernel/module.c
+++ b/arch/x86_64/kernel/module.c
@@ -145,26 +145,38 @@ int apply_relocate(Elf_Shdr *sechdrs,
         return -ENOSYS;
 } 
 
-extern void apply_alternatives(void *start, void *end); 
-
 int module_finalize(const Elf_Ehdr *hdr,
-                    const Elf_Shdr *sechdrs,
-                    struct module *me)
+                    const Elf_Shdr *sechdrs,
+                    struct module *me)
 {
-        const Elf_Shdr *s;
+        const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL;
         char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
 
-        /* look for .altinstructions to patch */ 
-        for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { 
-                void *seg;              
-                if (strcmp(".altinstructions", secstrings + s->sh_name))
-                        continue;
-                seg = (void *)s->sh_addr; 
-                apply_alternatives(seg, seg + s->sh_size); 
-        }       
+        for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
+                if (!strcmp(".text", secstrings + s->sh_name))
+                        text = s;
+                if (!strcmp(".altinstructions", secstrings + s->sh_name))
+                        alt = s;
+                if (!strcmp(".smp_locks", secstrings + s->sh_name))
+                        locks= s;
+        }
+
+        if (alt) {
+                /* patch .altinstructions */
+                void *aseg = (void *)alt->sh_addr;
+                apply_alternatives(aseg, aseg + alt->sh_size);
+        }
+        if (locks && text) {
+                void *lseg = (void *)locks->sh_addr;
+                void *tseg = (void *)text->sh_addr;
+                alternatives_smp_module_add(me, me->name,
+                                            lseg, lseg + locks->sh_size,
+                                            tseg, tseg + text->sh_size);
+        }
         return 0;
 }
 
 void module_arch_cleanup(struct module *mod)
 {
+        alternatives_smp_module_del(mod);
 }
diff --git a/arch/x86_64/kernel/setup.c b/arch/x86_64/kernel/setup.c
index 4b7e02216970..64640c8f5eed 100644
--- a/arch/x86_64/kernel/setup.c
+++ b/arch/x86_64/kernel/setup.c
@@ -473,80 +473,6 @@ contig_initmem_init(unsigned long start_pfn, unsigned long end_pfn)
 } 
 #endif
 
-/* Use inline assembly to define this because the nops are defined 
-   as inline assembly strings in the include files and we cannot 
-   get them easily into strings. */
-asm("\t.data\nk8nops: " 
-    K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
-    K8_NOP7 K8_NOP8); 
-    
-extern unsigned char k8nops[];
-static unsigned char *k8_nops[ASM_NOP_MAX+1] = { 
-     NULL,
-     k8nops,
-     k8nops + 1,
-     k8nops + 1 + 2,
-     k8nops + 1 + 2 + 3,
-     k8nops + 1 + 2 + 3 + 4,
-     k8nops + 1 + 2 + 3 + 4 + 5,
-     k8nops + 1 + 2 + 3 + 4 + 5 + 6,
-     k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
-}; 
-
-extern char __vsyscall_0;
-
-/* Replace instructions with better alternatives for this CPU type.
-
-   This runs before SMP is initialized to avoid SMP problems with
-   self modifying code. This implies that assymetric systems where
-   APs have less capabilities than the boot processor are not handled. 
-   In this case boot with "noreplacement". */ 
-void apply_alternatives(void *start, void *end) 
-{ 
-        struct alt_instr *a; 
-        int diff, i, k;
-        for (a = start; (void *)a < end; a++) { 
-                u8 *instr;
-
-                if (!boot_cpu_has(a->cpuid))
-                        continue;
-
-                BUG_ON(a->replacementlen > a->instrlen); 
-                instr = a->instr;
-                /* vsyscall code is not mapped yet. resolve it manually. */
-                if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END)
-                        instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
-                __inline_memcpy(instr, a->replacement, a->replacementlen);
-                diff = a->instrlen - a->replacementlen; 
-
-                /* Pad the rest with nops */
-                for (i = a->replacementlen; diff > 0; diff -= k, i += k) {
-                        k = diff;
-                        if (k > ASM_NOP_MAX)
-                                k = ASM_NOP_MAX;
-                        __inline_memcpy(instr + i, k8_nops[k], k);
-                } 
-        }
-} 
-
-static int no_replacement __initdata = 0; 
- 
-void __init alternative_instructions(void)
-{
-        extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
-        if (no_replacement) 
-                return;
-        apply_alternatives(__alt_instructions, __alt_instructions_end);
-}
-
-static int __init noreplacement_setup(char *s)
-{ 
-     no_replacement = 1; 
-     return 1;
-} 
-
-__setup("noreplacement", noreplacement_setup); 
-
 #if defined(CONFIG_EDD) || defined(CONFIG_EDD_MODULE)
 struct edd edd;
 #ifdef CONFIG_EDD_MODULE
@@ -1303,7 +1229,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
                 /* Other (Linux-defined) */
                 "cxmmx", NULL, "cyrix_arr", "centaur_mcr", NULL,
                 "constant_tsc", NULL, NULL,
-                NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+                "up", NULL, NULL, NULL, NULL, NULL, NULL, NULL,
                 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
                 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
 
diff --git a/arch/x86_64/kernel/smpboot.c b/arch/x86_64/kernel/smpboot.c
index 71a7222cf9ce..06535e7687ce 100644
--- a/arch/x86_64/kernel/smpboot.c
+++ b/arch/x86_64/kernel/smpboot.c
@@ -797,6 +797,8 @@ static int __cpuinit do_boot_cpu(int cpu, int apicid)
         }
 
 
+        alternatives_smp_switch(1);
+
         c_idle.idle = get_idle_for_cpu(cpu);
 
         if (c_idle.idle) {
@@ -1259,6 +1261,8 @@ void __cpu_die(unsigned int cpu)
                 /* They ack this in play_dead by setting CPU_DEAD */
                 if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
                         printk ("CPU %d is now offline\n", cpu);
+                        if (1 == num_online_cpus())
+                                alternatives_smp_switch(0);
                         return;
                 }
                 msleep(100);
diff --git a/arch/x86_64/kernel/vmlinux.lds.S b/arch/x86_64/kernel/vmlinux.lds.S
index b81f473c4a19..5968c2415da9 100644
--- a/arch/x86_64/kernel/vmlinux.lds.S
+++ b/arch/x86_64/kernel/vmlinux.lds.S
@@ -131,6 +131,26 @@ SECTIONS
         *(.data.page_aligned)
   }
 
+  /* might get freed after init */
+  . = ALIGN(4096);
+  __smp_alt_begin = .;
+  __smp_alt_instructions = .;
+  .smp_altinstructions : AT(ADDR(.smp_altinstructions) - LOAD_OFFSET) {
+        *(.smp_altinstructions)
+  }
+  __smp_alt_instructions_end = .;
+  . = ALIGN(8);
+  __smp_locks = .;
+  .smp_locks : AT(ADDR(.smp_locks) - LOAD_OFFSET) {
+        *(.smp_locks)
+  }
+  __smp_locks_end = .;
+  .smp_altinstr_replacement : AT(ADDR(.smp_altinstr_replacement) - LOAD_OFFSET) {
+        *(.smp_altinstr_replacement)
+  }
+  . = ALIGN(4096);
+  __smp_alt_end = .;
+
   . = ALIGN(4096);              /* Init code and data */
   __init_begin = .;
   .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) {
diff --git a/arch/x86_64/mm/init.c b/arch/x86_64/mm/init.c
index 4ba34e95d835..a70a8e3e312d 100644
--- a/arch/x86_64/mm/init.c
+++ b/arch/x86_64/mm/init.c
@@ -644,20 +644,29 @@ void __init mem_init(void)
 #endif
 }
 
-void free_initmem(void)
+void free_init_pages(char *what, unsigned long begin, unsigned long end)
 {
         unsigned long addr;
 
-        addr = (unsigned long)(&__init_begin);
-        for (; addr < (unsigned long)(&__init_end); addr += PAGE_SIZE) {
+        if (begin >= end)
+                return;
+
+        printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
+        for (addr = begin; addr < end; addr += PAGE_SIZE) {
                 ClearPageReserved(virt_to_page(addr));
                 init_page_count(virt_to_page(addr));
                 memset((void *)(addr & ~(PAGE_SIZE-1)), 0xcc, PAGE_SIZE); 
                 free_page(addr);
                 totalram_pages++;
         }
+}
+
+void free_initmem(void)
+{
         memset(__initdata_begin, 0xba, __initdata_end - __initdata_begin);
-        printk ("Freeing unused kernel memory: %luk freed\n", (__init_end - __init_begin) >> 10);
+        free_init_pages("unused kernel memory",
+                        (unsigned long)(&__init_begin),
+                        (unsigned long)(&__init_end));
 }
 
 #ifdef CONFIG_DEBUG_RODATA
@@ -686,15 +695,7 @@ void mark_rodata_ro(void)
 #ifdef CONFIG_BLK_DEV_INITRD
 void free_initrd_mem(unsigned long start, unsigned long end)
 {
-        if (start >= end)
-                return;
-        printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
-        for (; start < end; start += PAGE_SIZE) {
-                ClearPageReserved(virt_to_page(start));
-                init_page_count(virt_to_page(start));
-                free_page(start);
-                totalram_pages++;
-        }
+        free_init_pages("initrd memory", start, end);
 }
 #endif