16 files changed, 269 insertions, 66 deletions

diff --git a/arch/Kconfig b/arch/Kconfig
index fe48fc7a3eba..53d7f619a1b9 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -158,4 +158,7 @@ config HAVE_PERF_EVENTS_NMI
           subsystem.  Also has support for calculating CPU cycle events
           to determine how many clock cycles in a given period.
 
+config HAVE_ARCH_JUMP_LABEL
+        bool
+
 source "kernel/gcov/Kconfig"
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index 491e9d6de191..9212cd42a832 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -30,6 +30,7 @@ config SPARC
         select PERF_USE_VMALLOC
         select HAVE_DMA_ATTRS
         select HAVE_DMA_API_DEBUG
+        select HAVE_ARCH_JUMP_LABEL
 
 config SPARC32
         def_bool !64BIT
diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h
new file mode 100644
index 000000000000..62e66d7b2fb6
--- /dev/null
+++ b/arch/sparc/include/asm/jump_label.h
@@ -0,0 +1,32 @@
+#ifndef _ASM_SPARC_JUMP_LABEL_H
+#define _ASM_SPARC_JUMP_LABEL_H
+
+#ifdef __KERNEL__
+
+#include <linux/types.h>
+#include <asm/system.h>
+
+#define JUMP_LABEL_NOP_SIZE 4
+
+#define JUMP_LABEL(key, label)                                  \
+        do {                                                    \
+                asm goto("1:\n\t"                               \
+                         "nop\n\t"                              \
+                         "nop\n\t"                              \
+                         ".pushsection __jump_table,  \"a\"\n\t"\
+                         ".word 1b, %l[" #label "], %c0\n\t"    \
+                         ".popsection \n\t"                     \
+                         : :  "i" (key) :  : label);\
+        } while (0)
+
+#endif /* __KERNEL__ */
+
+typedef u32 jump_label_t;
+
+struct jump_entry {
+        jump_label_t code;
+        jump_label_t target;
+        jump_label_t key;
+};
+
+#endif
diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile
index 0c2dc1f24a9a..599398fbbc7c 100644
--- a/arch/sparc/kernel/Makefile
+++ b/arch/sparc/kernel/Makefile
@@ -119,3 +119,5 @@ obj-$(CONFIG_COMPAT)    += $(audit--y)
 
 pc--$(CONFIG_PERF_EVENTS) := perf_event.o
 obj-$(CONFIG_SPARC64)   += $(pc--y)
+
+obj-$(CONFIG_SPARC64)   += jump_label.o
diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c
new file mode 100644
index 000000000000..ea2dafc93d78
--- /dev/null
+++ b/arch/sparc/kernel/jump_label.c
@@ -0,0 +1,47 @@
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/mutex.h>
+#include <linux/cpu.h>
+
+#include <linux/jump_label.h>
+#include <linux/memory.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+void arch_jump_label_transform(struct jump_entry *entry,
+                               enum jump_label_type type)
+{
+        u32 val;
+        u32 *insn = (u32 *) (unsigned long) entry->code;
+
+        if (type == JUMP_LABEL_ENABLE) {
+                s32 off = (s32)entry->target - (s32)entry->code;
+
+#ifdef CONFIG_SPARC64
+                /* ba,pt %xcc, . + (off << 2) */
+                val = 0x10680000 | ((u32) off >> 2);
+#else
+                /* ba . + (off << 2) */
+                val = 0x10800000 | ((u32) off >> 2);
+#endif
+        } else {
+                val = 0x01000000;
+        }
+
+        get_online_cpus();
+        mutex_lock(&text_mutex);
+        *insn = val;
+        flushi(insn);
+        mutex_unlock(&text_mutex);
+        put_online_cpus();
+}
+
+void arch_jump_label_text_poke_early(jump_label_t addr)
+{
+        u32 *insn_p = (u32 *) (unsigned long) addr;
+
+        *insn_p = 0x01000000;
+        flushi(insn_p);
+}
+
+#endif
diff --git a/arch/sparc/kernel/module.c b/arch/sparc/kernel/module.c
index f848aadf54dc..ee3c7dde8d9f 100644
--- a/arch/sparc/kernel/module.c
+++ b/arch/sparc/kernel/module.c
@@ -18,6 +18,9 @@
 #include <asm/spitfire.h>
 
 #ifdef CONFIG_SPARC64
+
+#include <linux/jump_label.h>
+
 static void *module_map(unsigned long size)
 {
         struct vm_struct *area;
@@ -227,6 +230,9 @@ int module_finalize(const Elf_Ehdr *hdr,
                     const Elf_Shdr *sechdrs,
                     struct module *me)
 {
+        /* make jump label nops */
+        jump_label_apply_nops(me);
+
         /* Cheetah's I-cache is fully coherent.  */
         if (tlb_type == spitfire) {
                 unsigned long va;
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index cea0cd9a316f..b431a0824a93 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -59,6 +59,7 @@ config X86
         select ANON_INODES
         select HAVE_ARCH_KMEMCHECK
         select HAVE_USER_RETURN_NOTIFIER
+        select HAVE_ARCH_JUMP_LABEL
 
 config INSTRUCTION_DECODER
         def_bool (KPROBES || PERF_EVENTS)
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index bc6abb7bc7ee..76561d20ea2f 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -4,6 +4,7 @@
 #include <linux/types.h>
 #include <linux/stddef.h>
 #include <linux/stringify.h>
+#include <linux/jump_label.h>
 #include <asm/asm.h>
 
 /*
@@ -160,6 +161,8 @@ static inline void apply_paravirt(struct paravirt_patch_site *start,
 #define __parainstructions_end  NULL
 #endif
 
+extern void *text_poke_early(void *addr, const void *opcode, size_t len);
+
 /*
  * Clear and restore the kernel write-protection flag on the local CPU.
  * Allows the kernel to edit read-only pages.
@@ -180,4 +183,12 @@ static inline void apply_paravirt(struct paravirt_patch_site *start,
 extern void *text_poke(void *addr, const void *opcode, size_t len);
 extern void *text_poke_smp(void *addr, const void *opcode, size_t len);
 
+#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)
+#define IDEAL_NOP_SIZE_5 5
+extern unsigned char ideal_nop5[IDEAL_NOP_SIZE_5];
+extern void arch_init_ideal_nop5(void);
+#else
+static inline void arch_init_ideal_nop5(void) {}
+#endif
+
 #endif /* _ASM_X86_ALTERNATIVE_H */
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
new file mode 100644
index 000000000000..f52d42e80585
--- /dev/null
+++ b/arch/x86/include/asm/jump_label.h
@@ -0,0 +1,37 @@
+#ifndef _ASM_X86_JUMP_LABEL_H
+#define _ASM_X86_JUMP_LABEL_H
+
+#ifdef __KERNEL__
+
+#include <linux/types.h>
+#include <asm/nops.h>
+
+#define JUMP_LABEL_NOP_SIZE 5
+
+# define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
+
+# define JUMP_LABEL(key, label)                                 \
+        do {                                                    \
+                asm goto("1:"                                   \
+                        JUMP_LABEL_INITIAL_NOP                  \
+                        ".pushsection __jump_table,  \"a\" \n\t"\
+                        _ASM_PTR "1b, %l[" #label "], %c0 \n\t" \
+                        ".popsection \n\t"                      \
+                        : :  "i" (key) :  : label);             \
+        } while (0)
+
+#endif /* __KERNEL__ */
+
+#ifdef CONFIG_X86_64
+typedef u64 jump_label_t;
+#else
+typedef u32 jump_label_t;
+#endif
+
+struct jump_entry {
+        jump_label_t code;
+        jump_label_t target;
+        jump_label_t key;
+};
+
+#endif
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 0925676266bd..24fa1718ddb9 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -32,7 +32,7 @@ GCOV_PROFILE_paravirt.o		:= n
 obj-y                   := process_$(BITS).o signal.o entry_$(BITS).o
 obj-y                   += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
 obj-y                   += time.o ioport.o ldt.o dumpstack.o
-obj-y                   += setup.o x86_init.o i8259.o irqinit.o
+obj-y                   += setup.o x86_init.o i8259.o irqinit.o jump_label.o
 obj-$(CONFIG_X86_VISWS) += visws_quirks.o
 obj-$(CONFIG_X86_32)    += probe_roms_32.o
 obj-$(CONFIG_X86_32)    += sys_i386_32.o i386_ksyms_32.o
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index f65ab8b014c4..cb0e6d385f6d 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -195,7 +195,7 @@ static void __init_or_module add_nops(void *insns, unsigned int len)
 
 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
 extern s32 __smp_locks[], __smp_locks_end[];
-static void *text_poke_early(void *addr, const void *opcode, size_t len);
+void *text_poke_early(void *addr, const void *opcode, size_t len);
 
 /* Replace instructions with better alternatives for this CPU type.
    This runs before SMP is initialized to avoid SMP problems with
@@ -522,7 +522,7 @@ void __init alternative_instructions(void)
  * instructions. And on the local CPU you need to be protected again NMI or MCE
  * handlers seeing an inconsistent instruction while you patch.
  */
-static void *__init_or_module text_poke_early(void *addr, const void *opcode,
+void *__init_or_module text_poke_early(void *addr, const void *opcode,
                                               size_t len)
 {
         unsigned long flags;
@@ -641,3 +641,67 @@ void *__kprobes text_poke_smp(void *addr, const void *opcode, size_t len)
         return addr;
 }
 
+#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)
+
+unsigned char ideal_nop5[IDEAL_NOP_SIZE_5];
+
+void __init arch_init_ideal_nop5(void)
+{
+        extern const unsigned char ftrace_test_p6nop[];
+        extern const unsigned char ftrace_test_nop5[];
+        extern const unsigned char ftrace_test_jmp[];
+        int faulted = 0;
+
+        /*
+         * There is no good nop for all x86 archs.
+         * We will default to using the P6_NOP5, but first we
+         * will test to make sure that the nop will actually
+         * work on this CPU. If it faults, we will then
+         * go to a lesser efficient 5 byte nop. If that fails
+         * we then just use a jmp as our nop. This isn't the most
+         * efficient nop, but we can not use a multi part nop
+         * since we would then risk being preempted in the middle
+         * of that nop, and if we enabled tracing then, it might
+         * cause a system crash.
+         *
+         * TODO: check the cpuid to determine the best nop.
+         */
+        asm volatile (
+                "ftrace_test_jmp:"
+                "jmp ftrace_test_p6nop\n"
+                "nop\n"
+                "nop\n"
+                "nop\n"  /* 2 byte jmp + 3 bytes */
+                "ftrace_test_p6nop:"
+                P6_NOP5
+                "jmp 1f\n"
+                "ftrace_test_nop5:"
+                ".byte 0x66,0x66,0x66,0x66,0x90\n"
+                "1:"
+                ".section .fixup, \"ax\"\n"
+                "2:     movl $1, %0\n"
+                "       jmp ftrace_test_nop5\n"
+                "3:     movl $2, %0\n"
+                "       jmp 1b\n"
+                ".previous\n"
+                _ASM_EXTABLE(ftrace_test_p6nop, 2b)
+                _ASM_EXTABLE(ftrace_test_nop5, 3b)
+                : "=r"(faulted) : "0" (faulted));
+
+        switch (faulted) {
+        case 0:
+                pr_info("converting mcount calls to 0f 1f 44 00 00\n");
+                memcpy(ideal_nop5, ftrace_test_p6nop, IDEAL_NOP_SIZE_5);
+                break;
+        case 1:
+                pr_info("converting mcount calls to 66 66 66 66 90\n");
+                memcpy(ideal_nop5, ftrace_test_nop5, IDEAL_NOP_SIZE_5);
+                break;
+        case 2:
+                pr_info("converting mcount calls to jmp . + 5\n");
+                memcpy(ideal_nop5, ftrace_test_jmp, IDEAL_NOP_SIZE_5);
+                break;
+        }
+
+}
+#endif
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index cd37469b54ee..3afb33f14d2d 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -257,14 +257,9 @@ do_ftrace_mod_code(unsigned long ip, void *new_code)
         return mod_code_status;
 }
 
-
-
-
-static unsigned char ftrace_nop[MCOUNT_INSN_SIZE];
-
 static unsigned char *ftrace_nop_replace(void)
 {
-        return ftrace_nop;
+        return ideal_nop5;
 }
 
 static int
@@ -338,62 +333,6 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
 
 int __init ftrace_dyn_arch_init(void *data)
 {
-        extern const unsigned char ftrace_test_p6nop[];
-        extern const unsigned char ftrace_test_nop5[];
-        extern const unsigned char ftrace_test_jmp[];
-        int faulted = 0;
-
-        /*
-         * There is no good nop for all x86 archs.
-         * We will default to using the P6_NOP5, but first we
-         * will test to make sure that the nop will actually
-         * work on this CPU. If it faults, we will then
-         * go to a lesser efficient 5 byte nop. If that fails
-         * we then just use a jmp as our nop. This isn't the most
-         * efficient nop, but we can not use a multi part nop
-         * since we would then risk being preempted in the middle
-         * of that nop, and if we enabled tracing then, it might
-         * cause a system crash.
-         *
-         * TODO: check the cpuid to determine the best nop.
-         */
-        asm volatile (
-                "ftrace_test_jmp:"
-                "jmp ftrace_test_p6nop\n"
-                "nop\n"
-                "nop\n"
-                "nop\n"  /* 2 byte jmp + 3 bytes */
-                "ftrace_test_p6nop:"
-                P6_NOP5
-                "jmp 1f\n"
-                "ftrace_test_nop5:"
-                ".byte 0x66,0x66,0x66,0x66,0x90\n"
-                "1:"
-                ".section .fixup, \"ax\"\n"
-                "2:     movl $1, %0\n"
-                "       jmp ftrace_test_nop5\n"
-                "3:     movl $2, %0\n"
-                "       jmp 1b\n"
-                ".previous\n"
-                _ASM_EXTABLE(ftrace_test_p6nop, 2b)
-                _ASM_EXTABLE(ftrace_test_nop5, 3b)
-                : "=r"(faulted) : "0" (faulted));
-
-        switch (faulted) {
-        case 0:
-                pr_info("converting mcount calls to 0f 1f 44 00 00\n");
-                memcpy(ftrace_nop, ftrace_test_p6nop, MCOUNT_INSN_SIZE);
-                break;
-        case 1:
-                pr_info("converting mcount calls to 66 66 66 66 90\n");
-                memcpy(ftrace_nop, ftrace_test_nop5, MCOUNT_INSN_SIZE);
-                break;
-        case 2:
-                pr_info("converting mcount calls to jmp . + 5\n");
-                memcpy(ftrace_nop, ftrace_test_jmp, MCOUNT_INSN_SIZE);
-                break;
-        }
-
         /* The return code is retured via data */
         *(unsigned long *)data = 0;
 
diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
new file mode 100644
index 000000000000..961b6b30ba90
--- /dev/null
+++ b/arch/x86/kernel/jump_label.c
@@ -0,0 +1,50 @@
+/*
+ * jump label x86 support
+ *
+ * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
+ *
+ */
+#include <linux/jump_label.h>
+#include <linux/memory.h>
+#include <linux/uaccess.h>
+#include <linux/module.h>
+#include <linux/list.h>
+#include <linux/jhash.h>
+#include <linux/cpu.h>
+#include <asm/kprobes.h>
+#include <asm/alternative.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+union jump_code_union {
+        char code[JUMP_LABEL_NOP_SIZE];
+        struct {
+                char jump;
+                int offset;
+        } __attribute__((packed));
+};
+
+void arch_jump_label_transform(struct jump_entry *entry,
+                               enum jump_label_type type)
+{
+        union jump_code_union code;
+
+        if (type == JUMP_LABEL_ENABLE) {
+                code.jump = 0xe9;
+                code.offset = entry->target -
+                                (entry->code + JUMP_LABEL_NOP_SIZE);
+        } else
+                memcpy(&code, ideal_nop5, JUMP_LABEL_NOP_SIZE);
+        get_online_cpus();
+        mutex_lock(&text_mutex);
+        text_poke_smp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
+        mutex_unlock(&text_mutex);
+        put_online_cpus();
+}
+
+void arch_jump_label_text_poke_early(jump_label_t addr)
+{
+        text_poke_early((void *)addr, ideal_nop5, JUMP_LABEL_NOP_SIZE);
+}
+
+#endif
diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes.c
index e05952af5d26..1cbd54c0df99 100644
--- a/arch/x86/kernel/kprobes.c
+++ b/arch/x86/kernel/kprobes.c
@@ -1218,7 +1218,8 @@ static int __kprobes copy_optimized_instructions(u8 *dest, u8 *src)
         }
         /* Check whether the address range is reserved */
         if (ftrace_text_reserved(src, src + len - 1) ||
-            alternatives_text_reserved(src, src + len - 1))
+            alternatives_text_reserved(src, src + len - 1) ||
+            jump_label_text_reserved(src, src + len - 1))
                 return -EBUSY;
 
         return len;
diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index e0bc186d7501..5399f58de7ed 100644
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -239,6 +239,9 @@ int module_finalize(const Elf_Ehdr *hdr,
                 apply_paravirt(pseg, pseg + para->sh_size);
         }
 
+        /* make jump label nops */
+        jump_label_apply_nops(me);
+
         return module_bug_finalize(hdr, sechdrs, me);
 }
 
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index c3a4fbb2b996..00e167870f71 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -112,6 +112,7 @@
 #include <asm/numa_64.h>
 #endif
 #include <asm/mce.h>
+#include <asm/alternative.h>
 
 /*
  * end_pfn only includes RAM, while max_pfn_mapped includes all e820 entries.
@@ -726,6 +727,7 @@ void __init setup_arch(char **cmdline_p)
 {
         int acpi = 0;
         int k8 = 0;
+        unsigned long flags;
 
 #ifdef CONFIG_X86_32
         memcpy(&boot_cpu_data, &new_cpu_data, sizeof(new_cpu_data));
@@ -1071,6 +1073,10 @@ void __init setup_arch(char **cmdline_p)
         x86_init.oem.banner();
 
         mcheck_init();
+
+        local_irq_save(flags);
+        arch_init_ideal_nop5();
+        local_irq_restore(flags);
 }
 
 #ifdef CONFIG_X86_32