6 files changed, 456 insertions, 9 deletions

diff --git a/kernel/Makefile b/kernel/Makefile
index 0b72d1a74be0..d52b473c99a1 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -10,7 +10,7 @@ obj-y     = sched.o fork.o exec_domain.o panic.o printk.o \
             kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \
             hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
             notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o \
-            async.o range.o
+            async.o range.o jump_label.o
 obj-$(CONFIG_HAVE_EARLY_RES) += early_res.o
 obj-y += groups.o
 
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
new file mode 100644
index 000000000000..7be868bf25c6
--- /dev/null
+++ b/kernel/jump_label.c
@@ -0,0 +1,429 @@
+/*
+ * jump label support
+ *
+ * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
+ *
+ */
+#include <linux/jump_label.h>
+#include <linux/memory.h>
+#include <linux/uaccess.h>
+#include <linux/module.h>
+#include <linux/list.h>
+#include <linux/jhash.h>
+#include <linux/slab.h>
+#include <linux/sort.h>
+#include <linux/err.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+#define JUMP_LABEL_HASH_BITS 6
+#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
+static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];
+
+/* mutex to protect coming/going of the the jump_label table */
+static DEFINE_MUTEX(jump_label_mutex);
+
+struct jump_label_entry {
+        struct hlist_node hlist;
+        struct jump_entry *table;
+        int nr_entries;
+        /* hang modules off here */
+        struct hlist_head modules;
+        unsigned long key;
+};
+
+struct jump_label_module_entry {
+        struct hlist_node hlist;
+        struct jump_entry *table;
+        int nr_entries;
+        struct module *mod;
+};
+
+static int jump_label_cmp(const void *a, const void *b)
+{
+        const struct jump_entry *jea = a;
+        const struct jump_entry *jeb = b;
+
+        if (jea->key < jeb->key)
+                return -1;
+
+        if (jea->key > jeb->key)
+                return 1;
+
+        return 0;
+}
+
+static void
+sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
+{
+        unsigned long size;
+
+        size = (((unsigned long)stop - (unsigned long)start)
+                                        / sizeof(struct jump_entry));
+        sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
+}
+
+static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
+{
+        struct hlist_head *head;
+        struct hlist_node *node;
+        struct jump_label_entry *e;
+        u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
+
+        head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
+        hlist_for_each_entry(e, node, head, hlist) {
+                if (key == e->key)
+                        return e;
+        }
+        return NULL;
+}
+
+static struct jump_label_entry *
+add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
+{
+        struct hlist_head *head;
+        struct jump_label_entry *e;
+        u32 hash;
+
+        e = get_jump_label_entry(key);
+        if (e)
+                return ERR_PTR(-EEXIST);
+
+        e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);
+        if (!e)
+                return ERR_PTR(-ENOMEM);
+
+        hash = jhash((void *)&key, sizeof(jump_label_t), 0);
+        head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
+        e->key = key;
+        e->table = table;
+        e->nr_entries = nr_entries;
+        INIT_HLIST_HEAD(&(e->modules));
+        hlist_add_head(&e->hlist, head);
+        return e;
+}
+
+static int
+build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop)
+{
+        struct jump_entry *iter, *iter_begin;
+        struct jump_label_entry *entry;
+        int count;
+
+        sort_jump_label_entries(start, stop);
+        iter = start;
+        while (iter < stop) {
+                entry = get_jump_label_entry(iter->key);
+                if (!entry) {
+                        iter_begin = iter;
+                        count = 0;
+                        while ((iter < stop) &&
+                                (iter->key == iter_begin->key)) {
+                                iter++;
+                                count++;
+                        }
+                        entry = add_jump_label_entry(iter_begin->key,
+                                                        count, iter_begin);
+                        if (IS_ERR(entry))
+                                return PTR_ERR(entry);
+                 } else {
+                        WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
+                        return -1;
+                }
+        }
+        return 0;
+}
+
+/***
+ * jump_label_update - update jump label text
+ * @key -  key value associated with a a jump label
+ * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE
+ *
+ * Will enable/disable the jump for jump label @key, depending on the
+ * value of @type.
+ *
+ */
+
+void jump_label_update(unsigned long key, enum jump_label_type type)
+{
+        struct jump_entry *iter;
+        struct jump_label_entry *entry;
+        struct hlist_node *module_node;
+        struct jump_label_module_entry *e_module;
+        int count;
+
+        mutex_lock(&jump_label_mutex);
+        entry = get_jump_label_entry((jump_label_t)key);
+        if (entry) {
+                count = entry->nr_entries;
+                iter = entry->table;
+                while (count--) {
+                        if (kernel_text_address(iter->code))
+                                arch_jump_label_transform(iter, type);
+                        iter++;
+                }
+                /* eanble/disable jump labels in modules */
+                hlist_for_each_entry(e_module, module_node, &(entry->modules),
+                                                        hlist) {
+                        count = e_module->nr_entries;
+                        iter = e_module->table;
+                        while (count--) {
+                                if (kernel_text_address(iter->code))
+                                        arch_jump_label_transform(iter, type);
+                                iter++;
+                        }
+                }
+        }
+        mutex_unlock(&jump_label_mutex);
+}
+
+static int addr_conflict(struct jump_entry *entry, void *start, void *end)
+{
+        if (entry->code <= (unsigned long)end &&
+                entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
+                return 1;
+
+        return 0;
+}
+
+#ifdef CONFIG_MODULES
+
+static int module_conflict(void *start, void *end)
+{
+        struct hlist_head *head;
+        struct hlist_node *node, *node_next, *module_node, *module_node_next;
+        struct jump_label_entry *e;
+        struct jump_label_module_entry *e_module;
+        struct jump_entry *iter;
+        int i, count;
+        int conflict = 0;
+
+        for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
+                head = &jump_label_table[i];
+                hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
+                        hlist_for_each_entry_safe(e_module, module_node,
+                                                        module_node_next,
+                                                        &(e->modules), hlist) {
+                                count = e_module->nr_entries;
+                                iter = e_module->table;
+                                while (count--) {
+                                        if (addr_conflict(iter, start, end)) {
+                                                conflict = 1;
+                                                goto out;
+                                        }
+                                        iter++;
+                                }
+                        }
+                }
+        }
+out:
+        return conflict;
+}
+
+#endif
+
+/***
+ * jump_label_text_reserved - check if addr range is reserved
+ * @start: start text addr
+ * @end: end text addr
+ *
+ * checks if the text addr located between @start and @end
+ * overlaps with any of the jump label patch addresses. Code
+ * that wants to modify kernel text should first verify that
+ * it does not overlap with any of the jump label addresses.
+ *
+ * returns 1 if there is an overlap, 0 otherwise
+ */
+int jump_label_text_reserved(void *start, void *end)
+{
+        struct jump_entry *iter;
+        struct jump_entry *iter_start = __start___jump_table;
+        struct jump_entry *iter_stop = __start___jump_table;
+        int conflict = 0;
+
+        mutex_lock(&jump_label_mutex);
+        iter = iter_start;
+        while (iter < iter_stop) {
+                if (addr_conflict(iter, start, end)) {
+                        conflict = 1;
+                        goto out;
+                }
+                iter++;
+        }
+
+        /* now check modules */
+#ifdef CONFIG_MODULES
+        conflict = module_conflict(start, end);
+#endif
+out:
+        mutex_unlock(&jump_label_mutex);
+        return conflict;
+}
+
+static __init int init_jump_label(void)
+{
+        int ret;
+        struct jump_entry *iter_start = __start___jump_table;
+        struct jump_entry *iter_stop = __stop___jump_table;
+        struct jump_entry *iter;
+
+        mutex_lock(&jump_label_mutex);
+        ret = build_jump_label_hashtable(__start___jump_table,
+                                         __stop___jump_table);
+        iter = iter_start;
+        while (iter < iter_stop) {
+                arch_jump_label_text_poke_early(iter->code);
+                iter++;
+        }
+        mutex_unlock(&jump_label_mutex);
+        return ret;
+}
+early_initcall(init_jump_label);
+
+#ifdef CONFIG_MODULES
+
+static struct jump_label_module_entry *
+add_jump_label_module_entry(struct jump_label_entry *entry,
+                            struct jump_entry *iter_begin,
+                            int count, struct module *mod)
+{
+        struct jump_label_module_entry *e;
+
+        e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);
+        if (!e)
+                return ERR_PTR(-ENOMEM);
+        e->mod = mod;
+        e->nr_entries = count;
+        e->table = iter_begin;
+        hlist_add_head(&e->hlist, &entry->modules);
+        return e;
+}
+
+static int add_jump_label_module(struct module *mod)
+{
+        struct jump_entry *iter, *iter_begin;
+        struct jump_label_entry *entry;
+        struct jump_label_module_entry *module_entry;
+        int count;
+
+        /* if the module doesn't have jump label entries, just return */
+        if (!mod->num_jump_entries)
+                return 0;
+
+        sort_jump_label_entries(mod->jump_entries,
+                                mod->jump_entries + mod->num_jump_entries);
+        iter = mod->jump_entries;
+        while (iter < mod->jump_entries + mod->num_jump_entries) {
+                entry = get_jump_label_entry(iter->key);
+                iter_begin = iter;
+                count = 0;
+                while ((iter < mod->jump_entries + mod->num_jump_entries) &&
+                        (iter->key == iter_begin->key)) {
+                                iter++;
+                                count++;
+                }
+                if (!entry) {
+                        entry = add_jump_label_entry(iter_begin->key, 0, NULL);
+                        if (IS_ERR(entry))
+                                return PTR_ERR(entry);
+                }
+                module_entry = add_jump_label_module_entry(entry, iter_begin,
+                                                           count, mod);
+                if (IS_ERR(module_entry))
+                        return PTR_ERR(module_entry);
+        }
+        return 0;
+}
+
+static void remove_jump_label_module(struct module *mod)
+{
+        struct hlist_head *head;
+        struct hlist_node *node, *node_next, *module_node, *module_node_next;
+        struct jump_label_entry *e;
+        struct jump_label_module_entry *e_module;
+        int i;
+
+        /* if the module doesn't have jump label entries, just return */
+        if (!mod->num_jump_entries)
+                return;
+
+        for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
+                head = &jump_label_table[i];
+                hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
+                        hlist_for_each_entry_safe(e_module, module_node,
+                                                  module_node_next,
+                                                  &(e->modules), hlist) {
+                                if (e_module->mod == mod) {
+                                        hlist_del(&e_module->hlist);
+                                        kfree(e_module);
+                                }
+                        }
+                        if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
+                                hlist_del(&e->hlist);
+                                kfree(e);
+                        }
+                }
+        }
+}
+
+static int
+jump_label_module_notify(struct notifier_block *self, unsigned long val,
+                         void *data)
+{
+        struct module *mod = data;
+        int ret = 0;
+
+        switch (val) {
+        case MODULE_STATE_COMING:
+                mutex_lock(&jump_label_mutex);
+                ret = add_jump_label_module(mod);
+                if (ret)
+                        remove_jump_label_module(mod);
+                mutex_unlock(&jump_label_mutex);
+                break;
+        case MODULE_STATE_GOING:
+                mutex_lock(&jump_label_mutex);
+                remove_jump_label_module(mod);
+                mutex_unlock(&jump_label_mutex);
+                break;
+        }
+        return ret;
+}
+
+/***
+ * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
+ * @mod: module to patch
+ *
+ * Allow for run-time selection of the optimal nops. Before the module
+ * loads patch these with arch_get_jump_label_nop(), which is specified by
+ * the arch specific jump label code.
+ */
+void jump_label_apply_nops(struct module *mod)
+{
+        struct jump_entry *iter;
+
+        /* if the module doesn't have jump label entries, just return */
+        if (!mod->num_jump_entries)
+                return;
+
+        iter = mod->jump_entries;
+        while (iter < mod->jump_entries + mod->num_jump_entries) {
+                arch_jump_label_text_poke_early(iter->code);
+                iter++;
+        }
+}
+
+struct notifier_block jump_label_module_nb = {
+        .notifier_call = jump_label_module_notify,
+        .priority = 0,
+};
+
+static __init int init_jump_label_module(void)
+{
+        return register_module_notifier(&jump_label_module_nb);
+}
+early_initcall(init_jump_label_module);
+
+#endif /* CONFIG_MODULES */
+
+#endif
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 6dd5359e1f0e..ec4210c6501e 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -47,6 +47,7 @@
 #include <linux/memory.h>
 #include <linux/ftrace.h>
 #include <linux/cpu.h>
+#include <linux/jump_label.h>
 
 #include <asm-generic/sections.h>
 #include <asm/cacheflush.h>
@@ -1146,7 +1147,8 @@ int __kprobes register_kprobe(struct kprobe *p)
         preempt_disable();
         if (!kernel_text_address((unsigned long) p->addr) ||
             in_kprobes_functions((unsigned long) p->addr) ||
-            ftrace_text_reserved(p->addr, p->addr)) {
+            ftrace_text_reserved(p->addr, p->addr) ||
+            jump_label_text_reserved(p->addr, p->addr)) {
                 preempt_enable();
                 return -EINVAL;
         }
diff --git a/kernel/module.c b/kernel/module.c
index d0b5f8db11b4..eba134157ef6 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -55,6 +55,7 @@
 #include <linux/async.h>
 #include <linux/percpu.h>
 #include <linux/kmemleak.h>
+#include <linux/jump_label.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/module.h>
@@ -2308,6 +2309,11 @@ static void find_module_sections(struct module *mod, struct load_info *info)
                                         sizeof(*mod->tracepoints),
                                         &mod->num_tracepoints);
 #endif
+#ifdef HAVE_JUMP_LABEL
+        mod->jump_entries = section_objs(info, "__jump_table",
+                                        sizeof(*mod->jump_entries),
+                                        &mod->num_jump_entries);
+#endif
 #ifdef CONFIG_EVENT_TRACING
         mod->trace_events = section_objs(info, "_ftrace_events",
                                          sizeof(*mod->trace_events),
diff --git a/kernel/trace/trace_workqueue.c b/kernel/trace/trace_workqueue.c
index a7cc3793baf6..209b379a4721 100644
--- a/kernel/trace/trace_workqueue.c
+++ b/kernel/trace/trace_workqueue.c
@@ -263,6 +263,11 @@ int __init trace_workqueue_early_init(void)
 {
         int ret, cpu;
 
+        for_each_possible_cpu(cpu) {
+                spin_lock_init(&workqueue_cpu_stat(cpu)->lock);
+                INIT_LIST_HEAD(&workqueue_cpu_stat(cpu)->list);
+        }
+
         ret = register_trace_workqueue_insertion(probe_workqueue_insertion, NULL);
         if (ret)
                 goto out;
@@ -279,11 +284,6 @@ int __init trace_workqueue_early_init(void)
         if (ret)
                 goto no_creation;
 
-        for_each_possible_cpu(cpu) {
-                spin_lock_init(&workqueue_cpu_stat(cpu)->lock);
-                INIT_LIST_HEAD(&workqueue_cpu_stat(cpu)->list);
-        }
-
         return 0;
 
 no_creation:
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index c77f3eceea25..d6073a50a6ca 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -25,6 +25,7 @@
 #include <linux/err.h>
 #include <linux/slab.h>
 #include <linux/sched.h>
+#include <linux/jump_label.h>
 
 extern struct tracepoint __start___tracepoints[];
 extern struct tracepoint __stop___tracepoints[];
@@ -263,7 +264,13 @@ static void set_tracepoint(struct tracepoint_entry **entry,
          * is used.
          */
         rcu_assign_pointer(elem->funcs, (*entry)->funcs);
-        elem->state = active;
+        if (!elem->state && active) {
+                enable_jump_label(&elem->state);
+                elem->state = active;
+        } else if (elem->state && !active) {
+                disable_jump_label(&elem->state);
+                elem->state = active;
+        }
 }
 
 /*
@@ -277,7 +284,10 @@ static void disable_tracepoint(struct tracepoint *elem)
         if (elem->unregfunc && elem->state)
                 elem->unregfunc();
 
-        elem->state = 0;
+        if (elem->state) {
+                disable_jump_label(&elem->state);
+                elem->state = 0;
+        }
         rcu_assign_pointer(elem->funcs, NULL);
 }