1 files changed, 51 insertions, 56 deletions

diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 2e62503bea0d..b28028b08d44 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -38,23 +38,43 @@ static int jump_label_cmp(const void *a, const void *b)
         const struct jump_entry *jea = a;
         const struct jump_entry *jeb = b;
 
-        if (jea->key < jeb->key)
+        if (jump_entry_key(jea) < jump_entry_key(jeb))
                 return -1;
 
-        if (jea->key > jeb->key)
+        if (jump_entry_key(jea) > jump_entry_key(jeb))
                 return 1;
 
         return 0;
 }
 
+static void jump_label_swap(void *a, void *b, int size)
+{
+        long delta = (unsigned long)a - (unsigned long)b;
+        struct jump_entry *jea = a;
+        struct jump_entry *jeb = b;
+        struct jump_entry tmp = *jea;
+
+        jea->code       = jeb->code - delta;
+        jea->target     = jeb->target - delta;
+        jea->key        = jeb->key - delta;
+
+        jeb->code       = tmp.code + delta;
+        jeb->target     = tmp.target + delta;
+        jeb->key        = tmp.key + delta;
+}
+
 static void
 jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
 {
         unsigned long size;
+        void *swapfn = NULL;
+
+        if (IS_ENABLED(CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE))
+                swapfn = jump_label_swap;
 
         size = (((unsigned long)stop - (unsigned long)start)
                                         / sizeof(struct jump_entry));
-        sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
+        sort(start, size, sizeof(struct jump_entry), jump_label_cmp, swapfn);
 }
 
 static void jump_label_update(struct static_key *key);
@@ -85,6 +105,7 @@ void static_key_slow_inc_cpuslocked(struct static_key *key)
         int v, v1;
 
         STATIC_KEY_CHECK_USE(key);
+        lockdep_assert_cpus_held();
 
         /*
          * Careful if we get concurrent static_key_slow_inc() calls;
@@ -130,6 +151,7 @@ EXPORT_SYMBOL_GPL(static_key_slow_inc);
 void static_key_enable_cpuslocked(struct static_key *key)
 {
         STATIC_KEY_CHECK_USE(key);
+        lockdep_assert_cpus_held();
 
         if (atomic_read(&key->enabled) > 0) {
                 WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
@@ -160,6 +182,7 @@ EXPORT_SYMBOL_GPL(static_key_enable);
 void static_key_disable_cpuslocked(struct static_key *key)
 {
         STATIC_KEY_CHECK_USE(key);
+        lockdep_assert_cpus_held();
 
         if (atomic_read(&key->enabled) != 1) {
                 WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
@@ -185,6 +208,8 @@ static void __static_key_slow_dec_cpuslocked(struct static_key *key,
                                            unsigned long rate_limit,
                                            struct delayed_work *work)
 {
+        lockdep_assert_cpus_held();
+
         /*
          * The negative count check is valid even when a negative
          * key->enabled is in use by static_key_slow_inc(); a
@@ -261,8 +286,8 @@ EXPORT_SYMBOL_GPL(jump_label_rate_limit);
 
 static int addr_conflict(struct jump_entry *entry, void *start, void *end)
 {
-        if (entry->code <= (unsigned long)end &&
-                entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
+        if (jump_entry_code(entry) <= (unsigned long)end &&
+            jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
                 return 1;
 
         return 0;
@@ -321,16 +346,6 @@ static inline void static_key_set_linked(struct static_key *key)
         key->type |= JUMP_TYPE_LINKED;
 }
 
-static inline struct static_key *jump_entry_key(struct jump_entry *entry)
-{
-        return (struct static_key *)((unsigned long)entry->key & ~1UL);
-}
-
-static bool jump_entry_branch(struct jump_entry *entry)
-{
-        return (unsigned long)entry->key & 1UL;
-}
-
 /***
  * A 'struct static_key' uses a union such that it either points directly
  * to a table of 'struct jump_entry' or to a linked list of modules which in
@@ -355,7 +370,7 @@ static enum jump_label_type jump_label_type(struct jump_entry *entry)
 {
         struct static_key *key = jump_entry_key(entry);
         bool enabled = static_key_enabled(key);
-        bool branch = jump_entry_branch(entry);
+        bool branch = jump_entry_is_branch(entry);
 
         /* See the comment in linux/jump_label.h */
         return enabled ^ branch;
@@ -363,19 +378,20 @@ static enum jump_label_type jump_label_type(struct jump_entry *entry)
 
 static void __jump_label_update(struct static_key *key,
                                 struct jump_entry *entry,
-                                struct jump_entry *stop)
+                                struct jump_entry *stop,
+                                bool init)
 {
         for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
                 /*
                  * An entry->code of 0 indicates an entry which has been
                  * disabled because it was in an init text area.
                  */
-                if (entry->code) {
-                        if (kernel_text_address(entry->code))
+                if (init || !jump_entry_is_init(entry)) {
+                        if (kernel_text_address(jump_entry_code(entry)))
                                 arch_jump_label_transform(entry, jump_label_type(entry));
                         else
                                 WARN_ONCE(1, "can't patch jump_label at %pS",
-                                          (void *)(unsigned long)entry->code);
+                                          (void *)jump_entry_code(entry));
                 }
         }
 }
@@ -410,6 +426,9 @@ void __init jump_label_init(void)
                 if (jump_label_type(iter) == JUMP_LABEL_NOP)
                         arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
 
+                if (init_section_contains((void *)jump_entry_code(iter), 1))
+                        jump_entry_set_init(iter);
+
                 iterk = jump_entry_key(iter);
                 if (iterk == key)
                         continue;
@@ -422,26 +441,13 @@ void __init jump_label_init(void)
         cpus_read_unlock();
 }
 
-/* Disable any jump label entries in __init/__exit code */
-void __init jump_label_invalidate_initmem(void)
-{
-        struct jump_entry *iter_start = __start___jump_table;
-        struct jump_entry *iter_stop = __stop___jump_table;
-        struct jump_entry *iter;
-
-        for (iter = iter_start; iter < iter_stop; iter++) {
-                if (init_section_contains((void *)(unsigned long)iter->code, 1))
-                        iter->code = 0;
-        }
-}
-
 #ifdef CONFIG_MODULES
 
 static enum jump_label_type jump_label_init_type(struct jump_entry *entry)
 {
         struct static_key *key = jump_entry_key(entry);
         bool type = static_key_type(key);
-        bool branch = jump_entry_branch(entry);
+        bool branch = jump_entry_is_branch(entry);
 
         /* See the comment in linux/jump_label.h */
         return type ^ branch;
@@ -455,7 +461,7 @@ struct static_key_mod {
 
 static inline struct static_key_mod *static_key_mod(struct static_key *key)
 {
-        WARN_ON_ONCE(!(key->type & JUMP_TYPE_LINKED));
+        WARN_ON_ONCE(!static_key_linked(key));
         return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK);
 }
 
@@ -514,7 +520,8 @@ static void __jump_label_mod_update(struct static_key *key)
                         stop = __stop___jump_table;
                 else
                         stop = m->jump_entries + m->num_jump_entries;
-                __jump_label_update(key, mod->entries, stop);
+                __jump_label_update(key, mod->entries, stop,
+                                    m && m->state == MODULE_STATE_COMING);
         }
 }
 
@@ -560,12 +567,15 @@ static int jump_label_add_module(struct module *mod)
         for (iter = iter_start; iter < iter_stop; iter++) {
                 struct static_key *iterk;
 
+                if (within_module_init(jump_entry_code(iter), mod))
+                        jump_entry_set_init(iter);
+
                 iterk = jump_entry_key(iter);
                 if (iterk == key)
                         continue;
 
                 key = iterk;
-                if (within_module(iter->key, mod)) {
+                if (within_module((unsigned long)key, mod)) {
                         static_key_set_entries(key, iter);
                         continue;
                 }
@@ -595,7 +605,7 @@ static int jump_label_add_module(struct module *mod)
 
                 /* Only update if we've changed from our initial state */
                 if (jump_label_type(iter) != jump_label_init_type(iter))
-                        __jump_label_update(key, iter, iter_stop);
+                        __jump_label_update(key, iter, iter_stop, true);
         }
 
         return 0;
@@ -615,7 +625,7 @@ static void jump_label_del_module(struct module *mod)
 
                 key = jump_entry_key(iter);
 
-                if (within_module(iter->key, mod))
+                if (within_module((unsigned long)key, mod))
                         continue;
 
                 /* No memory during module load */
@@ -651,19 +661,6 @@ static void jump_label_del_module(struct module *mod)
         }
 }
 
-/* Disable any jump label entries in module init code */
-static void jump_label_invalidate_module_init(struct module *mod)
-{
-        struct jump_entry *iter_start = mod->jump_entries;
-        struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
-        struct jump_entry *iter;
-
-        for (iter = iter_start; iter < iter_stop; iter++) {
-                if (within_module_init(iter->code, mod))
-                        iter->code = 0;
-        }
-}
-
 static int
 jump_label_module_notify(struct notifier_block *self, unsigned long val,
                          void *data)
@@ -685,9 +682,6 @@ jump_label_module_notify(struct notifier_block *self, unsigned long val,
         case MODULE_STATE_GOING:
                 jump_label_del_module(mod);
                 break;
-        case MODULE_STATE_LIVE:
-                jump_label_invalidate_module_init(mod);
-                break;
         }
 
         jump_label_unlock();
@@ -757,7 +751,8 @@ static void jump_label_update(struct static_key *key)
         entry = static_key_entries(key);
         /* if there are no users, entry can be NULL */
         if (entry)
-                __jump_label_update(key, entry, stop);
+                __jump_label_update(key, entry, stop,
+                                    system_state < SYSTEM_RUNNING);
 }
 
 #ifdef CONFIG_STATIC_KEYS_SELFTEST