1 files changed, 92 insertions, 107 deletions

diff --git a/kernel/module.c b/kernel/module.c
index 14b224967e7b..a0a3d6d9d5e8 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -108,13 +108,6 @@ static LIST_HEAD(modules);
  * Use a latched RB-tree for __module_address(); this allows us to use
  * RCU-sched lookups of the address from any context.
  *
- * Because modules have two address ranges: init and core, we need two
- * latch_tree_nodes entries. Therefore we need the back-pointer from
- * mod_tree_node.
- *
- * Because init ranges are short lived we mark them unlikely and have placed
- * them outside the critical cacheline in struct module.
- *
  * This is conditional on PERF_EVENTS || TRACING because those can really hit
  * __module_address() hard by doing a lot of stack unwinding; potentially from
  * NMI context.
@@ -122,24 +115,16 @@ static LIST_HEAD(modules);
 
 static __always_inline unsigned long __mod_tree_val(struct latch_tree_node *n)
 {
-        struct mod_tree_node *mtn = container_of(n, struct mod_tree_node, node);
-        struct module *mod = mtn->mod;
+        struct module_layout *layout = container_of(n, struct module_layout, mtn.node);
 
-        if (unlikely(mtn == &mod->mtn_init))
-                return (unsigned long)mod->module_init;
-
-        return (unsigned long)mod->module_core;
+        return (unsigned long)layout->base;
 }
 
 static __always_inline unsigned long __mod_tree_size(struct latch_tree_node *n)
 {
-        struct mod_tree_node *mtn = container_of(n, struct mod_tree_node, node);
-        struct module *mod = mtn->mod;
-
-        if (unlikely(mtn == &mod->mtn_init))
-                return (unsigned long)mod->init_size;
+        struct module_layout *layout = container_of(n, struct module_layout, mtn.node);
 
-        return (unsigned long)mod->core_size;
+        return (unsigned long)layout->size;
 }
 
 static __always_inline bool
@@ -197,23 +182,23 @@ static void __mod_tree_remove(struct mod_tree_node *node)
  */
 static void mod_tree_insert(struct module *mod)
 {
-        mod->mtn_core.mod = mod;
-        mod->mtn_init.mod = mod;
+        mod->core_layout.mtn.mod = mod;
+        mod->init_layout.mtn.mod = mod;
 
-        __mod_tree_insert(&mod->mtn_core);
-        if (mod->init_size)
-                __mod_tree_insert(&mod->mtn_init);
+        __mod_tree_insert(&mod->core_layout.mtn);
+        if (mod->init_layout.size)
+                __mod_tree_insert(&mod->init_layout.mtn);
 }
 
 static void mod_tree_remove_init(struct module *mod)
 {
-        if (mod->init_size)
-                __mod_tree_remove(&mod->mtn_init);
+        if (mod->init_layout.size)
+                __mod_tree_remove(&mod->init_layout.mtn);
 }
 
 static void mod_tree_remove(struct module *mod)
 {
-        __mod_tree_remove(&mod->mtn_core);
+        __mod_tree_remove(&mod->core_layout.mtn);
         mod_tree_remove_init(mod);
 }
 
@@ -267,9 +252,9 @@ static void __mod_update_bounds(void *base, unsigned int size)
 
 static void mod_update_bounds(struct module *mod)
 {
-        __mod_update_bounds(mod->module_core, mod->core_size);
-        if (mod->init_size)
-                __mod_update_bounds(mod->module_init, mod->init_size);
+        __mod_update_bounds(mod->core_layout.base, mod->core_layout.size);
+        if (mod->init_layout.size)
+                __mod_update_bounds(mod->init_layout.base, mod->init_layout.size);
 }
 
 #ifdef CONFIG_KGDB_KDB
@@ -1214,7 +1199,7 @@ struct module_attribute module_uevent =
 static ssize_t show_coresize(struct module_attribute *mattr,
                              struct module_kobject *mk, char *buffer)
 {
-        return sprintf(buffer, "%u\n", mk->mod->core_size);
+        return sprintf(buffer, "%u\n", mk->mod->core_layout.size);
 }
 
 static struct module_attribute modinfo_coresize =
@@ -1223,7 +1208,7 @@ static struct module_attribute modinfo_coresize =
 static ssize_t show_initsize(struct module_attribute *mattr,
                              struct module_kobject *mk, char *buffer)
 {
-        return sprintf(buffer, "%u\n", mk->mod->init_size);
+        return sprintf(buffer, "%u\n", mk->mod->init_layout.size);
 }
 
 static struct module_attribute modinfo_initsize =
@@ -1917,29 +1902,29 @@ static void set_section_ro_nx(void *base,
 
 static void set_module_core_ro_nx(struct module *mod)
 {
-        set_section_ro_nx(mod->module_core, mod->core_text_size,
-                          mod->core_ro_size, mod->core_size,
+        set_section_ro_nx(mod->core_layout.base, mod->core_layout.text_size,
+                          mod->core_layout.ro_size, mod->core_layout.size,
                           set_memory_ro, set_memory_nx);
 }
 
 static void unset_module_core_ro_nx(struct module *mod)
 {
-        set_section_ro_nx(mod->module_core, mod->core_text_size,
-                          mod->core_ro_size, mod->core_size,
+        set_section_ro_nx(mod->core_layout.base, mod->core_layout.text_size,
+                          mod->core_layout.ro_size, mod->core_layout.size,
                           set_memory_rw, set_memory_x);
 }
 
 static void set_module_init_ro_nx(struct module *mod)
 {
-        set_section_ro_nx(mod->module_init, mod->init_text_size,
-                          mod->init_ro_size, mod->init_size,
+        set_section_ro_nx(mod->init_layout.base, mod->init_layout.text_size,
+                          mod->init_layout.ro_size, mod->init_layout.size,
                           set_memory_ro, set_memory_nx);
 }
 
 static void unset_module_init_ro_nx(struct module *mod)
 {
-        set_section_ro_nx(mod->module_init, mod->init_text_size,
-                          mod->init_ro_size, mod->init_size,
+        set_section_ro_nx(mod->init_layout.base, mod->init_layout.text_size,
+                          mod->init_layout.ro_size, mod->init_layout.size,
                           set_memory_rw, set_memory_x);
 }
 
@@ -1952,14 +1937,14 @@ void set_all_modules_text_rw(void)
         list_for_each_entry_rcu(mod, &modules, list) {
                 if (mod->state == MODULE_STATE_UNFORMED)
                         continue;
-                if ((mod->module_core) && (mod->core_text_size)) {
-                        set_page_attributes(mod->module_core,
-                                                mod->module_core + mod->core_text_size,
+                if ((mod->core_layout.base) && (mod->core_layout.text_size)) {
+                        set_page_attributes(mod->core_layout.base,
+                                                mod->core_layout.base + mod->core_layout.text_size,
                                                 set_memory_rw);
                 }
-                if ((mod->module_init) && (mod->init_text_size)) {
-                        set_page_attributes(mod->module_init,
-                                                mod->module_init + mod->init_text_size,
+                if ((mod->init_layout.base) && (mod->init_layout.text_size)) {
+                        set_page_attributes(mod->init_layout.base,
+                                                mod->init_layout.base + mod->init_layout.text_size,
                                                 set_memory_rw);
                 }
         }
@@ -1975,14 +1960,14 @@ void set_all_modules_text_ro(void)
         list_for_each_entry_rcu(mod, &modules, list) {
                 if (mod->state == MODULE_STATE_UNFORMED)
                         continue;
-                if ((mod->module_core) && (mod->core_text_size)) {
-                        set_page_attributes(mod->module_core,
-                                                mod->module_core + mod->core_text_size,
+                if ((mod->core_layout.base) && (mod->core_layout.text_size)) {
+                        set_page_attributes(mod->core_layout.base,
+                                                mod->core_layout.base + mod->core_layout.text_size,
                                                 set_memory_ro);
                 }
-                if ((mod->module_init) && (mod->init_text_size)) {
-                        set_page_attributes(mod->module_init,
-                                                mod->module_init + mod->init_text_size,
+                if ((mod->init_layout.base) && (mod->init_layout.text_size)) {
+                        set_page_attributes(mod->init_layout.base,
+                                                mod->init_layout.base + mod->init_layout.text_size,
                                                 set_memory_ro);
                 }
         }
@@ -2047,16 +2032,16 @@ static void free_module(struct module *mod)
         /* This may be NULL, but that's OK */
         unset_module_init_ro_nx(mod);
         module_arch_freeing_init(mod);
-        module_memfree(mod->module_init);
+        module_memfree(mod->init_layout.base);
         kfree(mod->args);
         percpu_modfree(mod);
 
         /* Free lock-classes; relies on the preceding sync_rcu(). */
-        lockdep_free_key_range(mod->module_core, mod->core_size);
+        lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size);
 
         /* Finally, free the core (containing the module structure) */
         unset_module_core_ro_nx(mod);
-        module_memfree(mod->module_core);
+        module_memfree(mod->core_layout.base);
 
 #ifdef CONFIG_MPU
         update_protections(current->mm);
@@ -2259,20 +2244,20 @@ static void layout_sections(struct module *mod, struct load_info *info)
                             || s->sh_entsize != ~0UL
                             || strstarts(sname, ".init"))
                                 continue;
-                        s->sh_entsize = get_offset(mod, &mod->core_size, s, i);
+                        s->sh_entsize = get_offset(mod, &mod->core_layout.size, s, i);
                         pr_debug("\t%s\n", sname);
                 }
                 switch (m) {
                 case 0: /* executable */
-                        mod->core_size = debug_align(mod->core_size);
-                        mod->core_text_size = mod->core_size;
+                        mod->core_layout.size = debug_align(mod->core_layout.size);
+                        mod->core_layout.text_size = mod->core_layout.size;
                         break;
                 case 1: /* RO: text and ro-data */
-                        mod->core_size = debug_align(mod->core_size);
-                        mod->core_ro_size = mod->core_size;
+                        mod->core_layout.size = debug_align(mod->core_layout.size);
+                        mod->core_layout.ro_size = mod->core_layout.size;
                         break;
                 case 3: /* whole core */
-                        mod->core_size = debug_align(mod->core_size);
+                        mod->core_layout.size = debug_align(mod->core_layout.size);
                         break;
                 }
         }
@@ -2288,21 +2273,21 @@ static void layout_sections(struct module *mod, struct load_info *info)
                             || s->sh_entsize != ~0UL
                             || !strstarts(sname, ".init"))
                                 continue;
-                        s->sh_entsize = (get_offset(mod, &mod->init_size, s, i)
+                        s->sh_entsize = (get_offset(mod, &mod->init_layout.size, s, i)
                                          | INIT_OFFSET_MASK);
                         pr_debug("\t%s\n", sname);
                 }
                 switch (m) {
                 case 0: /* executable */
-                        mod->init_size = debug_align(mod->init_size);
-                        mod->init_text_size = mod->init_size;
+                        mod->init_layout.size = debug_align(mod->init_layout.size);
+                        mod->init_layout.text_size = mod->init_layout.size;
                         break;
                 case 1: /* RO: text and ro-data */
-                        mod->init_size = debug_align(mod->init_size);
-                        mod->init_ro_size = mod->init_size;
+                        mod->init_layout.size = debug_align(mod->init_layout.size);
+                        mod->init_layout.ro_size = mod->init_layout.size;
                         break;
                 case 3: /* whole init */
-                        mod->init_size = debug_align(mod->init_size);
+                        mod->init_layout.size = debug_align(mod->init_layout.size);
                         break;
                 }
         }
@@ -2477,7 +2462,7 @@ static void layout_symtab(struct module *mod, struct load_info *info)
 
         /* Put symbol section at end of init part of module. */
         symsect->sh_flags |= SHF_ALLOC;
-        symsect->sh_entsize = get_offset(mod, &mod->init_size, symsect,
+        symsect->sh_entsize = get_offset(mod, &mod->init_layout.size, symsect,
                                          info->index.sym) | INIT_OFFSET_MASK;
         pr_debug("\t%s\n", info->secstrings + symsect->sh_name);
 
@@ -2494,16 +2479,16 @@ static void layout_symtab(struct module *mod, struct load_info *info)
         }
 
         /* Append room for core symbols at end of core part. */
-        info->symoffs = ALIGN(mod->core_size, symsect->sh_addralign ?: 1);
-        info->stroffs = mod->core_size = info->symoffs + ndst * sizeof(Elf_Sym);
-        mod->core_size += strtab_size;
-        mod->core_size = debug_align(mod->core_size);
+        info->symoffs = ALIGN(mod->core_layout.size, symsect->sh_addralign ?: 1);
+        info->stroffs = mod->core_layout.size = info->symoffs + ndst * sizeof(Elf_Sym);
+        mod->core_layout.size += strtab_size;
+        mod->core_layout.size = debug_align(mod->core_layout.size);
 
         /* Put string table section at end of init part of module. */
         strsect->sh_flags |= SHF_ALLOC;
-        strsect->sh_entsize = get_offset(mod, &mod->init_size, strsect,
+        strsect->sh_entsize = get_offset(mod, &mod->init_layout.size, strsect,
                                          info->index.str) | INIT_OFFSET_MASK;
-        mod->init_size = debug_align(mod->init_size);
+        mod->init_layout.size = debug_align(mod->init_layout.size);
         pr_debug("\t%s\n", info->secstrings + strsect->sh_name);
 }
 
@@ -2524,8 +2509,8 @@ static void add_kallsyms(struct module *mod, const struct load_info *info)
         for (i = 0; i < mod->num_symtab; i++)
                 mod->symtab[i].st_info = elf_type(&mod->symtab[i], info);
 
-        mod->core_symtab = dst = mod->module_core + info->symoffs;
-        mod->core_strtab = s = mod->module_core + info->stroffs;
+        mod->core_symtab = dst = mod->core_layout.base + info->symoffs;
+        mod->core_strtab = s = mod->core_layout.base + info->stroffs;
         src = mod->symtab;
         for (ndst = i = 0; i < mod->num_symtab; i++) {
                 if (i == 0 ||
@@ -2975,7 +2960,7 @@ static int move_module(struct module *mod, struct load_info *info)
         void *ptr;
 
         /* Do the allocs. */
-        ptr = module_alloc(mod->core_size);
+        ptr = module_alloc(mod->core_layout.size);
         /*
          * The pointer to this block is stored in the module structure
          * which is inside the block. Just mark it as not being a
@@ -2985,11 +2970,11 @@ static int move_module(struct module *mod, struct load_info *info)
         if (!ptr)
                 return -ENOMEM;
 
-        memset(ptr, 0, mod->core_size);
-        mod->module_core = ptr;
+        memset(ptr, 0, mod->core_layout.size);
+        mod->core_layout.base = ptr;
 
-        if (mod->init_size) {
-                ptr = module_alloc(mod->init_size);
+        if (mod->init_layout.size) {
+                ptr = module_alloc(mod->init_layout.size);
                 /*
                  * The pointer to this block is stored in the module structure
                  * which is inside the block. This block doesn't need to be
@@ -2998,13 +2983,13 @@ static int move_module(struct module *mod, struct load_info *info)
                  */
                 kmemleak_ignore(ptr);
                 if (!ptr) {
-                        module_memfree(mod->module_core);
+                        module_memfree(mod->core_layout.base);
                         return -ENOMEM;
                 }
-                memset(ptr, 0, mod->init_size);
-                mod->module_init = ptr;
+                memset(ptr, 0, mod->init_layout.size);
+                mod->init_layout.base = ptr;
         } else
-                mod->module_init = NULL;
+                mod->init_layout.base = NULL;
 
         /* Transfer each section which specifies SHF_ALLOC */
         pr_debug("final section addresses:\n");
@@ -3016,10 +3001,10 @@ static int move_module(struct module *mod, struct load_info *info)
                         continue;
 
                 if (shdr->sh_entsize & INIT_OFFSET_MASK)
-                        dest = mod->module_init
+                        dest = mod->init_layout.base
                                 + (shdr->sh_entsize & ~INIT_OFFSET_MASK);
                 else
-                        dest = mod->module_core + shdr->sh_entsize;
+                        dest = mod->core_layout.base + shdr->sh_entsize;
 
                 if (shdr->sh_type != SHT_NOBITS)
                         memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
@@ -3081,12 +3066,12 @@ static void flush_module_icache(const struct module *mod)
          * Do it before processing of module parameters, so the module
          * can provide parameter accessor functions of its own.
          */
-        if (mod->module_init)
-                flush_icache_range((unsigned long)mod->module_init,
-                                   (unsigned long)mod->module_init
-                                   + mod->init_size);
-        flush_icache_range((unsigned long)mod->module_core,
-                           (unsigned long)mod->module_core + mod->core_size);
+        if (mod->init_layout.base)
+                flush_icache_range((unsigned long)mod->init_layout.base,
+                                   (unsigned long)mod->init_layout.base
+                                   + mod->init_layout.size);
+        flush_icache_range((unsigned long)mod->core_layout.base,
+                           (unsigned long)mod->core_layout.base + mod->core_layout.size);
 
         set_fs(old_fs);
 }
@@ -3144,8 +3129,8 @@ static void module_deallocate(struct module *mod, struct load_info *info)
 {
         percpu_modfree(mod);
         module_arch_freeing_init(mod);
-        module_memfree(mod->module_init);
-        module_memfree(mod->module_core);
+        module_memfree(mod->init_layout.base);
+        module_memfree(mod->core_layout.base);
 }
 
 int __weak module_finalize(const Elf_Ehdr *hdr,
@@ -3232,7 +3217,7 @@ static noinline int do_init_module(struct module *mod)
                 ret = -ENOMEM;
                 goto fail;
         }
-        freeinit->module_init = mod->module_init;
+        freeinit->module_init = mod->init_layout.base;
 
         /*
          * We want to find out whether @mod uses async during init.  Clear
@@ -3292,10 +3277,10 @@ static noinline int do_init_module(struct module *mod)
         mod_tree_remove_init(mod);
         unset_module_init_ro_nx(mod);
         module_arch_freeing_init(mod);
-        mod->module_init = NULL;
-        mod->init_size = 0;
-        mod->init_ro_size = 0;
-        mod->init_text_size = 0;
+        mod->init_layout.base = NULL;
+        mod->init_layout.size = 0;
+        mod->init_layout.ro_size = 0;
+        mod->init_layout.text_size = 0;
         /*
          * We want to free module_init, but be aware that kallsyms may be
          * walking this with preempt disabled.  In all the failure paths, we
@@ -3575,7 +3560,7 @@ static int load_module(struct load_info *info, const char __user *uargs,
         mutex_unlock(&module_mutex);
  free_module:
         /* Free lock-classes; relies on the preceding sync_rcu() */
-        lockdep_free_key_range(mod->module_core, mod->core_size);
+        lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size);
 
         module_deallocate(mod, info);
  free_copy:
@@ -3653,9 +3638,9 @@ static const char *get_ksymbol(struct module *mod,
 
         /* At worse, next value is at end of module */
         if (within_module_init(addr, mod))
-                nextval = (unsigned long)mod->module_init+mod->init_text_size;
+                nextval = (unsigned long)mod->init_layout.base+mod->init_layout.text_size;
         else
-                nextval = (unsigned long)mod->module_core+mod->core_text_size;
+                nextval = (unsigned long)mod->core_layout.base+mod->core_layout.text_size;
 
         /* Scan for closest preceding symbol, and next symbol. (ELF
            starts real symbols at 1). */
@@ -3902,7 +3887,7 @@ static int m_show(struct seq_file *m, void *p)
                 return 0;
 
         seq_printf(m, "%s %u",
-                   mod->name, mod->init_size + mod->core_size);
+                   mod->name, mod->init_layout.size + mod->core_layout.size);
         print_unload_info(m, mod);
 
         /* Informative for users. */
@@ -3911,7 +3896,7 @@ static int m_show(struct seq_file *m, void *p)
                    mod->state == MODULE_STATE_COMING ? "Loading" :
                    "Live");
         /* Used by oprofile and other similar tools. */
-        seq_printf(m, " 0x%pK", mod->module_core);
+        seq_printf(m, " 0x%pK", mod->core_layout.base);
 
         /* Taints info */
         if (mod->taints)
@@ -4054,8 +4039,8 @@ struct module *__module_text_address(unsigned long addr)
         struct module *mod = __module_address(addr);
         if (mod) {
                 /* Make sure it's within the text section. */
-                if (!within(addr, mod->module_init, mod->init_text_size)
-                    && !within(addr, mod->module_core, mod->core_text_size))
+                if (!within(addr, mod->init_layout.base, mod->init_layout.text_size)
+                    && !within(addr, mod->core_layout.base, mod->core_layout.text_size))
                         mod = NULL;
         }
         return mod;