1 files changed, 87 insertions, 65 deletions

diff --git a/kernel/module.c b/kernel/module.c
index 1f4cc00e0c20..ba22484a987e 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -43,7 +43,6 @@
 #include <linux/device.h>
 #include <linux/string.h>
 #include <linux/mutex.h>
-#include <linux/unwind.h>
 #include <linux/rculist.h>
 #include <asm/uaccess.h>
 #include <asm/cacheflush.h>
@@ -51,6 +50,7 @@
 #include <asm/sections.h>
 #include <linux/tracepoint.h>
 #include <linux/ftrace.h>
+#include <linux/async.h>
 
 #if 0
 #define DEBUGP printk
@@ -573,13 +573,13 @@ static char last_unloaded_module[MODULE_NAME_LEN+1];
 /* Init the unload section of the module. */
 static void module_unload_init(struct module *mod)
 {
-        unsigned int i;
+        int cpu;
 
         INIT_LIST_HEAD(&mod->modules_which_use_me);
-        for (i = 0; i < NR_CPUS; i++)
-                local_set(&mod->ref[i].count, 0);
+        for_each_possible_cpu(cpu)
+                local_set(__module_ref_addr(mod, cpu), 0);
         /* Hold reference count during initialization. */
-        local_set(&mod->ref[raw_smp_processor_id()].count, 1);
+        local_set(__module_ref_addr(mod, raw_smp_processor_id()), 1);
         /* Backwards compatibility macros put refcount during init. */
         mod->waiter = current;
 }
@@ -717,10 +717,11 @@ static int try_stop_module(struct module *mod, int flags, int *forced)
 
 unsigned int module_refcount(struct module *mod)
 {
-        unsigned int i, total = 0;
+        unsigned int total = 0;
+        int cpu;
 
-        for (i = 0; i < NR_CPUS; i++)
-                total += local_read(&mod->ref[i].count);
+        for_each_possible_cpu(cpu)
+                total += local_read(__module_ref_addr(mod, cpu));
         return total;
 }
 EXPORT_SYMBOL(module_refcount);
@@ -743,8 +744,8 @@ static void wait_for_zero_refcount(struct module *mod)
         mutex_lock(&module_mutex);
 }
 
-asmlinkage long
-sys_delete_module(const char __user *name_user, unsigned int flags)
+SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
+                unsigned int, flags)
 {
         struct module *mod;
         char name[MODULE_NAME_LEN];
@@ -757,8 +758,16 @@ sys_delete_module(const char __user *name_user, unsigned int flags)
                 return -EFAULT;
         name[MODULE_NAME_LEN-1] = '\0';
 
-        if (mutex_lock_interruptible(&module_mutex) != 0)
-                return -EINTR;
+        /* Create stop_machine threads since free_module relies on
+         * a non-failing stop_machine call. */
+        ret = stop_machine_create();
+        if (ret)
+                return ret;
+
+        if (mutex_lock_interruptible(&module_mutex) != 0) {
+                ret = -EINTR;
+                goto out_stop;
+        }
 
         mod = find_module(name);
         if (!mod) {
@@ -809,6 +818,7 @@ sys_delete_module(const char __user *name_user, unsigned int flags)
                 mod->exit();
         blocking_notifier_call_chain(&module_notify_list,
                                      MODULE_STATE_GOING, mod);
+        async_synchronize_full();
         mutex_lock(&module_mutex);
         /* Store the name of the last unloaded module for diagnostic purposes */
         strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module));
@@ -817,10 +827,12 @@ sys_delete_module(const char __user *name_user, unsigned int flags)
 
  out:
         mutex_unlock(&module_mutex);
+out_stop:
+        stop_machine_destroy();
         return ret;
 }
 
-static void print_unload_info(struct seq_file *m, struct module *mod)
+static inline void print_unload_info(struct seq_file *m, struct module *mod)
 {
         struct module_use *use;
         int printed_something = 0;
@@ -883,7 +895,7 @@ void module_put(struct module *module)
 {
         if (module) {
                 unsigned int cpu = get_cpu();
-                local_dec(&module->ref[cpu].count);
+                local_dec(__module_ref_addr(module, cpu));
                 /* Maybe they're waiting for us to drop reference? */
                 if (unlikely(!module_is_live(module)))
                         wake_up_process(module->waiter);
@@ -893,7 +905,7 @@ void module_put(struct module *module)
 EXPORT_SYMBOL(module_put);
 
 #else /* !CONFIG_MODULE_UNLOAD */
-static void print_unload_info(struct seq_file *m, struct module *mod)
+static inline void print_unload_info(struct seq_file *m, struct module *mod)
 {
         /* We don't know the usage count, or what modules are using. */
         seq_printf(m, " - -");
@@ -1439,8 +1451,6 @@ static void free_module(struct module *mod)
         remove_sect_attrs(mod);
         mod_kobject_remove(mod);
 
-        unwind_remove_table(mod->unwind_info, 0);
-
         /* Arch-specific cleanup. */
         module_arch_cleanup(mod);
 
@@ -1455,7 +1465,10 @@ static void free_module(struct module *mod)
         kfree(mod->args);
         if (mod->percpu)
                 percpu_modfree(mod->percpu);
-
+#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP)
+        if (mod->refptr)
+                percpu_modfree(mod->refptr);
+#endif
         /* Free lock-classes: */
         lockdep_free_key_range(mod->module_core, mod->core_size);
 
@@ -1578,11 +1591,21 @@ static int simplify_symbols(Elf_Shdr *sechdrs,
         return ret;
 }
 
+/* Additional bytes needed by arch in front of individual sections */
+unsigned int __weak arch_mod_section_prepend(struct module *mod,
+                                             unsigned int section)
+{
+        /* default implementation just returns zero */
+        return 0;
+}
+
 /* Update size with this section: return offset. */
-static long get_offset(unsigned int *size, Elf_Shdr *sechdr)
+static long get_offset(struct module *mod, unsigned int *size,
+                       Elf_Shdr *sechdr, unsigned int section)
 {
         long ret;
 
+        *size += arch_mod_section_prepend(mod, section);
         ret = ALIGN(*size, sechdr->sh_addralign ?: 1);
         *size = ret + sechdr->sh_size;
         return ret;
@@ -1622,7 +1645,7 @@ static void layout_sections(struct module *mod,
                             || strncmp(secstrings + s->sh_name,
                                        ".init", 5) == 0)
                                 continue;
-                        s->sh_entsize = get_offset(&mod->core_size, s);
+                        s->sh_entsize = get_offset(mod, &mod->core_size, s, i);
                         DEBUGP("\t%s\n", secstrings + s->sh_name);
                 }
                 if (m == 0)
@@ -1640,7 +1663,7 @@ static void layout_sections(struct module *mod,
                             || strncmp(secstrings + s->sh_name,
                                        ".init", 5) != 0)
                                 continue;
-                        s->sh_entsize = (get_offset(&mod->init_size, s)
+                        s->sh_entsize = (get_offset(mod, &mod->init_size, s, i)
                                          | INIT_OFFSET_MASK);
                         DEBUGP("\t%s\n", secstrings + s->sh_name);
                 }
@@ -1725,15 +1748,15 @@ static const struct kernel_symbol *lookup_symbol(const char *name,
         return NULL;
 }
 
-static int is_exported(const char *name, const struct module *mod)
+static int is_exported(const char *name, unsigned long value,
+                       const struct module *mod)
 {
-        if (!mod && lookup_symbol(name, __start___ksymtab, __stop___ksymtab))
-                return 1;
+        const struct kernel_symbol *ks;
+        if (!mod)
+                ks = lookup_symbol(name, __start___ksymtab, __stop___ksymtab);
         else
-                if (mod && lookup_symbol(name, mod->syms, mod->syms + mod->num_syms))
-                        return 1;
-                else
-                        return 0;
+                ks = lookup_symbol(name, mod->syms, mod->syms + mod->num_syms);
+        return ks != NULL && ks->value == value;
 }
 
 /* As per nm */
@@ -1847,7 +1870,6 @@ static noinline struct module *load_module(void __user *umod,
         unsigned int symindex = 0;
         unsigned int strindex = 0;
         unsigned int modindex, versindex, infoindex, pcpuindex;
-        unsigned int unwindex = 0;
         unsigned int num_kp, num_mcount;
         struct kernel_param *kp;
         struct module *mod;
@@ -1865,6 +1887,13 @@ static noinline struct module *load_module(void __user *umod,
         /* vmalloc barfs on "unusual" numbers.  Check here */
         if (len > 64 * 1024 * 1024 || (hdr = vmalloc(len)) == NULL)
                 return ERR_PTR(-ENOMEM);
+
+        /* Create stop_machine threads since the error path relies on
+         * a non-failing stop_machine call. */
+        err = stop_machine_create();
+        if (err)
+                goto free_hdr;
+
         if (copy_from_user(hdr, umod, len) != 0) {
                 err = -EFAULT;
                 goto free_hdr;
@@ -1930,9 +1959,6 @@ static noinline struct module *load_module(void __user *umod,
         versindex = find_sec(hdr, sechdrs, secstrings, "__versions");
         infoindex = find_sec(hdr, sechdrs, secstrings, ".modinfo");
         pcpuindex = find_pcpusec(hdr, sechdrs, secstrings);
-#ifdef ARCH_UNWIND_SECTION_NAME
-        unwindex = find_sec(hdr, sechdrs, secstrings, ARCH_UNWIND_SECTION_NAME);
-#endif
 
         /* Don't keep modinfo and version sections. */
         sechdrs[infoindex].sh_flags &= ~(unsigned long)SHF_ALLOC;
@@ -1942,8 +1968,6 @@ static noinline struct module *load_module(void __user *umod,
         sechdrs[symindex].sh_flags |= SHF_ALLOC;
         sechdrs[strindex].sh_flags |= SHF_ALLOC;
 #endif
-        if (unwindex)
-                sechdrs[unwindex].sh_flags |= SHF_ALLOC;
 
         /* Check module struct version now, before we try to use module. */
         if (!check_modstruct_version(sechdrs, versindex, mod)) {
@@ -1991,6 +2015,14 @@ static noinline struct module *load_module(void __user *umod,
         if (err < 0)
                 goto free_mod;
 
+#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP)
+        mod->refptr = percpu_modalloc(sizeof(local_t), __alignof__(local_t),
+                                      mod->name);
+        if (!mod->refptr) {
+                err = -ENOMEM;
+                goto free_mod;
+        }
+#endif
         if (pcpuindex) {
                 /* We have a special allocation for this section. */
                 percpu = percpu_modalloc(sechdrs[pcpuindex].sh_size,
@@ -1998,7 +2030,7 @@ static noinline struct module *load_module(void __user *umod,
                                          mod->name);
                 if (!percpu) {
                         err = -ENOMEM;
-                        goto free_mod;
+                        goto free_percpu;
                 }
                 sechdrs[pcpuindex].sh_flags &= ~(unsigned long)SHF_ALLOC;
                 mod->percpu = percpu;
@@ -2184,24 +2216,15 @@ static noinline struct module *load_module(void __user *umod,
                 struct mod_debug *debug;
                 unsigned int num_debug;
 
-#ifdef CONFIG_MARKERS
-                marker_update_probe_range(mod->markers,
-                        mod->markers + mod->num_markers);
-#endif
                 debug = section_objs(hdr, sechdrs, secstrings, "__verbose",
                                      sizeof(*debug), &num_debug);
                 dynamic_printk_setup(debug, num_debug);
-
-#ifdef CONFIG_TRACEPOINTS
-                tracepoint_update_probe_range(mod->tracepoints,
-                        mod->tracepoints + mod->num_tracepoints);
-#endif
         }
 
         /* sechdrs[0].sh_size is always zero */
         mseg = section_objs(hdr, sechdrs, secstrings, "__mcount_loc",
                             sizeof(*mseg), &num_mcount);
-        ftrace_init_module(mseg, mseg + num_mcount);
+        ftrace_init_module(mod, mseg, mseg + num_mcount);
 
         err = module_finalize(hdr, sechdrs, mod);
         if (err < 0)
@@ -2249,14 +2272,10 @@ static noinline struct module *load_module(void __user *umod,
         add_sect_attrs(mod, hdr->e_shnum, secstrings, sechdrs);
         add_notes_attrs(mod, hdr->e_shnum, secstrings, sechdrs);
 
-        /* Size of section 0 is 0, so this works well if no unwind info. */
-        mod->unwind_info = unwind_add_table(mod,
-                                            (void *)sechdrs[unwindex].sh_addr,
-                                            sechdrs[unwindex].sh_size);
-
         /* Get rid of temporary copy */
         vfree(hdr);
 
+        stop_machine_destroy();
         /* Done! */
         return mod;
 
@@ -2275,10 +2294,14 @@ static noinline struct module *load_module(void __user *umod,
  free_percpu:
         if (percpu)
                 percpu_modfree(percpu);
+#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP)
+        percpu_modfree(mod->refptr);
+#endif
  free_mod:
         kfree(args);
  free_hdr:
         vfree(hdr);
+        stop_machine_destroy();
         return ERR_PTR(err);
 
  truncated:
@@ -2288,10 +2311,8 @@ static noinline struct module *load_module(void __user *umod,
 }
 
 /* This is where the real work happens */
-asmlinkage long
-sys_init_module(void __user *umod,
-                unsigned long len,
-                const char __user *uargs)
+SYSCALL_DEFINE3(init_module, void __user *, umod,
+                unsigned long, len, const char __user *, uargs)
 {
         struct module *mod;
         int ret = 0;
@@ -2346,11 +2367,12 @@ sys_init_module(void __user *umod,
         /* Now it's a first class citizen!  Wake up anyone waiting for it. */
         mod->state = MODULE_STATE_LIVE;
         wake_up(&module_wq);
+        blocking_notifier_call_chain(&module_notify_list,
+                                     MODULE_STATE_LIVE, mod);
 
         mutex_lock(&module_mutex);
         /* Drop initial reference. */
         module_put(mod);
-        unwind_remove_table(mod->unwind_info, 1);
         module_free(mod, mod->module_init);
         mod->module_init = NULL;
         mod->init_size = 0;
@@ -2385,7 +2407,7 @@ static const char *get_ksymbol(struct module *mod,
         unsigned long nextval;
 
         /* At worse, next value is at end of module */
-        if (within(addr, mod->module_init, mod->init_size))
+        if (within_module_init(addr, mod))
                 nextval = (unsigned long)mod->module_init+mod->init_text_size;
         else
                 nextval = (unsigned long)mod->module_core+mod->core_text_size;
@@ -2433,8 +2455,8 @@ const char *module_address_lookup(unsigned long addr,
 
         preempt_disable();
         list_for_each_entry_rcu(mod, &modules, list) {
-                if (within(addr, mod->module_init, mod->init_size)
-                    || within(addr, mod->module_core, mod->core_size)) {
+                if (within_module_init(addr, mod) ||
+                    within_module_core(addr, mod)) {
                         if (modname)
                                 *modname = mod->name;
                         ret = get_ksymbol(mod, addr, size, offset);
@@ -2456,8 +2478,8 @@ int lookup_module_symbol_name(unsigned long addr, char *symname)
 
         preempt_disable();
         list_for_each_entry_rcu(mod, &modules, list) {
-                if (within(addr, mod->module_init, mod->init_size) ||
-                    within(addr, mod->module_core, mod->core_size)) {
+                if (within_module_init(addr, mod) ||
+                    within_module_core(addr, mod)) {
                         const char *sym;
 
                         sym = get_ksymbol(mod, addr, NULL, NULL);
@@ -2480,8 +2502,8 @@ int lookup_module_symbol_attrs(unsigned long addr, unsigned long *size,
 
         preempt_disable();
         list_for_each_entry_rcu(mod, &modules, list) {
-                if (within(addr, mod->module_init, mod->init_size) ||
-                    within(addr, mod->module_core, mod->core_size)) {
+                if (within_module_init(addr, mod) ||
+                    within_module_core(addr, mod)) {
                         const char *sym;
 
                         sym = get_ksymbol(mod, addr, size, offset);
@@ -2513,7 +2535,7 @@ int module_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
                         strlcpy(name, mod->strtab + mod->symtab[symnum].st_name,
                                 KSYM_NAME_LEN);
                         strlcpy(module_name, mod->name, MODULE_NAME_LEN);
-                        *exported = is_exported(name, mod);
+                        *exported = is_exported(name, *value, mod);
                         preempt_enable();
                         return 0;
                 }
@@ -2700,7 +2722,7 @@ int is_module_address(unsigned long addr)
         preempt_disable();
 
         list_for_each_entry_rcu(mod, &modules, list) {
-                if (within(addr, mod->module_core, mod->core_size)) {
+                if (within_module_core(addr, mod)) {
                         preempt_enable();
                         return 1;
                 }
@@ -2713,7 +2735,7 @@ int is_module_address(unsigned long addr)
 
 
 /* Is this a valid kernel address? */
-struct module *__module_text_address(unsigned long addr)
+__notrace_funcgraph struct module *__module_text_address(unsigned long addr)
 {
         struct module *mod;