1 files changed, 166 insertions, 115 deletions

diff --git a/kernel/module.c b/kernel/module.c
index 88cec1ddb1e3..d856e96a3cce 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -42,7 +42,6 @@
 #include <linux/vermagic.h>
 #include <linux/notifier.h>
 #include <linux/sched.h>
-#include <linux/stop_machine.h>
 #include <linux/device.h>
 #include <linux/string.h>
 #include <linux/mutex.h>
@@ -98,7 +97,7 @@
  * 1) List of modules (also safely readable with preempt_disable),
  * 2) module_use links,
  * 3) module_addr_min/module_addr_max.
- * (delete uses stop_machine/add uses RCU list operations). */
+ * (delete and add uses RCU list operations). */
 DEFINE_MUTEX(module_mutex);
 EXPORT_SYMBOL_GPL(module_mutex);
 static LIST_HEAD(modules);
@@ -158,13 +157,13 @@ static BLOCKING_NOTIFIER_HEAD(module_notify_list);
  * Protected by module_mutex. */
 static unsigned long module_addr_min = -1UL, module_addr_max = 0;
 
-int register_module_notifier(struct notifier_block * nb)
+int register_module_notifier(struct notifier_block *nb)
 {
         return blocking_notifier_chain_register(&module_notify_list, nb);
 }
 EXPORT_SYMBOL(register_module_notifier);
 
-int unregister_module_notifier(struct notifier_block * nb)
+int unregister_module_notifier(struct notifier_block *nb)
 {
         return blocking_notifier_chain_unregister(&module_notify_list, nb);
 }
@@ -628,18 +627,23 @@ static char last_unloaded_module[MODULE_NAME_LEN+1];
 
 EXPORT_TRACEPOINT_SYMBOL(module_get);
 
+/* MODULE_REF_BASE is the base reference count by kmodule loader. */
+#define MODULE_REF_BASE 1
+
 /* Init the unload section of the module. */
 static int module_unload_init(struct module *mod)
 {
-        mod->refptr = alloc_percpu(struct module_ref);
-        if (!mod->refptr)
-                return -ENOMEM;
+        /*
+         * Initialize reference counter to MODULE_REF_BASE.
+         * refcnt == 0 means module is going.
+         */
+        atomic_set(&mod->refcnt, MODULE_REF_BASE);
 
         INIT_LIST_HEAD(&mod->source_list);
         INIT_LIST_HEAD(&mod->target_list);
 
         /* Hold reference count during initialization. */
-        raw_cpu_write(mod->refptr->incs, 1);
+        atomic_inc(&mod->refcnt);
 
         return 0;
 }
@@ -721,8 +725,6 @@ static void module_unload_free(struct module *mod)
                 kfree(use);
         }
         mutex_unlock(&module_mutex);
-
-        free_percpu(mod->refptr);
 }
 
 #ifdef CONFIG_MODULE_FORCE_UNLOAD
@@ -740,60 +742,48 @@ static inline int try_force_unload(unsigned int flags)
 }
 #endif /* CONFIG_MODULE_FORCE_UNLOAD */
 
-struct stopref
+/* Try to release refcount of module, 0 means success. */
+static int try_release_module_ref(struct module *mod)
 {
-        struct module *mod;
-        int flags;
-        int *forced;
-};
+        int ret;
 
-/* Whole machine is stopped with interrupts off when this runs. */
-static int __try_stop_module(void *_sref)
-{
-        struct stopref *sref = _sref;
+        /* Try to decrement refcnt which we set at loading */
+        ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt);
+        BUG_ON(ret < 0);
+        if (ret)
+                /* Someone can put this right now, recover with checking */
+                ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0);
+
+        return ret;
+}
 
+static int try_stop_module(struct module *mod, int flags, int *forced)
+{
         /* If it's not unused, quit unless we're forcing. */
-        if (module_refcount(sref->mod) != 0) {
-                if (!(*sref->forced = try_force_unload(sref->flags)))
+        if (try_release_module_ref(mod) != 0) {
+                *forced = try_force_unload(flags);
+                if (!(*forced))
                         return -EWOULDBLOCK;
         }
 
         /* Mark it as dying. */
-        sref->mod->state = MODULE_STATE_GOING;
-        return 0;
-}
+        mod->state = MODULE_STATE_GOING;
 
-static int try_stop_module(struct module *mod, int flags, int *forced)
-{
-        struct stopref sref = { mod, flags, forced };
-
-        return stop_machine(__try_stop_module, &sref, NULL);
+        return 0;
 }
 
-unsigned long module_refcount(struct module *mod)
+/**
+ * module_refcount - return the refcount or -1 if unloading
+ *
+ * @mod:        the module we're checking
+ *
+ * Returns:
+ *      -1 if the module is in the process of unloading
+ *      otherwise the number of references in the kernel to the module
+ */
+int module_refcount(struct module *mod)
 {
-        unsigned long incs = 0, decs = 0;
-        int cpu;
-
-        for_each_possible_cpu(cpu)
-                decs += per_cpu_ptr(mod->refptr, cpu)->decs;
-        /*
-         * ensure the incs are added up after the decs.
-         * module_put ensures incs are visible before decs with smp_wmb.
-         *
-         * This 2-count scheme avoids the situation where the refcount
-         * for CPU0 is read, then CPU0 increments the module refcount,
-         * then CPU1 drops that refcount, then the refcount for CPU1 is
-         * read. We would record a decrement but not its corresponding
-         * increment so we would see a low count (disaster).
-         *
-         * Rare situation? But module_refcount can be preempted, and we
-         * might be tallying up 4096+ CPUs. So it is not impossible.
-         */
-        smp_rmb();
-        for_each_possible_cpu(cpu)
-                incs += per_cpu_ptr(mod->refptr, cpu)->incs;
-        return incs - decs;
+        return atomic_read(&mod->refcnt) - MODULE_REF_BASE;
 }
 EXPORT_SYMBOL(module_refcount);
 
@@ -875,10 +865,12 @@ static inline void print_unload_info(struct seq_file *m, struct module *mod)
         struct module_use *use;
         int printed_something = 0;
 
-        seq_printf(m, " %lu ", module_refcount(mod));
+        seq_printf(m, " %i ", module_refcount(mod));
 
-        /* Always include a trailing , so userspace can differentiate
-           between this and the old multi-field proc format. */
+        /*
+         * Always include a trailing , so userspace can differentiate
+         * between this and the old multi-field proc format.
+         */
         list_for_each_entry(use, &mod->source_list, source_list) {
                 printed_something = 1;
                 seq_printf(m, "%s,", use->source->name);
@@ -886,11 +878,11 @@ static inline void print_unload_info(struct seq_file *m, struct module *mod)
 
         if (mod->init != NULL && mod->exit == NULL) {
                 printed_something = 1;
-                seq_printf(m, "[permanent],");
+                seq_puts(m, "[permanent],");
         }
 
         if (!printed_something)
-                seq_printf(m, "-");
+                seq_puts(m, "-");
 }
 
 void __symbol_put(const char *symbol)
@@ -925,7 +917,7 @@ EXPORT_SYMBOL_GPL(symbol_put_addr);
 static ssize_t show_refcnt(struct module_attribute *mattr,
                            struct module_kobject *mk, char *buffer)
 {
-        return sprintf(buffer, "%lu\n", module_refcount(mk->mod));
+        return sprintf(buffer, "%i\n", module_refcount(mk->mod));
 }
 
 static struct module_attribute modinfo_refcnt =
@@ -935,7 +927,7 @@ void __module_get(struct module *module)
 {
         if (module) {
                 preempt_disable();
-                __this_cpu_inc(module->refptr->incs);
+                atomic_inc(&module->refcnt);
                 trace_module_get(module, _RET_IP_);
                 preempt_enable();
         }
@@ -948,11 +940,11 @@ bool try_module_get(struct module *module)
 
         if (module) {
                 preempt_disable();
-
-                if (likely(module_is_live(module))) {
-                        __this_cpu_inc(module->refptr->incs);
+                /* Note: here, we can fail to get a reference */
+                if (likely(module_is_live(module) &&
+                           atomic_inc_not_zero(&module->refcnt) != 0))
                         trace_module_get(module, _RET_IP_);
-                } else
+                else
                         ret = false;
 
                 preempt_enable();
@@ -963,11 +955,12 @@ EXPORT_SYMBOL(try_module_get);
 
 void module_put(struct module *module)
 {
+        int ret;
+
         if (module) {
                 preempt_disable();
-                smp_wmb(); /* see comment in module_refcount */
-                __this_cpu_inc(module->refptr->decs);
-
+                ret = atomic_dec_if_positive(&module->refcnt);
+                WARN_ON(ret < 0);       /* Failed to put refcount */
                 trace_module_put(module, _RET_IP_);
                 preempt_enable();
         }
@@ -978,7 +971,7 @@ EXPORT_SYMBOL(module_put);
 static inline void print_unload_info(struct seq_file *m, struct module *mod)
 {
         /* We don't know the usage count, or what modules are using. */
-        seq_printf(m, " - -");
+        seq_puts(m, " - -");
 }
 
 static inline void module_unload_free(struct module *mod)
@@ -1131,7 +1124,7 @@ static unsigned long maybe_relocated(unsigned long crc,
 static int check_version(Elf_Shdr *sechdrs,
                          unsigned int versindex,
                          const char *symname,
-                         struct module *mod, 
+                         struct module *mod,
                          const unsigned long *crc,
                          const struct module *crc_owner)
 {
@@ -1165,7 +1158,7 @@ static int check_version(Elf_Shdr *sechdrs,
         return 0;
 
 bad_version:
-        printk("%s: disagrees about version of symbol %s\n",
+        pr_warn("%s: disagrees about version of symbol %s\n",
                mod->name, symname);
         return 0;
 }
@@ -1200,7 +1193,7 @@ static inline int same_magic(const char *amagic, const char *bmagic,
 static inline int check_version(Elf_Shdr *sechdrs,
                                 unsigned int versindex,
                                 const char *symname,
-                                struct module *mod, 
+                                struct module *mod,
                                 const unsigned long *crc,
                                 const struct module *crc_owner)
 {
@@ -1288,15 +1281,13 @@ static inline bool sect_empty(const Elf_Shdr *sect)
         return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
 }
 
-struct module_sect_attr
-{
+struct module_sect_attr {
         struct module_attribute mattr;
         char *name;
         unsigned long address;
 };
 
-struct module_sect_attrs
-{
+struct module_sect_attrs {
         struct attribute_group grp;
         unsigned int nsections;
         struct module_sect_attr attrs[0];
@@ -1550,7 +1541,8 @@ static int module_add_modinfo_attrs(struct module *mod)
                     (attr->test && attr->test(mod))) {
                         memcpy(temp_attr, attr, sizeof(*temp_attr));
                         sysfs_attr_init(&temp_attr->attr);
-                        error = sysfs_create_file(&mod->mkobj.kobj,&temp_attr->attr);
+                        error = sysfs_create_file(&mod->mkobj.kobj,
+                                        &temp_attr->attr);
                         ++temp_attr;
                 }
         }
@@ -1566,7 +1558,7 @@ static void module_remove_modinfo_attrs(struct module *mod)
                 /* pick a field to test for end of list */
                 if (!attr->attr.name)
                         break;
-                sysfs_remove_file(&mod->mkobj.kobj,&attr->attr);
+                sysfs_remove_file(&mod->mkobj.kobj, &attr->attr);
                 if (attr->free)
                         attr->free(mod);
         }
@@ -1697,18 +1689,6 @@ static void mod_sysfs_teardown(struct module *mod)
         mod_sysfs_fini(mod);
 }
 
-/*
- * unlink the module with the whole machine is stopped with interrupts off
- * - this defends against kallsyms not taking locks
- */
-static int __unlink_module(void *_mod)
-{
-        struct module *mod = _mod;
-        list_del(&mod->list);
-        module_bug_cleanup(mod);
-        return 0;
-}
-
 #ifdef CONFIG_DEBUG_SET_MODULE_RONX
 /*
  * LKM RO/NX protection: protect module's text/ro-data
@@ -1824,7 +1804,7 @@ static void unset_module_core_ro_nx(struct module *mod) { }
 static void unset_module_init_ro_nx(struct module *mod) { }
 #endif
 
-void __weak module_free(struct module *mod, void *module_region)
+void __weak module_memfree(void *module_region)
 {
         vfree(module_region);
 }
@@ -1833,6 +1813,10 @@ void __weak module_arch_cleanup(struct module *mod)
 {
 }
 
+void __weak module_arch_freeing_init(struct module *mod)
+{
+}
+
 /* Free a module, remove from lists, etc. */
 static void free_module(struct module *mod)
 {
@@ -1860,12 +1844,18 @@ static void free_module(struct module *mod)
 
         /* Now we can delete it from the lists */
         mutex_lock(&module_mutex);
-        stop_machine(__unlink_module, mod, NULL);
+        /* Unlink carefully: kallsyms could be walking list. */
+        list_del_rcu(&mod->list);
+        /* Remove this module from bug list, this uses list_del_rcu */
+        module_bug_cleanup(mod);
+        /* Wait for RCU synchronizing before releasing mod->list and buglist. */
+        synchronize_rcu();
         mutex_unlock(&module_mutex);
 
         /* This may be NULL, but that's OK */
         unset_module_init_ro_nx(mod);
-        module_free(mod, mod->module_init);
+        module_arch_freeing_init(mod);
+        module_memfree(mod->module_init);
         kfree(mod->args);
         percpu_modfree(mod);
 
@@ -1874,7 +1864,7 @@ static void free_module(struct module *mod)
 
         /* Finally, free the core (containing the module structure) */
         unset_module_core_ro_nx(mod);
-        module_free(mod, mod->module_core);
+        module_memfree(mod->module_core);
 
 #ifdef CONFIG_MPU
         update_protections(current->mm);
@@ -1955,7 +1945,7 @@ static int simplify_symbols(struct module *mod, const struct load_info *info)
                         /* We compiled with -fno-common.  These are not
                            supposed to happen.  */
                         pr_debug("Common symbol: %s\n", name);
-                        printk("%s: please compile with -fno-common\n",
+                        pr_warn("%s: please compile with -fno-common\n",
                                mod->name);
                         ret = -ENOEXEC;
                         break;
@@ -2259,7 +2249,7 @@ static char elf_type(const Elf_Sym *sym, const struct load_info *info)
 }
 
 static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs,
-                           unsigned int shnum)
+                        unsigned int shnum)
 {
         const Elf_Shdr *sec;
 
@@ -2735,7 +2725,7 @@ static int find_module_sections(struct module *mod, struct load_info *info)
                  * This shouldn't happen with same compiler and binutils
                  * building all parts of the module.
                  */
-                printk(KERN_WARNING "%s: has both .ctors and .init_array.\n",
+                pr_warn("%s: has both .ctors and .init_array.\n",
                        mod->name);
                 return -EINVAL;
         }
@@ -2809,7 +2799,7 @@ static int move_module(struct module *mod, struct load_info *info)
                  */
                 kmemleak_ignore(ptr);
                 if (!ptr) {
-                        module_free(mod, mod->module_core);
+                        module_memfree(mod->module_core);
                         return -ENOMEM;
                 }
                 memset(ptr, 0, mod->init_size);
@@ -2954,8 +2944,9 @@ static struct module *layout_and_allocate(struct load_info *info, int flags)
 static void module_deallocate(struct module *mod, struct load_info *info)
 {
         percpu_modfree(mod);
-        module_free(mod, mod->module_init);
-        module_free(mod, mod->module_core);
+        module_arch_freeing_init(mod);
+        module_memfree(mod->module_init);
+        module_memfree(mod->module_core);
 }
 
 int __weak module_finalize(const Elf_Ehdr *hdr,
@@ -3007,10 +2998,31 @@ static void do_mod_ctors(struct module *mod)
 #endif
 }
 
+/* For freeing module_init on success, in case kallsyms traversing */
+struct mod_initfree {
+        struct rcu_head rcu;
+        void *module_init;
+};
+
+static void do_free_init(struct rcu_head *head)
+{
+        struct mod_initfree *m = container_of(head, struct mod_initfree, rcu);
+        module_memfree(m->module_init);
+        kfree(m);
+}
+
 /* This is where the real work happens */
 static int do_init_module(struct module *mod)
 {
         int ret = 0;
+        struct mod_initfree *freeinit;
+
+        freeinit = kmalloc(sizeof(*freeinit), GFP_KERNEL);
+        if (!freeinit) {
+                ret = -ENOMEM;
+                goto fail;
+        }
+        freeinit->module_init = mod->module_init;
 
         /*
          * We want to find out whether @mod uses async during init.  Clear
@@ -3023,16 +3035,7 @@ static int do_init_module(struct module *mod)
         if (mod->init != NULL)
                 ret = do_one_initcall(mod->init);
         if (ret < 0) {
-                /* Init routine failed: abort.  Try to protect us from
-                   buggy refcounters. */
-                mod->state = MODULE_STATE_GOING;
-                synchronize_sched();
-                module_put(mod);
-                blocking_notifier_call_chain(&module_notify_list,
-                                             MODULE_STATE_GOING, mod);
-                free_module(mod);
-                wake_up_all(&module_wq);
-                return ret;
+                goto fail_free_freeinit;
         }
         if (ret > 0) {
                 pr_warn("%s: '%s'->init suspiciously returned %d, it should "
@@ -3077,15 +3080,35 @@ static int do_init_module(struct module *mod)
         mod->strtab = mod->core_strtab;
 #endif
         unset_module_init_ro_nx(mod);
-        module_free(mod, mod->module_init);
+        module_arch_freeing_init(mod);
         mod->module_init = NULL;
         mod->init_size = 0;
         mod->init_ro_size = 0;
         mod->init_text_size = 0;
+        /*
+         * We want to free module_init, but be aware that kallsyms may be
+         * walking this with preempt disabled.  In all the failure paths,
+         * we call synchronize_rcu/synchronize_sched, but we don't want
+         * to slow down the success path, so use actual RCU here.
+         */
+        call_rcu(&freeinit->rcu, do_free_init);
         mutex_unlock(&module_mutex);
         wake_up_all(&module_wq);
 
         return 0;
+
+fail_free_freeinit:
+        kfree(freeinit);
+fail:
+        /* Try to protect us from buggy refcounters. */
+        mod->state = MODULE_STATE_GOING;
+        synchronize_sched();
+        module_put(mod);
+        blocking_notifier_call_chain(&module_notify_list,
+                                     MODULE_STATE_GOING, mod);
+        free_module(mod);
+        wake_up_all(&module_wq);
+        return ret;
 }
 
 static int may_init_module(void)
@@ -3097,6 +3120,32 @@ static int may_init_module(void)
 }
 
 /*
+ * Can't use wait_event_interruptible() because our condition
+ * 'finished_loading()' contains a blocking primitive itself (mutex_lock).
+ */
+static int wait_finished_loading(struct module *mod)
+{
+        DEFINE_WAIT_FUNC(wait, woken_wake_function);
+        int ret = 0;
+
+        add_wait_queue(&module_wq, &wait);
+        for (;;) {
+                if (finished_loading(mod->name))
+                        break;
+
+                if (signal_pending(current)) {
+                        ret = -ERESTARTSYS;
+                        break;
+                }
+
+                wait_woken(&wait, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
+        }
+        remove_wait_queue(&module_wq, &wait);
+
+        return ret;
+}
+
+/*
  * We try to place it in the list now to make sure it's unique before
  * we dedicate too many resources.  In particular, temporary percpu
  * memory exhaustion.
@@ -3116,8 +3165,8 @@ again:
                     || old->state == MODULE_STATE_UNFORMED) {
                         /* Wait in case it fails to load. */
                         mutex_unlock(&module_mutex);
-                        err = wait_event_interruptible(module_wq,
-                                               finished_loading(mod->name));
+
+                        err = wait_finished_loading(mod);
                         if (err)
                                 goto out_unlocked;
                         goto again;
@@ -3176,7 +3225,7 @@ out:
 
 static int unknown_module_param_cb(char *param, char *val, const char *modname)
 {
-        /* Check for magic 'dyndbg' arg */ 
+        /* Check for magic 'dyndbg' arg */
         int ret = ddebug_dyndbg_module_param_cb(param, val, modname);
         if (ret != 0)
                 pr_warn("%s: unknown parameter '%s' ignored\n", modname, param);
@@ -3326,6 +3375,8 @@ static int load_module(struct load_info *info, const char __user *uargs,
         /* Unlink carefully: kallsyms could be walking list. */
         list_del_rcu(&mod->list);
         wake_up_all(&module_wq);
+        /* Wait for RCU synchronizing before releasing mod->list. */
+        synchronize_rcu();
         mutex_unlock(&module_mutex);
  free_module:
         module_deallocate(mod, info);
@@ -3659,8 +3710,8 @@ static int m_show(struct seq_file *m, void *p)
 
         /* Informative for users. */
         seq_printf(m, " %s",
-                   mod->state == MODULE_STATE_GOING ? "Unloading":
-                   mod->state == MODULE_STATE_COMING ? "Loading":
+                   mod->state == MODULE_STATE_GOING ? "Unloading" :
+                   mod->state == MODULE_STATE_COMING ? "Loading" :
                    "Live");
         /* Used by oprofile and other similar tools. */
         seq_printf(m, " 0x%pK", mod->module_core);
@@ -3669,7 +3720,7 @@ static int m_show(struct seq_file *m, void *p)
         if (mod->taints)
                 seq_printf(m, " %s", module_flags(mod, buf));
 
-        seq_printf(m, "\n");
+        seq_puts(m, "\n");
         return 0;
 }