2 files changed, 34 insertions, 15 deletions

diff --git a/include/linux/module.h b/include/linux/module.h
index 8bd399a00343..515d53ae6a79 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -368,7 +368,8 @@ struct module
         void (*exit)(void);
 
         struct module_ref {
-                int count;
+                unsigned int incs;
+                unsigned int decs;
         } __percpu *refptr;
 #endif
 
@@ -463,9 +464,9 @@ static inline void __module_get(struct module *module)
 {
         if (module) {
                 preempt_disable();
-                __this_cpu_inc(module->refptr->count);
+                __this_cpu_inc(module->refptr->incs);
                 trace_module_get(module, _THIS_IP_,
-                                 __this_cpu_read(module->refptr->count));
+                                 __this_cpu_read(module->refptr->incs));
                 preempt_enable();
         }
 }
@@ -478,11 +479,10 @@ static inline int try_module_get(struct module *module)
                 preempt_disable();
 
                 if (likely(module_is_live(module))) {
-                        __this_cpu_inc(module->refptr->count);
+                        __this_cpu_inc(module->refptr->incs);
                         trace_module_get(module, _THIS_IP_,
-                                __this_cpu_read(module->refptr->count));
-                }
-                else
+                                __this_cpu_read(module->refptr->incs));
+                } else
                         ret = 0;
 
                 preempt_enable();
diff --git a/kernel/module.c b/kernel/module.c
index 9f8d23d8b3a8..1016b75b026a 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -521,11 +521,13 @@ static void module_unload_init(struct module *mod)
         int cpu;
 
         INIT_LIST_HEAD(&mod->modules_which_use_me);
-        for_each_possible_cpu(cpu)
-                per_cpu_ptr(mod->refptr, cpu)->count = 0;
+        for_each_possible_cpu(cpu) {
+                per_cpu_ptr(mod->refptr, cpu)->incs = 0;
+                per_cpu_ptr(mod->refptr, cpu)->decs = 0;
+        }
 
         /* Hold reference count during initialization. */
-        __this_cpu_write(mod->refptr->count, 1);
+        __this_cpu_write(mod->refptr->incs, 1);
         /* Backwards compatibility macros put refcount during init. */
         mod->waiter = current;
 }
@@ -664,12 +666,28 @@ static int try_stop_module(struct module *mod, int flags, int *forced)
 
 unsigned int module_refcount(struct module *mod)
 {
-        unsigned int total = 0;
+        unsigned int incs = 0, decs = 0;
         int cpu;
 
         for_each_possible_cpu(cpu)
-                total += per_cpu_ptr(mod->refptr, cpu)->count;
-        return total;
+                decs += per_cpu_ptr(mod->refptr, cpu)->decs;
+        /*
+         * ensure the incs are added up after the decs.
+         * module_put ensures incs are visible before decs with smp_wmb.
+         *
+         * This 2-count scheme avoids the situation where the refcount
+         * for CPU0 is read, then CPU0 increments the module refcount,
+         * then CPU1 drops that refcount, then the refcount for CPU1 is
+         * read. We would record a decrement but not its corresponding
+         * increment so we would see a low count (disaster).
+         *
+         * Rare situation? But module_refcount can be preempted, and we
+         * might be tallying up 4096+ CPUs. So it is not impossible.
+         */
+        smp_rmb();
+        for_each_possible_cpu(cpu)
+                incs += per_cpu_ptr(mod->refptr, cpu)->incs;
+        return incs - decs;
 }
 EXPORT_SYMBOL(module_refcount);
 
@@ -846,10 +864,11 @@ void module_put(struct module *module)
 {
         if (module) {
                 preempt_disable();
-                __this_cpu_dec(module->refptr->count);
+                smp_wmb(); /* see comment in module_refcount */
+                __this_cpu_inc(module->refptr->decs);
 
                 trace_module_put(module, _RET_IP_,
-                                 __this_cpu_read(module->refptr->count));
+                                 __this_cpu_read(module->refptr->decs));
                 /* Maybe they're waiting for us to drop reference? */
                 if (unlikely(!module_is_live(module)))
                         wake_up_process(module->waiter);