module: Use this_cpu_xx to dynamically allocate counters

Use cpu ops to deal with the per cpu data instead of a local_t. Reduces memory
requirements, cache footprint and decreases cycle counts.

The this_cpu_xx operations are also used for !SMP mode. Otherwise we could
not drop the use of __module_ref_addr() which would make per cpu data handling
complicated. this_cpu_xx operations have their own fallback for !SMP.

V8-V9:
- Leave include asm/module.h since ringbuffer.c depends on it. Nothing else
  does though. Another patch will deal with that.
- Remove spurious free.

Signed-off-by: Christoph Lameter <cl@linux-foundation.org>
Acked-by: Rusty Russell <rusty@rustcorp.com.au>
Signed-off-by: Tejun Heo <tj@kernel.org>

1 files changed, 14 insertions, 22 deletions

diff --git a/include/linux/module.h b/include/linux/module.h
index 6cb1a3cab5d3..2302f09ea2d9 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -18,6 +18,7 @@
 #include <linux/tracepoint.h>
 
 #include <asm/local.h>
+#include <linux/percpu.h>
 #include <asm/module.h>
 
 #include <trace/events/module.h>
@@ -363,11 +364,9 @@ struct module
         /* Destruction function. */
         void (*exit)(void);
 
-#ifdef CONFIG_SMP
-        char *refptr;
-#else
-        local_t ref;
-#endif
+        struct module_ref {
+                int count;
+        } *refptr;
 #endif
 
 #ifdef CONFIG_CONSTRUCTORS
@@ -454,25 +453,16 @@ void __symbol_put(const char *symbol);
 #define symbol_put(x) __symbol_put(MODULE_SYMBOL_PREFIX #x)
 void symbol_put_addr(void *addr);
 
-static inline local_t *__module_ref_addr(struct module *mod, int cpu)
-{
-#ifdef CONFIG_SMP
-        return (local_t *) (mod->refptr + per_cpu_offset(cpu));
-#else
-        return &mod->ref;
-#endif
-}
-
 /* Sometimes we know we already have a refcount, and it's easier not
    to handle the error case (which only happens with rmmod --wait). */
 static inline void __module_get(struct module *module)
 {
         if (module) {
-                unsigned int cpu = get_cpu();
-                local_inc(__module_ref_addr(module, cpu));
+                preempt_disable();
+                __this_cpu_inc(module->refptr->count);
                 trace_module_get(module, _THIS_IP_,
-                                 local_read(__module_ref_addr(module, cpu)));
-                put_cpu();
+                                 __this_cpu_read(module->refptr->count));
+                preempt_enable();
         }
 }
 
@@ -481,15 +471,17 @@ static inline int try_module_get(struct module *module)
         int ret = 1;
 
         if (module) {
-                unsigned int cpu = get_cpu();
+                preempt_disable();
+
                 if (likely(module_is_live(module))) {
-                        local_inc(__module_ref_addr(module, cpu));
+                        __this_cpu_inc(module->refptr->count);
                         trace_module_get(module, _THIS_IP_,
-                                local_read(__module_ref_addr(module, cpu)));
+                                __this_cpu_read(module->refptr->count));
                 }
                 else
                         ret = 0;
-                put_cpu();
+
+                preempt_enable();
         }
         return ret;
 }