aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/module.h16
-rw-r--r--include/trace/events/module.h2
-rw-r--r--kernel/module.c39
3 files changed, 7 insertions, 50 deletions
diff --git a/include/linux/module.h b/include/linux/module.h
index 71f282a4e307..ebfb0e153c6a 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -210,20 +210,6 @@ enum module_state {
210 MODULE_STATE_UNFORMED, /* Still setting it up. */ 210 MODULE_STATE_UNFORMED, /* Still setting it up. */
211}; 211};
212 212
213/**
214 * struct module_ref - per cpu module reference counts
215 * @incs: number of module get on this cpu
216 * @decs: number of module put on this cpu
217 *
218 * We force an alignment on 8 or 16 bytes, so that alloc_percpu()
219 * put @incs/@decs in same cache line, with no extra memory cost,
220 * since alloc_percpu() is fine grained.
221 */
222struct module_ref {
223 unsigned long incs;
224 unsigned long decs;
225} __attribute((aligned(2 * sizeof(unsigned long))));
226
227struct module { 213struct module {
228 enum module_state state; 214 enum module_state state;
229 215
@@ -367,7 +353,7 @@ struct module {
367 /* Destruction function. */ 353 /* Destruction function. */
368 void (*exit)(void); 354 void (*exit)(void);
369 355
370 struct module_ref __percpu *refptr; 356 atomic_t refcnt;
371#endif 357#endif
372 358
373#ifdef CONFIG_CONSTRUCTORS 359#ifdef CONFIG_CONSTRUCTORS
diff --git a/include/trace/events/module.h b/include/trace/events/module.h
index 7c5cbfe3fc49..81c4c183d348 100644
--- a/include/trace/events/module.h
+++ b/include/trace/events/module.h
@@ -80,7 +80,7 @@ DECLARE_EVENT_CLASS(module_refcnt,
80 80
81 TP_fast_assign( 81 TP_fast_assign(
82 __entry->ip = ip; 82 __entry->ip = ip;
83 __entry->refcnt = __this_cpu_read(mod->refptr->incs) - __this_cpu_read(mod->refptr->decs); 83 __entry->refcnt = atomic_read(&mod->refcnt);
84 __assign_str(name, mod->name); 84 __assign_str(name, mod->name);
85 ), 85 ),
86 86
diff --git a/kernel/module.c b/kernel/module.c
index d596a306b0a1..b1d485df5ac1 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -631,15 +631,11 @@ EXPORT_TRACEPOINT_SYMBOL(module_get);
631/* Init the unload section of the module. */ 631/* Init the unload section of the module. */
632static int module_unload_init(struct module *mod) 632static int module_unload_init(struct module *mod)
633{ 633{
634 mod->refptr = alloc_percpu(struct module_ref);
635 if (!mod->refptr)
636 return -ENOMEM;
637
638 INIT_LIST_HEAD(&mod->source_list); 634 INIT_LIST_HEAD(&mod->source_list);
639 INIT_LIST_HEAD(&mod->target_list); 635 INIT_LIST_HEAD(&mod->target_list);
640 636
641 /* Hold reference count during initialization. */ 637 /* Hold reference count during initialization. */
642 raw_cpu_write(mod->refptr->incs, 1); 638 atomic_set(&mod->refcnt, 1);
643 639
644 return 0; 640 return 0;
645} 641}
@@ -721,8 +717,6 @@ static void module_unload_free(struct module *mod)
721 kfree(use); 717 kfree(use);
722 } 718 }
723 mutex_unlock(&module_mutex); 719 mutex_unlock(&module_mutex);
724
725 free_percpu(mod->refptr);
726} 720}
727 721
728#ifdef CONFIG_MODULE_FORCE_UNLOAD 722#ifdef CONFIG_MODULE_FORCE_UNLOAD
@@ -772,28 +766,7 @@ static int try_stop_module(struct module *mod, int flags, int *forced)
772 766
773unsigned long module_refcount(struct module *mod) 767unsigned long module_refcount(struct module *mod)
774{ 768{
775 unsigned long incs = 0, decs = 0; 769 return (unsigned long)atomic_read(&mod->refcnt);
776 int cpu;
777
778 for_each_possible_cpu(cpu)
779 decs += per_cpu_ptr(mod->refptr, cpu)->decs;
780 /*
781 * ensure the incs are added up after the decs.
782 * module_put ensures incs are visible before decs with smp_wmb.
783 *
784 * This 2-count scheme avoids the situation where the refcount
785 * for CPU0 is read, then CPU0 increments the module refcount,
786 * then CPU1 drops that refcount, then the refcount for CPU1 is
787 * read. We would record a decrement but not its corresponding
788 * increment so we would see a low count (disaster).
789 *
790 * Rare situation? But module_refcount can be preempted, and we
791 * might be tallying up 4096+ CPUs. So it is not impossible.
792 */
793 smp_rmb();
794 for_each_possible_cpu(cpu)
795 incs += per_cpu_ptr(mod->refptr, cpu)->incs;
796 return incs - decs;
797} 770}
798EXPORT_SYMBOL(module_refcount); 771EXPORT_SYMBOL(module_refcount);
799 772
@@ -935,7 +908,7 @@ void __module_get(struct module *module)
935{ 908{
936 if (module) { 909 if (module) {
937 preempt_disable(); 910 preempt_disable();
938 __this_cpu_inc(module->refptr->incs); 911 atomic_inc(&module->refcnt);
939 trace_module_get(module, _RET_IP_); 912 trace_module_get(module, _RET_IP_);
940 preempt_enable(); 913 preempt_enable();
941 } 914 }
@@ -950,7 +923,7 @@ bool try_module_get(struct module *module)
950 preempt_disable(); 923 preempt_disable();
951 924
952 if (likely(module_is_live(module))) { 925 if (likely(module_is_live(module))) {
953 __this_cpu_inc(module->refptr->incs); 926 atomic_inc(&module->refcnt);
954 trace_module_get(module, _RET_IP_); 927 trace_module_get(module, _RET_IP_);
955 } else 928 } else
956 ret = false; 929 ret = false;
@@ -965,9 +938,7 @@ void module_put(struct module *module)
965{ 938{
966 if (module) { 939 if (module) {
967 preempt_disable(); 940 preempt_disable();
968 smp_wmb(); /* see comment in module_refcount */ 941 atomic_dec(&module->refcnt);
969 __this_cpu_inc(module->refptr->decs);
970
971 trace_module_put(module, _RET_IP_); 942 trace_module_put(module, _RET_IP_);
972 preempt_enable(); 943 preempt_enable();
973 } 944 }