aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/module.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/module.c')
-rw-r--r--kernel/module.c1702
1 files changed, 1065 insertions, 637 deletions
diff --git a/kernel/module.c b/kernel/module.c
index 5daf0abd63c1..34e00b708fad 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -1,6 +1,6 @@
1/* 1/*
2 Copyright (C) 2002 Richard Henderson 2 Copyright (C) 2002 Richard Henderson
3 Copyright (C) 2001 Rusty Russell, 2002 Rusty Russell IBM. 3 Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
4 4
5 This program is free software; you can redistribute it and/or modify 5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by 6 it under the terms of the GNU General Public License as published by
@@ -55,12 +55,12 @@
55#include <linux/async.h> 55#include <linux/async.h>
56#include <linux/percpu.h> 56#include <linux/percpu.h>
57#include <linux/kmemleak.h> 57#include <linux/kmemleak.h>
58#include <linux/jump_label.h>
59#include <linux/pfn.h>
58 60
59#define CREATE_TRACE_POINTS 61#define CREATE_TRACE_POINTS
60#include <trace/events/module.h> 62#include <trace/events/module.h>
61 63
62EXPORT_TRACEPOINT_SYMBOL(module_get);
63
64#if 0 64#if 0
65#define DEBUGP printk 65#define DEBUGP printk
66#else 66#else
@@ -71,14 +71,42 @@ EXPORT_TRACEPOINT_SYMBOL(module_get);
71#define ARCH_SHF_SMALL 0 71#define ARCH_SHF_SMALL 0
72#endif 72#endif
73 73
74/*
75 * Modules' sections will be aligned on page boundaries
76 * to ensure complete separation of code and data, but
77 * only when CONFIG_DEBUG_SET_MODULE_RONX=y
78 */
79#ifdef CONFIG_DEBUG_SET_MODULE_RONX
80# define debug_align(X) ALIGN(X, PAGE_SIZE)
81#else
82# define debug_align(X) (X)
83#endif
84
85/*
86 * Given BASE and SIZE this macro calculates the number of pages the
87 * memory regions occupies
88 */
89#define MOD_NUMBER_OF_PAGES(BASE, SIZE) (((SIZE) > 0) ? \
90 (PFN_DOWN((unsigned long)(BASE) + (SIZE) - 1) - \
91 PFN_DOWN((unsigned long)BASE) + 1) \
92 : (0UL))
93
74/* If this is set, the section belongs in the init part of the module */ 94/* If this is set, the section belongs in the init part of the module */
75#define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG-1)) 95#define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG-1))
76 96
77/* List of modules, protected by module_mutex or preempt_disable 97/*
98 * Mutex protects:
99 * 1) List of modules (also safely readable with preempt_disable),
100 * 2) module_use links,
101 * 3) module_addr_min/module_addr_max.
78 * (delete uses stop_machine/add uses RCU list operations). */ 102 * (delete uses stop_machine/add uses RCU list operations). */
79DEFINE_MUTEX(module_mutex); 103DEFINE_MUTEX(module_mutex);
80EXPORT_SYMBOL_GPL(module_mutex); 104EXPORT_SYMBOL_GPL(module_mutex);
81static LIST_HEAD(modules); 105static LIST_HEAD(modules);
106#ifdef CONFIG_KGDB_KDB
107struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */
108#endif /* CONFIG_KGDB_KDB */
109
82 110
83/* Block module loading/unloading? */ 111/* Block module loading/unloading? */
84int modules_disabled = 0; 112int modules_disabled = 0;
@@ -88,7 +116,8 @@ static DECLARE_WAIT_QUEUE_HEAD(module_wq);
88 116
89static BLOCKING_NOTIFIER_HEAD(module_notify_list); 117static BLOCKING_NOTIFIER_HEAD(module_notify_list);
90 118
91/* Bounds of module allocation, for speeding __module_address */ 119/* Bounds of module allocation, for speeding __module_address.
120 * Protected by module_mutex. */
92static unsigned long module_addr_min = -1UL, module_addr_max = 0; 121static unsigned long module_addr_min = -1UL, module_addr_max = 0;
93 122
94int register_module_notifier(struct notifier_block * nb) 123int register_module_notifier(struct notifier_block * nb)
@@ -103,6 +132,20 @@ int unregister_module_notifier(struct notifier_block * nb)
103} 132}
104EXPORT_SYMBOL(unregister_module_notifier); 133EXPORT_SYMBOL(unregister_module_notifier);
105 134
135struct load_info {
136 Elf_Ehdr *hdr;
137 unsigned long len;
138 Elf_Shdr *sechdrs;
139 char *secstrings, *strtab;
140 unsigned long *strmap;
141 unsigned long symoffs, stroffs;
142 struct _ddebug *debug;
143 unsigned int num_debug;
144 struct {
145 unsigned int sym, str, mod, vers, info, pcpu;
146 } index;
147};
148
106/* We require a truly strong try_module_get(): 0 means failure due to 149/* We require a truly strong try_module_get(): 0 means failure due to
107 ongoing or failed initialization etc. */ 150 ongoing or failed initialization etc. */
108static inline int strong_try_module_get(struct module *mod) 151static inline int strong_try_module_get(struct module *mod)
@@ -133,42 +176,38 @@ void __module_put_and_exit(struct module *mod, long code)
133EXPORT_SYMBOL(__module_put_and_exit); 176EXPORT_SYMBOL(__module_put_and_exit);
134 177
135/* Find a module section: 0 means not found. */ 178/* Find a module section: 0 means not found. */
136static unsigned int find_sec(Elf_Ehdr *hdr, 179static unsigned int find_sec(const struct load_info *info, const char *name)
137 Elf_Shdr *sechdrs,
138 const char *secstrings,
139 const char *name)
140{ 180{
141 unsigned int i; 181 unsigned int i;
142 182
143 for (i = 1; i < hdr->e_shnum; i++) 183 for (i = 1; i < info->hdr->e_shnum; i++) {
184 Elf_Shdr *shdr = &info->sechdrs[i];
144 /* Alloc bit cleared means "ignore it." */ 185 /* Alloc bit cleared means "ignore it." */
145 if ((sechdrs[i].sh_flags & SHF_ALLOC) 186 if ((shdr->sh_flags & SHF_ALLOC)
146 && strcmp(secstrings+sechdrs[i].sh_name, name) == 0) 187 && strcmp(info->secstrings + shdr->sh_name, name) == 0)
147 return i; 188 return i;
189 }
148 return 0; 190 return 0;
149} 191}
150 192
151/* Find a module section, or NULL. */ 193/* Find a module section, or NULL. */
152static void *section_addr(Elf_Ehdr *hdr, Elf_Shdr *shdrs, 194static void *section_addr(const struct load_info *info, const char *name)
153 const char *secstrings, const char *name)
154{ 195{
155 /* Section 0 has sh_addr 0. */ 196 /* Section 0 has sh_addr 0. */
156 return (void *)shdrs[find_sec(hdr, shdrs, secstrings, name)].sh_addr; 197 return (void *)info->sechdrs[find_sec(info, name)].sh_addr;
157} 198}
158 199
159/* Find a module section, or NULL. Fill in number of "objects" in section. */ 200/* Find a module section, or NULL. Fill in number of "objects" in section. */
160static void *section_objs(Elf_Ehdr *hdr, 201static void *section_objs(const struct load_info *info,
161 Elf_Shdr *sechdrs,
162 const char *secstrings,
163 const char *name, 202 const char *name,
164 size_t object_size, 203 size_t object_size,
165 unsigned int *num) 204 unsigned int *num)
166{ 205{
167 unsigned int sec = find_sec(hdr, sechdrs, secstrings, name); 206 unsigned int sec = find_sec(info, name);
168 207
169 /* Section 0 has sh_addr 0 and sh_size 0. */ 208 /* Section 0 has sh_addr 0 and sh_size 0. */
170 *num = sechdrs[sec].sh_size / object_size; 209 *num = info->sechdrs[sec].sh_size / object_size;
171 return (void *)sechdrs[sec].sh_addr; 210 return (void *)info->sechdrs[sec].sh_addr;
172} 211}
173 212
174/* Provided by the linker */ 213/* Provided by the linker */
@@ -178,8 +217,6 @@ extern const struct kernel_symbol __start___ksymtab_gpl[];
178extern const struct kernel_symbol __stop___ksymtab_gpl[]; 217extern const struct kernel_symbol __stop___ksymtab_gpl[];
179extern const struct kernel_symbol __start___ksymtab_gpl_future[]; 218extern const struct kernel_symbol __start___ksymtab_gpl_future[];
180extern const struct kernel_symbol __stop___ksymtab_gpl_future[]; 219extern const struct kernel_symbol __stop___ksymtab_gpl_future[];
181extern const struct kernel_symbol __start___ksymtab_gpl_future[];
182extern const struct kernel_symbol __stop___ksymtab_gpl_future[];
183extern const unsigned long __start___kcrctab[]; 220extern const unsigned long __start___kcrctab[];
184extern const unsigned long __start___kcrctab_gpl[]; 221extern const unsigned long __start___kcrctab_gpl[];
185extern const unsigned long __start___kcrctab_gpl_future[]; 222extern const unsigned long __start___kcrctab_gpl_future[];
@@ -222,7 +259,7 @@ bool each_symbol(bool (*fn)(const struct symsearch *arr, struct module *owner,
222 unsigned int symnum, void *data), void *data) 259 unsigned int symnum, void *data), void *data)
223{ 260{
224 struct module *mod; 261 struct module *mod;
225 const struct symsearch arr[] = { 262 static const struct symsearch arr[] = {
226 { __start___ksymtab, __stop___ksymtab, __start___kcrctab, 263 { __start___ksymtab, __stop___ksymtab, __start___kcrctab,
227 NOT_GPL_ONLY, false }, 264 NOT_GPL_ONLY, false },
228 { __start___ksymtab_gpl, __stop___ksymtab_gpl, 265 { __start___ksymtab_gpl, __stop___ksymtab_gpl,
@@ -329,7 +366,7 @@ static bool find_symbol_in_section(const struct symsearch *syms,
329} 366}
330 367
331/* Find a symbol and return it, along with, (optional) crc and 368/* Find a symbol and return it, along with, (optional) crc and
332 * (optional) module which owns it */ 369 * (optional) module which owns it. Needs preempt disabled or module_mutex. */
333const struct kernel_symbol *find_symbol(const char *name, 370const struct kernel_symbol *find_symbol(const char *name,
334 struct module **owner, 371 struct module **owner,
335 const unsigned long **crc, 372 const unsigned long **crc,
@@ -370,67 +407,112 @@ EXPORT_SYMBOL_GPL(find_module);
370 407
371#ifdef CONFIG_SMP 408#ifdef CONFIG_SMP
372 409
373static void *percpu_modalloc(unsigned long size, unsigned long align, 410static inline void __percpu *mod_percpu(struct module *mod)
374 const char *name)
375{ 411{
376 void *ptr; 412 return mod->percpu;
413}
377 414
415static int percpu_modalloc(struct module *mod,
416 unsigned long size, unsigned long align)
417{
378 if (align > PAGE_SIZE) { 418 if (align > PAGE_SIZE) {
379 printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n", 419 printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n",
380 name, align, PAGE_SIZE); 420 mod->name, align, PAGE_SIZE);
381 align = PAGE_SIZE; 421 align = PAGE_SIZE;
382 } 422 }
383 423
384 ptr = __alloc_reserved_percpu(size, align); 424 mod->percpu = __alloc_reserved_percpu(size, align);
385 if (!ptr) 425 if (!mod->percpu) {
386 printk(KERN_WARNING 426 printk(KERN_WARNING
387 "Could not allocate %lu bytes percpu data\n", size); 427 "%s: Could not allocate %lu bytes percpu data\n",
388 return ptr; 428 mod->name, size);
429 return -ENOMEM;
430 }
431 mod->percpu_size = size;
432 return 0;
389} 433}
390 434
391static void percpu_modfree(void *freeme) 435static void percpu_modfree(struct module *mod)
392{ 436{
393 free_percpu(freeme); 437 free_percpu(mod->percpu);
394} 438}
395 439
396static unsigned int find_pcpusec(Elf_Ehdr *hdr, 440static unsigned int find_pcpusec(struct load_info *info)
397 Elf_Shdr *sechdrs,
398 const char *secstrings)
399{ 441{
400 return find_sec(hdr, sechdrs, secstrings, ".data..percpu"); 442 return find_sec(info, ".data..percpu");
401} 443}
402 444
403static void percpu_modcopy(void *pcpudest, const void *from, unsigned long size) 445static void percpu_modcopy(struct module *mod,
446 const void *from, unsigned long size)
404{ 447{
405 int cpu; 448 int cpu;
406 449
407 for_each_possible_cpu(cpu) 450 for_each_possible_cpu(cpu)
408 memcpy(pcpudest + per_cpu_offset(cpu), from, size); 451 memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
452}
453
454/**
455 * is_module_percpu_address - test whether address is from module static percpu
456 * @addr: address to test
457 *
458 * Test whether @addr belongs to module static percpu area.
459 *
460 * RETURNS:
461 * %true if @addr is from module static percpu area
462 */
463bool is_module_percpu_address(unsigned long addr)
464{
465 struct module *mod;
466 unsigned int cpu;
467
468 preempt_disable();
469
470 list_for_each_entry_rcu(mod, &modules, list) {
471 if (!mod->percpu_size)
472 continue;
473 for_each_possible_cpu(cpu) {
474 void *start = per_cpu_ptr(mod->percpu, cpu);
475
476 if ((void *)addr >= start &&
477 (void *)addr < start + mod->percpu_size) {
478 preempt_enable();
479 return true;
480 }
481 }
482 }
483
484 preempt_enable();
485 return false;
409} 486}
410 487
411#else /* ... !CONFIG_SMP */ 488#else /* ... !CONFIG_SMP */
412 489
413static inline void *percpu_modalloc(unsigned long size, unsigned long align, 490static inline void __percpu *mod_percpu(struct module *mod)
414 const char *name)
415{ 491{
416 return NULL; 492 return NULL;
417} 493}
418static inline void percpu_modfree(void *pcpuptr) 494static inline int percpu_modalloc(struct module *mod,
495 unsigned long size, unsigned long align)
496{
497 return -ENOMEM;
498}
499static inline void percpu_modfree(struct module *mod)
419{ 500{
420 BUG();
421} 501}
422static inline unsigned int find_pcpusec(Elf_Ehdr *hdr, 502static unsigned int find_pcpusec(struct load_info *info)
423 Elf_Shdr *sechdrs,
424 const char *secstrings)
425{ 503{
426 return 0; 504 return 0;
427} 505}
428static inline void percpu_modcopy(void *pcpudst, const void *src, 506static inline void percpu_modcopy(struct module *mod,
429 unsigned long size) 507 const void *from, unsigned long size)
430{ 508{
431 /* pcpusec should be 0, and size of that section should be 0. */ 509 /* pcpusec should be 0, and size of that section should be 0. */
432 BUG_ON(size != 0); 510 BUG_ON(size != 0);
433} 511}
512bool is_module_percpu_address(unsigned long addr)
513{
514 return false;
515}
434 516
435#endif /* CONFIG_SMP */ 517#endif /* CONFIG_SMP */
436 518
@@ -467,34 +549,34 @@ MODINFO_ATTR(srcversion);
467static char last_unloaded_module[MODULE_NAME_LEN+1]; 549static char last_unloaded_module[MODULE_NAME_LEN+1];
468 550
469#ifdef CONFIG_MODULE_UNLOAD 551#ifdef CONFIG_MODULE_UNLOAD
552
553EXPORT_TRACEPOINT_SYMBOL(module_get);
554
470/* Init the unload section of the module. */ 555/* Init the unload section of the module. */
471static void module_unload_init(struct module *mod) 556static int module_unload_init(struct module *mod)
472{ 557{
473 int cpu; 558 mod->refptr = alloc_percpu(struct module_ref);
559 if (!mod->refptr)
560 return -ENOMEM;
561
562 INIT_LIST_HEAD(&mod->source_list);
563 INIT_LIST_HEAD(&mod->target_list);
474 564
475 INIT_LIST_HEAD(&mod->modules_which_use_me);
476 for_each_possible_cpu(cpu)
477 local_set(__module_ref_addr(mod, cpu), 0);
478 /* Hold reference count during initialization. */ 565 /* Hold reference count during initialization. */
479 local_set(__module_ref_addr(mod, raw_smp_processor_id()), 1); 566 __this_cpu_write(mod->refptr->incs, 1);
480 /* Backwards compatibility macros put refcount during init. */ 567 /* Backwards compatibility macros put refcount during init. */
481 mod->waiter = current; 568 mod->waiter = current;
482}
483 569
484/* modules using other modules */ 570 return 0;
485struct module_use 571}
486{
487 struct list_head list;
488 struct module *module_which_uses;
489};
490 572
491/* Does a already use b? */ 573/* Does a already use b? */
492static int already_uses(struct module *a, struct module *b) 574static int already_uses(struct module *a, struct module *b)
493{ 575{
494 struct module_use *use; 576 struct module_use *use;
495 577
496 list_for_each_entry(use, &b->modules_which_use_me, list) { 578 list_for_each_entry(use, &b->source_list, source_list) {
497 if (use->module_which_uses == a) { 579 if (use->source == a) {
498 DEBUGP("%s uses %s!\n", a->name, b->name); 580 DEBUGP("%s uses %s!\n", a->name, b->name);
499 return 1; 581 return 1;
500 } 582 }
@@ -503,62 +585,70 @@ static int already_uses(struct module *a, struct module *b)
503 return 0; 585 return 0;
504} 586}
505 587
506/* Module a uses b */ 588/*
507int use_module(struct module *a, struct module *b) 589 * Module a uses b
590 * - we add 'a' as a "source", 'b' as a "target" of module use
591 * - the module_use is added to the list of 'b' sources (so
592 * 'b' can walk the list to see who sourced them), and of 'a'
593 * targets (so 'a' can see what modules it targets).
594 */
595static int add_module_usage(struct module *a, struct module *b)
508{ 596{
509 struct module_use *use; 597 struct module_use *use;
510 int no_warn, err;
511 598
512 if (b == NULL || already_uses(a, b)) return 1; 599 DEBUGP("Allocating new usage for %s.\n", a->name);
600 use = kmalloc(sizeof(*use), GFP_ATOMIC);
601 if (!use) {
602 printk(KERN_WARNING "%s: out of memory loading\n", a->name);
603 return -ENOMEM;
604 }
605
606 use->source = a;
607 use->target = b;
608 list_add(&use->source_list, &b->source_list);
609 list_add(&use->target_list, &a->target_list);
610 return 0;
611}
612
613/* Module a uses b: caller needs module_mutex() */
614int ref_module(struct module *a, struct module *b)
615{
616 int err;
513 617
514 /* If we're interrupted or time out, we fail. */ 618 if (b == NULL || already_uses(a, b))
515 if (wait_event_interruptible_timeout(
516 module_wq, (err = strong_try_module_get(b)) != -EBUSY,
517 30 * HZ) <= 0) {
518 printk("%s: gave up waiting for init of module %s.\n",
519 a->name, b->name);
520 return 0; 619 return 0;
521 }
522 620
523 /* If strong_try_module_get() returned a different error, we fail. */ 621 /* If module isn't available, we fail. */
622 err = strong_try_module_get(b);
524 if (err) 623 if (err)
525 return 0; 624 return err;
526 625
527 DEBUGP("Allocating new usage for %s.\n", a->name); 626 err = add_module_usage(a, b);
528 use = kmalloc(sizeof(*use), GFP_ATOMIC); 627 if (err) {
529 if (!use) {
530 printk("%s: out of memory loading\n", a->name);
531 module_put(b); 628 module_put(b);
532 return 0; 629 return err;
533 } 630 }
534 631 return 0;
535 use->module_which_uses = a;
536 list_add(&use->list, &b->modules_which_use_me);
537 no_warn = sysfs_create_link(b->holders_dir, &a->mkobj.kobj, a->name);
538 return 1;
539} 632}
540EXPORT_SYMBOL_GPL(use_module); 633EXPORT_SYMBOL_GPL(ref_module);
541 634
542/* Clear the unload stuff of the module. */ 635/* Clear the unload stuff of the module. */
543static void module_unload_free(struct module *mod) 636static void module_unload_free(struct module *mod)
544{ 637{
545 struct module *i; 638 struct module_use *use, *tmp;
546
547 list_for_each_entry(i, &modules, list) {
548 struct module_use *use;
549 639
550 list_for_each_entry(use, &i->modules_which_use_me, list) { 640 mutex_lock(&module_mutex);
551 if (use->module_which_uses == mod) { 641 list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {
552 DEBUGP("%s unusing %s\n", mod->name, i->name); 642 struct module *i = use->target;
553 module_put(i); 643 DEBUGP("%s unusing %s\n", mod->name, i->name);
554 list_del(&use->list); 644 module_put(i);
555 kfree(use); 645 list_del(&use->source_list);
556 sysfs_remove_link(i->holders_dir, mod->name); 646 list_del(&use->target_list);
557 /* There can be at most one match. */ 647 kfree(use);
558 break;
559 }
560 }
561 } 648 }
649 mutex_unlock(&module_mutex);
650
651 free_percpu(mod->refptr);
562} 652}
563 653
564#ifdef CONFIG_MODULE_FORCE_UNLOAD 654#ifdef CONFIG_MODULE_FORCE_UNLOAD
@@ -615,12 +705,28 @@ static int try_stop_module(struct module *mod, int flags, int *forced)
615 705
616unsigned int module_refcount(struct module *mod) 706unsigned int module_refcount(struct module *mod)
617{ 707{
618 unsigned int total = 0; 708 unsigned int incs = 0, decs = 0;
619 int cpu; 709 int cpu;
620 710
621 for_each_possible_cpu(cpu) 711 for_each_possible_cpu(cpu)
622 total += local_read(__module_ref_addr(mod, cpu)); 712 decs += per_cpu_ptr(mod->refptr, cpu)->decs;
623 return total; 713 /*
714 * ensure the incs are added up after the decs.
715 * module_put ensures incs are visible before decs with smp_wmb.
716 *
717 * This 2-count scheme avoids the situation where the refcount
718 * for CPU0 is read, then CPU0 increments the module refcount,
719 * then CPU1 drops that refcount, then the refcount for CPU1 is
720 * read. We would record a decrement but not its corresponding
721 * increment so we would see a low count (disaster).
722 *
723 * Rare situation? But module_refcount can be preempted, and we
724 * might be tallying up 4096+ CPUs. So it is not impossible.
725 */
726 smp_rmb();
727 for_each_possible_cpu(cpu)
728 incs += per_cpu_ptr(mod->refptr, cpu)->incs;
729 return incs - decs;
624} 730}
625EXPORT_SYMBOL(module_refcount); 731EXPORT_SYMBOL(module_refcount);
626 732
@@ -656,16 +762,8 @@ SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
656 return -EFAULT; 762 return -EFAULT;
657 name[MODULE_NAME_LEN-1] = '\0'; 763 name[MODULE_NAME_LEN-1] = '\0';
658 764
659 /* Create stop_machine threads since free_module relies on 765 if (mutex_lock_interruptible(&module_mutex) != 0)
660 * a non-failing stop_machine call. */ 766 return -EINTR;
661 ret = stop_machine_create();
662 if (ret)
663 return ret;
664
665 if (mutex_lock_interruptible(&module_mutex) != 0) {
666 ret = -EINTR;
667 goto out_stop;
668 }
669 767
670 mod = find_module(name); 768 mod = find_module(name);
671 if (!mod) { 769 if (!mod) {
@@ -673,7 +771,7 @@ SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
673 goto out; 771 goto out;
674 } 772 }
675 773
676 if (!list_empty(&mod->modules_which_use_me)) { 774 if (!list_empty(&mod->source_list)) {
677 /* Other modules depend on us: get rid of them first. */ 775 /* Other modules depend on us: get rid of them first. */
678 ret = -EWOULDBLOCK; 776 ret = -EWOULDBLOCK;
679 goto out; 777 goto out;
@@ -717,16 +815,14 @@ SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
717 blocking_notifier_call_chain(&module_notify_list, 815 blocking_notifier_call_chain(&module_notify_list,
718 MODULE_STATE_GOING, mod); 816 MODULE_STATE_GOING, mod);
719 async_synchronize_full(); 817 async_synchronize_full();
720 mutex_lock(&module_mutex); 818
721 /* Store the name of the last unloaded module for diagnostic purposes */ 819 /* Store the name of the last unloaded module for diagnostic purposes */
722 strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module)); 820 strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module));
723 ddebug_remove_module(mod->name);
724 free_module(mod);
725 821
726 out: 822 free_module(mod);
823 return 0;
824out:
727 mutex_unlock(&module_mutex); 825 mutex_unlock(&module_mutex);
728out_stop:
729 stop_machine_destroy();
730 return ret; 826 return ret;
731} 827}
732 828
@@ -739,9 +835,9 @@ static inline void print_unload_info(struct seq_file *m, struct module *mod)
739 835
740 /* Always include a trailing , so userspace can differentiate 836 /* Always include a trailing , so userspace can differentiate
741 between this and the old multi-field proc format. */ 837 between this and the old multi-field proc format. */
742 list_for_each_entry(use, &mod->modules_which_use_me, list) { 838 list_for_each_entry(use, &mod->source_list, source_list) {
743 printed_something = 1; 839 printed_something = 1;
744 seq_printf(m, "%s,", use->module_which_uses->name); 840 seq_printf(m, "%s,", use->source->name);
745 } 841 }
746 842
747 if (mod->init != NULL && mod->exit == NULL) { 843 if (mod->init != NULL && mod->exit == NULL) {
@@ -796,14 +892,15 @@ static struct module_attribute refcnt = {
796void module_put(struct module *module) 892void module_put(struct module *module)
797{ 893{
798 if (module) { 894 if (module) {
799 unsigned int cpu = get_cpu(); 895 preempt_disable();
800 local_dec(__module_ref_addr(module, cpu)); 896 smp_wmb(); /* see comment in module_refcount */
801 trace_module_put(module, _RET_IP_, 897 __this_cpu_inc(module->refptr->decs);
802 local_read(__module_ref_addr(module, cpu))); 898
899 trace_module_put(module, _RET_IP_);
803 /* Maybe they're waiting for us to drop reference? */ 900 /* Maybe they're waiting for us to drop reference? */
804 if (unlikely(!module_is_live(module))) 901 if (unlikely(!module_is_live(module)))
805 wake_up_process(module->waiter); 902 wake_up_process(module->waiter);
806 put_cpu(); 903 preempt_enable();
807 } 904 }
808} 905}
809EXPORT_SYMBOL(module_put); 906EXPORT_SYMBOL(module_put);
@@ -819,14 +916,15 @@ static inline void module_unload_free(struct module *mod)
819{ 916{
820} 917}
821 918
822int use_module(struct module *a, struct module *b) 919int ref_module(struct module *a, struct module *b)
823{ 920{
824 return strong_try_module_get(b) == 0; 921 return strong_try_module_get(b);
825} 922}
826EXPORT_SYMBOL_GPL(use_module); 923EXPORT_SYMBOL_GPL(ref_module);
827 924
828static inline void module_unload_init(struct module *mod) 925static inline int module_unload_init(struct module *mod)
829{ 926{
927 return 0;
830} 928}
831#endif /* CONFIG_MODULE_UNLOAD */ 929#endif /* CONFIG_MODULE_UNLOAD */
832 930
@@ -940,6 +1038,8 @@ static inline int check_modstruct_version(Elf_Shdr *sechdrs,
940{ 1038{
941 const unsigned long *crc; 1039 const unsigned long *crc;
942 1040
1041 /* Since this should be found in kernel (which can't be removed),
1042 * no locking is necessary. */
943 if (!find_symbol(MODULE_SYMBOL_PREFIX "module_layout", NULL, 1043 if (!find_symbol(MODULE_SYMBOL_PREFIX "module_layout", NULL,
944 &crc, true, false)) 1044 &crc, true, false))
945 BUG(); 1045 BUG();
@@ -982,35 +1082,68 @@ static inline int same_magic(const char *amagic, const char *bmagic,
982} 1082}
983#endif /* CONFIG_MODVERSIONS */ 1083#endif /* CONFIG_MODVERSIONS */
984 1084
985/* Resolve a symbol for this module. I.e. if we find one, record usage. 1085/* Resolve a symbol for this module. I.e. if we find one, record usage. */
986 Must be holding module_mutex. */ 1086static const struct kernel_symbol *resolve_symbol(struct module *mod,
987static const struct kernel_symbol *resolve_symbol(Elf_Shdr *sechdrs, 1087 const struct load_info *info,
988 unsigned int versindex,
989 const char *name, 1088 const char *name,
990 struct module *mod) 1089 char ownername[])
991{ 1090{
992 struct module *owner; 1091 struct module *owner;
993 const struct kernel_symbol *sym; 1092 const struct kernel_symbol *sym;
994 const unsigned long *crc; 1093 const unsigned long *crc;
1094 int err;
995 1095
1096 mutex_lock(&module_mutex);
996 sym = find_symbol(name, &owner, &crc, 1097 sym = find_symbol(name, &owner, &crc,
997 !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), true); 1098 !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), true);
998 /* use_module can fail due to OOM, 1099 if (!sym)
999 or module initialization or unloading */ 1100 goto unlock;
1000 if (sym) { 1101
1001 if (!check_version(sechdrs, versindex, name, mod, crc, owner) 1102 if (!check_version(info->sechdrs, info->index.vers, name, mod, crc,
1002 || !use_module(mod, owner)) 1103 owner)) {
1003 sym = NULL; 1104 sym = ERR_PTR(-EINVAL);
1105 goto getname;
1106 }
1107
1108 err = ref_module(mod, owner);
1109 if (err) {
1110 sym = ERR_PTR(err);
1111 goto getname;
1004 } 1112 }
1113
1114getname:
1115 /* We must make copy under the lock if we failed to get ref. */
1116 strncpy(ownername, module_name(owner), MODULE_NAME_LEN);
1117unlock:
1118 mutex_unlock(&module_mutex);
1005 return sym; 1119 return sym;
1006} 1120}
1007 1121
1122static const struct kernel_symbol *
1123resolve_symbol_wait(struct module *mod,
1124 const struct load_info *info,
1125 const char *name)
1126{
1127 const struct kernel_symbol *ksym;
1128 char owner[MODULE_NAME_LEN];
1129
1130 if (wait_event_interruptible_timeout(module_wq,
1131 !IS_ERR(ksym = resolve_symbol(mod, info, name, owner))
1132 || PTR_ERR(ksym) != -EBUSY,
1133 30 * HZ) <= 0) {
1134 printk(KERN_WARNING "%s: gave up waiting for init of module %s.\n",
1135 mod->name, owner);
1136 }
1137 return ksym;
1138}
1139
1008/* 1140/*
1009 * /sys/module/foo/sections stuff 1141 * /sys/module/foo/sections stuff
1010 * J. Corbet <corbet@lwn.net> 1142 * J. Corbet <corbet@lwn.net>
1011 */ 1143 */
1012#if defined(CONFIG_KALLSYMS) && defined(CONFIG_SYSFS) 1144#ifdef CONFIG_SYSFS
1013 1145
1146#ifdef CONFIG_KALLSYMS
1014static inline bool sect_empty(const Elf_Shdr *sect) 1147static inline bool sect_empty(const Elf_Shdr *sect)
1015{ 1148{
1016 return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0; 1149 return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
@@ -1047,8 +1180,7 @@ static void free_sect_attrs(struct module_sect_attrs *sect_attrs)
1047 kfree(sect_attrs); 1180 kfree(sect_attrs);
1048} 1181}
1049 1182
1050static void add_sect_attrs(struct module *mod, unsigned int nsect, 1183static void add_sect_attrs(struct module *mod, const struct load_info *info)
1051 char *secstrings, Elf_Shdr *sechdrs)
1052{ 1184{
1053 unsigned int nloaded = 0, i, size[2]; 1185 unsigned int nloaded = 0, i, size[2];
1054 struct module_sect_attrs *sect_attrs; 1186 struct module_sect_attrs *sect_attrs;
@@ -1056,8 +1188,8 @@ static void add_sect_attrs(struct module *mod, unsigned int nsect,
1056 struct attribute **gattr; 1188 struct attribute **gattr;
1057 1189
1058 /* Count loaded sections and allocate structures */ 1190 /* Count loaded sections and allocate structures */
1059 for (i = 0; i < nsect; i++) 1191 for (i = 0; i < info->hdr->e_shnum; i++)
1060 if (!sect_empty(&sechdrs[i])) 1192 if (!sect_empty(&info->sechdrs[i]))
1061 nloaded++; 1193 nloaded++;
1062 size[0] = ALIGN(sizeof(*sect_attrs) 1194 size[0] = ALIGN(sizeof(*sect_attrs)
1063 + nloaded * sizeof(sect_attrs->attrs[0]), 1195 + nloaded * sizeof(sect_attrs->attrs[0]),
@@ -1074,15 +1206,17 @@ static void add_sect_attrs(struct module *mod, unsigned int nsect,
1074 sect_attrs->nsections = 0; 1206 sect_attrs->nsections = 0;
1075 sattr = &sect_attrs->attrs[0]; 1207 sattr = &sect_attrs->attrs[0];
1076 gattr = &sect_attrs->grp.attrs[0]; 1208 gattr = &sect_attrs->grp.attrs[0];
1077 for (i = 0; i < nsect; i++) { 1209 for (i = 0; i < info->hdr->e_shnum; i++) {
1078 if (sect_empty(&sechdrs[i])) 1210 Elf_Shdr *sec = &info->sechdrs[i];
1211 if (sect_empty(sec))
1079 continue; 1212 continue;
1080 sattr->address = sechdrs[i].sh_addr; 1213 sattr->address = sec->sh_addr;
1081 sattr->name = kstrdup(secstrings + sechdrs[i].sh_name, 1214 sattr->name = kstrdup(info->secstrings + sec->sh_name,
1082 GFP_KERNEL); 1215 GFP_KERNEL);
1083 if (sattr->name == NULL) 1216 if (sattr->name == NULL)
1084 goto out; 1217 goto out;
1085 sect_attrs->nsections++; 1218 sect_attrs->nsections++;
1219 sysfs_attr_init(&sattr->mattr.attr);
1086 sattr->mattr.show = module_sect_show; 1220 sattr->mattr.show = module_sect_show;
1087 sattr->mattr.store = NULL; 1221 sattr->mattr.store = NULL;
1088 sattr->mattr.attr.name = sattr->name; 1222 sattr->mattr.attr.name = sattr->name;
@@ -1122,7 +1256,7 @@ struct module_notes_attrs {
1122 struct bin_attribute attrs[0]; 1256 struct bin_attribute attrs[0];
1123}; 1257};
1124 1258
1125static ssize_t module_notes_read(struct kobject *kobj, 1259static ssize_t module_notes_read(struct file *filp, struct kobject *kobj,
1126 struct bin_attribute *bin_attr, 1260 struct bin_attribute *bin_attr,
1127 char *buf, loff_t pos, size_t count) 1261 char *buf, loff_t pos, size_t count)
1128{ 1262{
@@ -1145,8 +1279,7 @@ static void free_notes_attrs(struct module_notes_attrs *notes_attrs,
1145 kfree(notes_attrs); 1279 kfree(notes_attrs);
1146} 1280}
1147 1281
1148static void add_notes_attrs(struct module *mod, unsigned int nsect, 1282static void add_notes_attrs(struct module *mod, const struct load_info *info)
1149 char *secstrings, Elf_Shdr *sechdrs)
1150{ 1283{
1151 unsigned int notes, loaded, i; 1284 unsigned int notes, loaded, i;
1152 struct module_notes_attrs *notes_attrs; 1285 struct module_notes_attrs *notes_attrs;
@@ -1158,9 +1291,9 @@ static void add_notes_attrs(struct module *mod, unsigned int nsect,
1158 1291
1159 /* Count notes sections and allocate structures. */ 1292 /* Count notes sections and allocate structures. */
1160 notes = 0; 1293 notes = 0;
1161 for (i = 0; i < nsect; i++) 1294 for (i = 0; i < info->hdr->e_shnum; i++)
1162 if (!sect_empty(&sechdrs[i]) && 1295 if (!sect_empty(&info->sechdrs[i]) &&
1163 (sechdrs[i].sh_type == SHT_NOTE)) 1296 (info->sechdrs[i].sh_type == SHT_NOTE))
1164 ++notes; 1297 ++notes;
1165 1298
1166 if (notes == 0) 1299 if (notes == 0)
@@ -1174,14 +1307,15 @@ static void add_notes_attrs(struct module *mod, unsigned int nsect,
1174 1307
1175 notes_attrs->notes = notes; 1308 notes_attrs->notes = notes;
1176 nattr = &notes_attrs->attrs[0]; 1309 nattr = &notes_attrs->attrs[0];
1177 for (loaded = i = 0; i < nsect; ++i) { 1310 for (loaded = i = 0; i < info->hdr->e_shnum; ++i) {
1178 if (sect_empty(&sechdrs[i])) 1311 if (sect_empty(&info->sechdrs[i]))
1179 continue; 1312 continue;
1180 if (sechdrs[i].sh_type == SHT_NOTE) { 1313 if (info->sechdrs[i].sh_type == SHT_NOTE) {
1314 sysfs_bin_attr_init(nattr);
1181 nattr->attr.name = mod->sect_attrs->attrs[loaded].name; 1315 nattr->attr.name = mod->sect_attrs->attrs[loaded].name;
1182 nattr->attr.mode = S_IRUGO; 1316 nattr->attr.mode = S_IRUGO;
1183 nattr->size = sechdrs[i].sh_size; 1317 nattr->size = info->sechdrs[i].sh_size;
1184 nattr->private = (void *) sechdrs[i].sh_addr; 1318 nattr->private = (void *) info->sechdrs[i].sh_addr;
1185 nattr->read = module_notes_read; 1319 nattr->read = module_notes_read;
1186 ++nattr; 1320 ++nattr;
1187 } 1321 }
@@ -1212,8 +1346,8 @@ static void remove_notes_attrs(struct module *mod)
1212 1346
1213#else 1347#else
1214 1348
1215static inline void add_sect_attrs(struct module *mod, unsigned int nsect, 1349static inline void add_sect_attrs(struct module *mod,
1216 char *sectstrings, Elf_Shdr *sechdrs) 1350 const struct load_info *info)
1217{ 1351{
1218} 1352}
1219 1353
@@ -1221,18 +1355,44 @@ static inline void remove_sect_attrs(struct module *mod)
1221{ 1355{
1222} 1356}
1223 1357
1224static inline void add_notes_attrs(struct module *mod, unsigned int nsect, 1358static inline void add_notes_attrs(struct module *mod,
1225 char *sectstrings, Elf_Shdr *sechdrs) 1359 const struct load_info *info)
1226{ 1360{
1227} 1361}
1228 1362
1229static inline void remove_notes_attrs(struct module *mod) 1363static inline void remove_notes_attrs(struct module *mod)
1230{ 1364{
1231} 1365}
1366#endif /* CONFIG_KALLSYMS */
1367
1368static void add_usage_links(struct module *mod)
1369{
1370#ifdef CONFIG_MODULE_UNLOAD
1371 struct module_use *use;
1372 int nowarn;
1373
1374 mutex_lock(&module_mutex);
1375 list_for_each_entry(use, &mod->target_list, target_list) {
1376 nowarn = sysfs_create_link(use->target->holders_dir,
1377 &mod->mkobj.kobj, mod->name);
1378 }
1379 mutex_unlock(&module_mutex);
1380#endif
1381}
1382
1383static void del_usage_links(struct module *mod)
1384{
1385#ifdef CONFIG_MODULE_UNLOAD
1386 struct module_use *use;
1387
1388 mutex_lock(&module_mutex);
1389 list_for_each_entry(use, &mod->target_list, target_list)
1390 sysfs_remove_link(use->target->holders_dir, mod->name);
1391 mutex_unlock(&module_mutex);
1232#endif 1392#endif
1393}
1233 1394
1234#ifdef CONFIG_SYSFS 1395static int module_add_modinfo_attrs(struct module *mod)
1235int module_add_modinfo_attrs(struct module *mod)
1236{ 1396{
1237 struct module_attribute *attr; 1397 struct module_attribute *attr;
1238 struct module_attribute *temp_attr; 1398 struct module_attribute *temp_attr;
@@ -1250,6 +1410,7 @@ int module_add_modinfo_attrs(struct module *mod)
1250 if (!attr->test || 1410 if (!attr->test ||
1251 (attr->test && attr->test(mod))) { 1411 (attr->test && attr->test(mod))) {
1252 memcpy(temp_attr, attr, sizeof(*temp_attr)); 1412 memcpy(temp_attr, attr, sizeof(*temp_attr));
1413 sysfs_attr_init(&temp_attr->attr);
1253 error = sysfs_create_file(&mod->mkobj.kobj,&temp_attr->attr); 1414 error = sysfs_create_file(&mod->mkobj.kobj,&temp_attr->attr);
1254 ++temp_attr; 1415 ++temp_attr;
1255 } 1416 }
@@ -1257,7 +1418,7 @@ int module_add_modinfo_attrs(struct module *mod)
1257 return error; 1418 return error;
1258} 1419}
1259 1420
1260void module_remove_modinfo_attrs(struct module *mod) 1421static void module_remove_modinfo_attrs(struct module *mod)
1261{ 1422{
1262 struct module_attribute *attr; 1423 struct module_attribute *attr;
1263 int i; 1424 int i;
@@ -1273,7 +1434,7 @@ void module_remove_modinfo_attrs(struct module *mod)
1273 kfree(mod->modinfo_attrs); 1434 kfree(mod->modinfo_attrs);
1274} 1435}
1275 1436
1276int mod_sysfs_init(struct module *mod) 1437static int mod_sysfs_init(struct module *mod)
1277{ 1438{
1278 int err; 1439 int err;
1279 struct kobject *kobj; 1440 struct kobject *kobj;
@@ -1307,12 +1468,17 @@ out:
1307 return err; 1468 return err;
1308} 1469}
1309 1470
1310int mod_sysfs_setup(struct module *mod, 1471static int mod_sysfs_setup(struct module *mod,
1472 const struct load_info *info,
1311 struct kernel_param *kparam, 1473 struct kernel_param *kparam,
1312 unsigned int num_params) 1474 unsigned int num_params)
1313{ 1475{
1314 int err; 1476 int err;
1315 1477
1478 err = mod_sysfs_init(mod);
1479 if (err)
1480 goto out;
1481
1316 mod->holders_dir = kobject_create_and_add("holders", &mod->mkobj.kobj); 1482 mod->holders_dir = kobject_create_and_add("holders", &mod->mkobj.kobj);
1317 if (!mod->holders_dir) { 1483 if (!mod->holders_dir) {
1318 err = -ENOMEM; 1484 err = -ENOMEM;
@@ -1327,6 +1493,10 @@ int mod_sysfs_setup(struct module *mod,
1327 if (err) 1493 if (err)
1328 goto out_unreg_param; 1494 goto out_unreg_param;
1329 1495
1496 add_usage_links(mod);
1497 add_sect_attrs(mod, info);
1498 add_notes_attrs(mod, info);
1499
1330 kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD); 1500 kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
1331 return 0; 1501 return 0;
1332 1502
@@ -1336,24 +1506,44 @@ out_unreg_holders:
1336 kobject_put(mod->holders_dir); 1506 kobject_put(mod->holders_dir);
1337out_unreg: 1507out_unreg:
1338 kobject_put(&mod->mkobj.kobj); 1508 kobject_put(&mod->mkobj.kobj);
1509out:
1339 return err; 1510 return err;
1340} 1511}
1341 1512
1342static void mod_sysfs_fini(struct module *mod) 1513static void mod_sysfs_fini(struct module *mod)
1343{ 1514{
1515 remove_notes_attrs(mod);
1516 remove_sect_attrs(mod);
1344 kobject_put(&mod->mkobj.kobj); 1517 kobject_put(&mod->mkobj.kobj);
1345} 1518}
1346 1519
1347#else /* CONFIG_SYSFS */ 1520#else /* !CONFIG_SYSFS */
1521
1522static int mod_sysfs_setup(struct module *mod,
1523 const struct load_info *info,
1524 struct kernel_param *kparam,
1525 unsigned int num_params)
1526{
1527 return 0;
1528}
1348 1529
1349static void mod_sysfs_fini(struct module *mod) 1530static void mod_sysfs_fini(struct module *mod)
1350{ 1531{
1351} 1532}
1352 1533
1534static void module_remove_modinfo_attrs(struct module *mod)
1535{
1536}
1537
1538static void del_usage_links(struct module *mod)
1539{
1540}
1541
1353#endif /* CONFIG_SYSFS */ 1542#endif /* CONFIG_SYSFS */
1354 1543
1355static void mod_kobject_remove(struct module *mod) 1544static void mod_sysfs_teardown(struct module *mod)
1356{ 1545{
1546 del_usage_links(mod);
1357 module_remove_modinfo_attrs(mod); 1547 module_remove_modinfo_attrs(mod);
1358 module_param_sysfs_remove(mod); 1548 module_param_sysfs_remove(mod);
1359 kobject_put(mod->mkobj.drivers_dir); 1549 kobject_put(mod->mkobj.drivers_dir);
@@ -1369,19 +1559,132 @@ static int __unlink_module(void *_mod)
1369{ 1559{
1370 struct module *mod = _mod; 1560 struct module *mod = _mod;
1371 list_del(&mod->list); 1561 list_del(&mod->list);
1562 module_bug_cleanup(mod);
1372 return 0; 1563 return 0;
1373} 1564}
1374 1565
1375/* Free a module, remove from lists, etc (must hold module_mutex). */ 1566#ifdef CONFIG_DEBUG_SET_MODULE_RONX
1567/*
1568 * LKM RO/NX protection: protect module's text/ro-data
1569 * from modification and any data from execution.
1570 */
1571void set_page_attributes(void *start, void *end, int (*set)(unsigned long start, int num_pages))
1572{
1573 unsigned long begin_pfn = PFN_DOWN((unsigned long)start);
1574 unsigned long end_pfn = PFN_DOWN((unsigned long)end);
1575
1576 if (end_pfn > begin_pfn)
1577 set(begin_pfn << PAGE_SHIFT, end_pfn - begin_pfn);
1578}
1579
1580static void set_section_ro_nx(void *base,
1581 unsigned long text_size,
1582 unsigned long ro_size,
1583 unsigned long total_size)
1584{
1585 /* begin and end PFNs of the current subsection */
1586 unsigned long begin_pfn;
1587 unsigned long end_pfn;
1588
1589 /*
1590 * Set RO for module text and RO-data:
1591 * - Always protect first page.
1592 * - Do not protect last partial page.
1593 */
1594 if (ro_size > 0)
1595 set_page_attributes(base, base + ro_size, set_memory_ro);
1596
1597 /*
1598 * Set NX permissions for module data:
1599 * - Do not protect first partial page.
1600 * - Always protect last page.
1601 */
1602 if (total_size > text_size) {
1603 begin_pfn = PFN_UP((unsigned long)base + text_size);
1604 end_pfn = PFN_UP((unsigned long)base + total_size);
1605 if (end_pfn > begin_pfn)
1606 set_memory_nx(begin_pfn << PAGE_SHIFT, end_pfn - begin_pfn);
1607 }
1608}
1609
1610/* Setting memory back to RW+NX before releasing it */
1611void unset_section_ro_nx(struct module *mod, void *module_region)
1612{
1613 unsigned long total_pages;
1614
1615 if (mod->module_core == module_region) {
1616 /* Set core as NX+RW */
1617 total_pages = MOD_NUMBER_OF_PAGES(mod->module_core, mod->core_size);
1618 set_memory_nx((unsigned long)mod->module_core, total_pages);
1619 set_memory_rw((unsigned long)mod->module_core, total_pages);
1620
1621 } else if (mod->module_init == module_region) {
1622 /* Set init as NX+RW */
1623 total_pages = MOD_NUMBER_OF_PAGES(mod->module_init, mod->init_size);
1624 set_memory_nx((unsigned long)mod->module_init, total_pages);
1625 set_memory_rw((unsigned long)mod->module_init, total_pages);
1626 }
1627}
1628
1629/* Iterate through all modules and set each module's text as RW */
1630void set_all_modules_text_rw()
1631{
1632 struct module *mod;
1633
1634 mutex_lock(&module_mutex);
1635 list_for_each_entry_rcu(mod, &modules, list) {
1636 if ((mod->module_core) && (mod->core_text_size)) {
1637 set_page_attributes(mod->module_core,
1638 mod->module_core + mod->core_text_size,
1639 set_memory_rw);
1640 }
1641 if ((mod->module_init) && (mod->init_text_size)) {
1642 set_page_attributes(mod->module_init,
1643 mod->module_init + mod->init_text_size,
1644 set_memory_rw);
1645 }
1646 }
1647 mutex_unlock(&module_mutex);
1648}
1649
1650/* Iterate through all modules and set each module's text as RO */
1651void set_all_modules_text_ro()
1652{
1653 struct module *mod;
1654
1655 mutex_lock(&module_mutex);
1656 list_for_each_entry_rcu(mod, &modules, list) {
1657 if ((mod->module_core) && (mod->core_text_size)) {
1658 set_page_attributes(mod->module_core,
1659 mod->module_core + mod->core_text_size,
1660 set_memory_ro);
1661 }
1662 if ((mod->module_init) && (mod->init_text_size)) {
1663 set_page_attributes(mod->module_init,
1664 mod->module_init + mod->init_text_size,
1665 set_memory_ro);
1666 }
1667 }
1668 mutex_unlock(&module_mutex);
1669}
1670#else
1671static inline void set_section_ro_nx(void *base, unsigned long text_size, unsigned long ro_size, unsigned long total_size) { }
1672static inline void unset_section_ro_nx(struct module *mod, void *module_region) { }
1673#endif
1674
1675/* Free a module, remove from lists, etc. */
1376static void free_module(struct module *mod) 1676static void free_module(struct module *mod)
1377{ 1677{
1378 trace_module_free(mod); 1678 trace_module_free(mod);
1379 1679
1380 /* Delete from various lists */ 1680 /* Delete from various lists */
1681 mutex_lock(&module_mutex);
1381 stop_machine(__unlink_module, mod, NULL); 1682 stop_machine(__unlink_module, mod, NULL);
1382 remove_notes_attrs(mod); 1683 mutex_unlock(&module_mutex);
1383 remove_sect_attrs(mod); 1684 mod_sysfs_teardown(mod);
1384 mod_kobject_remove(mod); 1685
1686 /* Remove dynamic debug info */
1687 ddebug_remove_module(mod->name);
1385 1688
1386 /* Arch-specific cleanup. */ 1689 /* Arch-specific cleanup. */
1387 module_arch_cleanup(mod); 1690 module_arch_cleanup(mod);
@@ -1393,18 +1696,16 @@ static void free_module(struct module *mod)
1393 destroy_params(mod->kp, mod->num_kp); 1696 destroy_params(mod->kp, mod->num_kp);
1394 1697
1395 /* This may be NULL, but that's OK */ 1698 /* This may be NULL, but that's OK */
1699 unset_section_ro_nx(mod, mod->module_init);
1396 module_free(mod, mod->module_init); 1700 module_free(mod, mod->module_init);
1397 kfree(mod->args); 1701 kfree(mod->args);
1398 if (mod->percpu) 1702 percpu_modfree(mod);
1399 percpu_modfree(mod->percpu); 1703
1400#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP)
1401 if (mod->refptr)
1402 percpu_modfree(mod->refptr);
1403#endif
1404 /* Free lock-classes: */ 1704 /* Free lock-classes: */
1405 lockdep_free_key_range(mod->module_core, mod->core_size); 1705 lockdep_free_key_range(mod->module_core, mod->core_size);
1406 1706
1407 /* Finally, free the core (containing the module structure) */ 1707 /* Finally, free the core (containing the module structure) */
1708 unset_section_ro_nx(mod, mod->module_core);
1408 module_free(mod, mod->module_core); 1709 module_free(mod, mod->module_core);
1409 1710
1410#ifdef CONFIG_MPU 1711#ifdef CONFIG_MPU
@@ -1430,6 +1731,8 @@ EXPORT_SYMBOL_GPL(__symbol_get);
1430/* 1731/*
1431 * Ensure that an exported symbol [global namespace] does not already exist 1732 * Ensure that an exported symbol [global namespace] does not already exist
1432 * in the kernel or in some other module's exported symbol table. 1733 * in the kernel or in some other module's exported symbol table.
1734 *
1735 * You must hold the module_mutex.
1433 */ 1736 */
1434static int verify_export_symbols(struct module *mod) 1737static int verify_export_symbols(struct module *mod)
1435{ 1738{
@@ -1464,25 +1767,23 @@ static int verify_export_symbols(struct module *mod)
1464} 1767}
1465 1768
1466/* Change all symbols so that st_value encodes the pointer directly. */ 1769/* Change all symbols so that st_value encodes the pointer directly. */
1467static int simplify_symbols(Elf_Shdr *sechdrs, 1770static int simplify_symbols(struct module *mod, const struct load_info *info)
1468 unsigned int symindex, 1771{
1469 const char *strtab, 1772 Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
1470 unsigned int versindex, 1773 Elf_Sym *sym = (void *)symsec->sh_addr;
1471 unsigned int pcpuindex,
1472 struct module *mod)
1473{
1474 Elf_Sym *sym = (void *)sechdrs[symindex].sh_addr;
1475 unsigned long secbase; 1774 unsigned long secbase;
1476 unsigned int i, n = sechdrs[symindex].sh_size / sizeof(Elf_Sym); 1775 unsigned int i;
1477 int ret = 0; 1776 int ret = 0;
1478 const struct kernel_symbol *ksym; 1777 const struct kernel_symbol *ksym;
1479 1778
1480 for (i = 1; i < n; i++) { 1779 for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) {
1780 const char *name = info->strtab + sym[i].st_name;
1781
1481 switch (sym[i].st_shndx) { 1782 switch (sym[i].st_shndx) {
1482 case SHN_COMMON: 1783 case SHN_COMMON:
1483 /* We compiled with -fno-common. These are not 1784 /* We compiled with -fno-common. These are not
1484 supposed to happen. */ 1785 supposed to happen. */
1485 DEBUGP("Common symbol: %s\n", strtab + sym[i].st_name); 1786 DEBUGP("Common symbol: %s\n", name);
1486 printk("%s: please compile with -fno-common\n", 1787 printk("%s: please compile with -fno-common\n",
1487 mod->name); 1788 mod->name);
1488 ret = -ENOEXEC; 1789 ret = -ENOEXEC;
@@ -1495,29 +1796,28 @@ static int simplify_symbols(Elf_Shdr *sechdrs,
1495 break; 1796 break;
1496 1797
1497 case SHN_UNDEF: 1798 case SHN_UNDEF:
1498 ksym = resolve_symbol(sechdrs, versindex, 1799 ksym = resolve_symbol_wait(mod, info, name);
1499 strtab + sym[i].st_name, mod);
1500 /* Ok if resolved. */ 1800 /* Ok if resolved. */
1501 if (ksym) { 1801 if (ksym && !IS_ERR(ksym)) {
1502 sym[i].st_value = ksym->value; 1802 sym[i].st_value = ksym->value;
1503 break; 1803 break;
1504 } 1804 }
1505 1805
1506 /* Ok if weak. */ 1806 /* Ok if weak. */
1507 if (ELF_ST_BIND(sym[i].st_info) == STB_WEAK) 1807 if (!ksym && ELF_ST_BIND(sym[i].st_info) == STB_WEAK)
1508 break; 1808 break;
1509 1809
1510 printk(KERN_WARNING "%s: Unknown symbol %s\n", 1810 printk(KERN_WARNING "%s: Unknown symbol %s (err %li)\n",
1511 mod->name, strtab + sym[i].st_name); 1811 mod->name, name, PTR_ERR(ksym));
1512 ret = -ENOENT; 1812 ret = PTR_ERR(ksym) ?: -ENOENT;
1513 break; 1813 break;
1514 1814
1515 default: 1815 default:
1516 /* Divert to percpu allocation if a percpu var. */ 1816 /* Divert to percpu allocation if a percpu var. */
1517 if (sym[i].st_shndx == pcpuindex) 1817 if (sym[i].st_shndx == info->index.pcpu)
1518 secbase = (unsigned long)mod->percpu; 1818 secbase = (unsigned long)mod_percpu(mod);
1519 else 1819 else
1520 secbase = sechdrs[sym[i].st_shndx].sh_addr; 1820 secbase = info->sechdrs[sym[i].st_shndx].sh_addr;
1521 sym[i].st_value += secbase; 1821 sym[i].st_value += secbase;
1522 break; 1822 break;
1523 } 1823 }
@@ -1526,6 +1826,35 @@ static int simplify_symbols(Elf_Shdr *sechdrs,
1526 return ret; 1826 return ret;
1527} 1827}
1528 1828
1829static int apply_relocations(struct module *mod, const struct load_info *info)
1830{
1831 unsigned int i;
1832 int err = 0;
1833
1834 /* Now do relocations. */
1835 for (i = 1; i < info->hdr->e_shnum; i++) {
1836 unsigned int infosec = info->sechdrs[i].sh_info;
1837
1838 /* Not a valid relocation section? */
1839 if (infosec >= info->hdr->e_shnum)
1840 continue;
1841
1842 /* Don't bother with non-allocated sections */
1843 if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC))
1844 continue;
1845
1846 if (info->sechdrs[i].sh_type == SHT_REL)
1847 err = apply_relocate(info->sechdrs, info->strtab,
1848 info->index.sym, i, mod);
1849 else if (info->sechdrs[i].sh_type == SHT_RELA)
1850 err = apply_relocate_add(info->sechdrs, info->strtab,
1851 info->index.sym, i, mod);
1852 if (err < 0)
1853 break;
1854 }
1855 return err;
1856}
1857
1529/* Additional bytes needed by arch in front of individual sections */ 1858/* Additional bytes needed by arch in front of individual sections */
1530unsigned int __weak arch_mod_section_prepend(struct module *mod, 1859unsigned int __weak arch_mod_section_prepend(struct module *mod,
1531 unsigned int section) 1860 unsigned int section)
@@ -1550,10 +1879,7 @@ static long get_offset(struct module *mod, unsigned int *size,
1550 might -- code, read-only data, read-write data, small data. Tally 1879 might -- code, read-only data, read-write data, small data. Tally
1551 sizes, and place the offsets into sh_entsize fields: high bit means it 1880 sizes, and place the offsets into sh_entsize fields: high bit means it
1552 belongs in init. */ 1881 belongs in init. */
1553static void layout_sections(struct module *mod, 1882static void layout_sections(struct module *mod, struct load_info *info)
1554 const Elf_Ehdr *hdr,
1555 Elf_Shdr *sechdrs,
1556 const char *secstrings)
1557{ 1883{
1558 static unsigned long const masks[][2] = { 1884 static unsigned long const masks[][2] = {
1559 /* NOTE: all executable code must be the first section 1885 /* NOTE: all executable code must be the first section
@@ -1566,42 +1892,66 @@ static void layout_sections(struct module *mod,
1566 }; 1892 };
1567 unsigned int m, i; 1893 unsigned int m, i;
1568 1894
1569 for (i = 0; i < hdr->e_shnum; i++) 1895 for (i = 0; i < info->hdr->e_shnum; i++)
1570 sechdrs[i].sh_entsize = ~0UL; 1896 info->sechdrs[i].sh_entsize = ~0UL;
1571 1897
1572 DEBUGP("Core section allocation order:\n"); 1898 DEBUGP("Core section allocation order:\n");
1573 for (m = 0; m < ARRAY_SIZE(masks); ++m) { 1899 for (m = 0; m < ARRAY_SIZE(masks); ++m) {
1574 for (i = 0; i < hdr->e_shnum; ++i) { 1900 for (i = 0; i < info->hdr->e_shnum; ++i) {
1575 Elf_Shdr *s = &sechdrs[i]; 1901 Elf_Shdr *s = &info->sechdrs[i];
1902 const char *sname = info->secstrings + s->sh_name;
1576 1903
1577 if ((s->sh_flags & masks[m][0]) != masks[m][0] 1904 if ((s->sh_flags & masks[m][0]) != masks[m][0]
1578 || (s->sh_flags & masks[m][1]) 1905 || (s->sh_flags & masks[m][1])
1579 || s->sh_entsize != ~0UL 1906 || s->sh_entsize != ~0UL
1580 || strstarts(secstrings + s->sh_name, ".init")) 1907 || strstarts(sname, ".init"))
1581 continue; 1908 continue;
1582 s->sh_entsize = get_offset(mod, &mod->core_size, s, i); 1909 s->sh_entsize = get_offset(mod, &mod->core_size, s, i);
1583 DEBUGP("\t%s\n", secstrings + s->sh_name); 1910 DEBUGP("\t%s\n", name);
1584 } 1911 }
1585 if (m == 0) 1912 switch (m) {
1913 case 0: /* executable */
1914 mod->core_size = debug_align(mod->core_size);
1586 mod->core_text_size = mod->core_size; 1915 mod->core_text_size = mod->core_size;
1916 break;
1917 case 1: /* RO: text and ro-data */
1918 mod->core_size = debug_align(mod->core_size);
1919 mod->core_ro_size = mod->core_size;
1920 break;
1921 case 3: /* whole core */
1922 mod->core_size = debug_align(mod->core_size);
1923 break;
1924 }
1587 } 1925 }
1588 1926
1589 DEBUGP("Init section allocation order:\n"); 1927 DEBUGP("Init section allocation order:\n");
1590 for (m = 0; m < ARRAY_SIZE(masks); ++m) { 1928 for (m = 0; m < ARRAY_SIZE(masks); ++m) {
1591 for (i = 0; i < hdr->e_shnum; ++i) { 1929 for (i = 0; i < info->hdr->e_shnum; ++i) {
1592 Elf_Shdr *s = &sechdrs[i]; 1930 Elf_Shdr *s = &info->sechdrs[i];
1931 const char *sname = info->secstrings + s->sh_name;
1593 1932
1594 if ((s->sh_flags & masks[m][0]) != masks[m][0] 1933 if ((s->sh_flags & masks[m][0]) != masks[m][0]
1595 || (s->sh_flags & masks[m][1]) 1934 || (s->sh_flags & masks[m][1])
1596 || s->sh_entsize != ~0UL 1935 || s->sh_entsize != ~0UL
1597 || !strstarts(secstrings + s->sh_name, ".init")) 1936 || !strstarts(sname, ".init"))
1598 continue; 1937 continue;
1599 s->sh_entsize = (get_offset(mod, &mod->init_size, s, i) 1938 s->sh_entsize = (get_offset(mod, &mod->init_size, s, i)
1600 | INIT_OFFSET_MASK); 1939 | INIT_OFFSET_MASK);
1601 DEBUGP("\t%s\n", secstrings + s->sh_name); 1940 DEBUGP("\t%s\n", sname);
1602 } 1941 }
1603 if (m == 0) 1942 switch (m) {
1943 case 0: /* executable */
1944 mod->init_size = debug_align(mod->init_size);
1604 mod->init_text_size = mod->init_size; 1945 mod->init_text_size = mod->init_size;
1946 break;
1947 case 1: /* RO: text and ro-data */
1948 mod->init_size = debug_align(mod->init_size);
1949 mod->init_ro_size = mod->init_size;
1950 break;
1951 case 3: /* whole init */
1952 mod->init_size = debug_align(mod->init_size);
1953 break;
1954 }
1605 } 1955 }
1606} 1956}
1607 1957
@@ -1637,33 +1987,28 @@ static char *next_string(char *string, unsigned long *secsize)
1637 return string; 1987 return string;
1638} 1988}
1639 1989
1640static char *get_modinfo(Elf_Shdr *sechdrs, 1990static char *get_modinfo(struct load_info *info, const char *tag)
1641 unsigned int info,
1642 const char *tag)
1643{ 1991{
1644 char *p; 1992 char *p;
1645 unsigned int taglen = strlen(tag); 1993 unsigned int taglen = strlen(tag);
1646 unsigned long size = sechdrs[info].sh_size; 1994 Elf_Shdr *infosec = &info->sechdrs[info->index.info];
1995 unsigned long size = infosec->sh_size;
1647 1996
1648 for (p = (char *)sechdrs[info].sh_addr; p; p = next_string(p, &size)) { 1997 for (p = (char *)infosec->sh_addr; p; p = next_string(p, &size)) {
1649 if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=') 1998 if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=')
1650 return p + taglen + 1; 1999 return p + taglen + 1;
1651 } 2000 }
1652 return NULL; 2001 return NULL;
1653} 2002}
1654 2003
1655static void setup_modinfo(struct module *mod, Elf_Shdr *sechdrs, 2004static void setup_modinfo(struct module *mod, struct load_info *info)
1656 unsigned int infoindex)
1657{ 2005{
1658 struct module_attribute *attr; 2006 struct module_attribute *attr;
1659 int i; 2007 int i;
1660 2008
1661 for (i = 0; (attr = modinfo_attrs[i]); i++) { 2009 for (i = 0; (attr = modinfo_attrs[i]); i++) {
1662 if (attr->setup) 2010 if (attr->setup)
1663 attr->setup(mod, 2011 attr->setup(mod, get_modinfo(info, attr->attr.name));
1664 get_modinfo(sechdrs,
1665 infoindex,
1666 attr->attr.name));
1667 } 2012 }
1668} 2013}
1669 2014
@@ -1704,11 +2049,10 @@ static int is_exported(const char *name, unsigned long value,
1704} 2049}
1705 2050
1706/* As per nm */ 2051/* As per nm */
1707static char elf_type(const Elf_Sym *sym, 2052static char elf_type(const Elf_Sym *sym, const struct load_info *info)
1708 Elf_Shdr *sechdrs,
1709 const char *secstrings,
1710 struct module *mod)
1711{ 2053{
2054 const Elf_Shdr *sechdrs = info->sechdrs;
2055
1712 if (ELF_ST_BIND(sym->st_info) == STB_WEAK) { 2056 if (ELF_ST_BIND(sym->st_info) == STB_WEAK) {
1713 if (ELF_ST_TYPE(sym->st_info) == STT_OBJECT) 2057 if (ELF_ST_TYPE(sym->st_info) == STT_OBJECT)
1714 return 'v'; 2058 return 'v';
@@ -1738,8 +2082,10 @@ static char elf_type(const Elf_Sym *sym,
1738 else 2082 else
1739 return 'b'; 2083 return 'b';
1740 } 2084 }
1741 if (strstarts(secstrings + sechdrs[sym->st_shndx].sh_name, ".debug")) 2085 if (strstarts(info->secstrings + sechdrs[sym->st_shndx].sh_name,
2086 ".debug")) {
1742 return 'n'; 2087 return 'n';
2088 }
1743 return '?'; 2089 return '?';
1744} 2090}
1745 2091
@@ -1764,127 +2110,96 @@ static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs,
1764 return true; 2110 return true;
1765} 2111}
1766 2112
1767static unsigned long layout_symtab(struct module *mod, 2113static void layout_symtab(struct module *mod, struct load_info *info)
1768 Elf_Shdr *sechdrs,
1769 unsigned int symindex,
1770 unsigned int strindex,
1771 const Elf_Ehdr *hdr,
1772 const char *secstrings,
1773 unsigned long *pstroffs,
1774 unsigned long *strmap)
1775{ 2114{
1776 unsigned long symoffs; 2115 Elf_Shdr *symsect = info->sechdrs + info->index.sym;
1777 Elf_Shdr *symsect = sechdrs + symindex; 2116 Elf_Shdr *strsect = info->sechdrs + info->index.str;
1778 Elf_Shdr *strsect = sechdrs + strindex;
1779 const Elf_Sym *src; 2117 const Elf_Sym *src;
1780 const char *strtab;
1781 unsigned int i, nsrc, ndst; 2118 unsigned int i, nsrc, ndst;
1782 2119
1783 /* Put symbol section at end of init part of module. */ 2120 /* Put symbol section at end of init part of module. */
1784 symsect->sh_flags |= SHF_ALLOC; 2121 symsect->sh_flags |= SHF_ALLOC;
1785 symsect->sh_entsize = get_offset(mod, &mod->init_size, symsect, 2122 symsect->sh_entsize = get_offset(mod, &mod->init_size, symsect,
1786 symindex) | INIT_OFFSET_MASK; 2123 info->index.sym) | INIT_OFFSET_MASK;
1787 DEBUGP("\t%s\n", secstrings + symsect->sh_name); 2124 DEBUGP("\t%s\n", info->secstrings + symsect->sh_name);
1788 2125
1789 src = (void *)hdr + symsect->sh_offset; 2126 src = (void *)info->hdr + symsect->sh_offset;
1790 nsrc = symsect->sh_size / sizeof(*src); 2127 nsrc = symsect->sh_size / sizeof(*src);
1791 strtab = (void *)hdr + strsect->sh_offset;
1792 for (ndst = i = 1; i < nsrc; ++i, ++src) 2128 for (ndst = i = 1; i < nsrc; ++i, ++src)
1793 if (is_core_symbol(src, sechdrs, hdr->e_shnum)) { 2129 if (is_core_symbol(src, info->sechdrs, info->hdr->e_shnum)) {
1794 unsigned int j = src->st_name; 2130 unsigned int j = src->st_name;
1795 2131
1796 while(!__test_and_set_bit(j, strmap) && strtab[j]) 2132 while (!__test_and_set_bit(j, info->strmap)
2133 && info->strtab[j])
1797 ++j; 2134 ++j;
1798 ++ndst; 2135 ++ndst;
1799 } 2136 }
1800 2137
1801 /* Append room for core symbols at end of core part. */ 2138 /* Append room for core symbols at end of core part. */
1802 symoffs = ALIGN(mod->core_size, symsect->sh_addralign ?: 1); 2139 info->symoffs = ALIGN(mod->core_size, symsect->sh_addralign ?: 1);
1803 mod->core_size = symoffs + ndst * sizeof(Elf_Sym); 2140 mod->core_size = info->symoffs + ndst * sizeof(Elf_Sym);
1804 2141
1805 /* Put string table section at end of init part of module. */ 2142 /* Put string table section at end of init part of module. */
1806 strsect->sh_flags |= SHF_ALLOC; 2143 strsect->sh_flags |= SHF_ALLOC;
1807 strsect->sh_entsize = get_offset(mod, &mod->init_size, strsect, 2144 strsect->sh_entsize = get_offset(mod, &mod->init_size, strsect,
1808 strindex) | INIT_OFFSET_MASK; 2145 info->index.str) | INIT_OFFSET_MASK;
1809 DEBUGP("\t%s\n", secstrings + strsect->sh_name); 2146 DEBUGP("\t%s\n", info->secstrings + strsect->sh_name);
1810 2147
1811 /* Append room for core symbols' strings at end of core part. */ 2148 /* Append room for core symbols' strings at end of core part. */
1812 *pstroffs = mod->core_size; 2149 info->stroffs = mod->core_size;
1813 __set_bit(0, strmap); 2150 __set_bit(0, info->strmap);
1814 mod->core_size += bitmap_weight(strmap, strsect->sh_size); 2151 mod->core_size += bitmap_weight(info->strmap, strsect->sh_size);
1815
1816 return symoffs;
1817} 2152}
1818 2153
1819static void add_kallsyms(struct module *mod, 2154static void add_kallsyms(struct module *mod, const struct load_info *info)
1820 Elf_Shdr *sechdrs,
1821 unsigned int shnum,
1822 unsigned int symindex,
1823 unsigned int strindex,
1824 unsigned long symoffs,
1825 unsigned long stroffs,
1826 const char *secstrings,
1827 unsigned long *strmap)
1828{ 2155{
1829 unsigned int i, ndst; 2156 unsigned int i, ndst;
1830 const Elf_Sym *src; 2157 const Elf_Sym *src;
1831 Elf_Sym *dst; 2158 Elf_Sym *dst;
1832 char *s; 2159 char *s;
2160 Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
1833 2161
1834 mod->symtab = (void *)sechdrs[symindex].sh_addr; 2162 mod->symtab = (void *)symsec->sh_addr;
1835 mod->num_symtab = sechdrs[symindex].sh_size / sizeof(Elf_Sym); 2163 mod->num_symtab = symsec->sh_size / sizeof(Elf_Sym);
1836 mod->strtab = (void *)sechdrs[strindex].sh_addr; 2164 /* Make sure we get permanent strtab: don't use info->strtab. */
2165 mod->strtab = (void *)info->sechdrs[info->index.str].sh_addr;
1837 2166
1838 /* Set types up while we still have access to sections. */ 2167 /* Set types up while we still have access to sections. */
1839 for (i = 0; i < mod->num_symtab; i++) 2168 for (i = 0; i < mod->num_symtab; i++)
1840 mod->symtab[i].st_info 2169 mod->symtab[i].st_info = elf_type(&mod->symtab[i], info);
1841 = elf_type(&mod->symtab[i], sechdrs, secstrings, mod);
1842 2170
1843 mod->core_symtab = dst = mod->module_core + symoffs; 2171 mod->core_symtab = dst = mod->module_core + info->symoffs;
1844 src = mod->symtab; 2172 src = mod->symtab;
1845 *dst = *src; 2173 *dst = *src;
1846 for (ndst = i = 1; i < mod->num_symtab; ++i, ++src) { 2174 for (ndst = i = 1; i < mod->num_symtab; ++i, ++src) {
1847 if (!is_core_symbol(src, sechdrs, shnum)) 2175 if (!is_core_symbol(src, info->sechdrs, info->hdr->e_shnum))
1848 continue; 2176 continue;
1849 dst[ndst] = *src; 2177 dst[ndst] = *src;
1850 dst[ndst].st_name = bitmap_weight(strmap, dst[ndst].st_name); 2178 dst[ndst].st_name = bitmap_weight(info->strmap,
2179 dst[ndst].st_name);
1851 ++ndst; 2180 ++ndst;
1852 } 2181 }
1853 mod->core_num_syms = ndst; 2182 mod->core_num_syms = ndst;
1854 2183
1855 mod->core_strtab = s = mod->module_core + stroffs; 2184 mod->core_strtab = s = mod->module_core + info->stroffs;
1856 for (*s = 0, i = 1; i < sechdrs[strindex].sh_size; ++i) 2185 for (*s = 0, i = 1; i < info->sechdrs[info->index.str].sh_size; ++i)
1857 if (test_bit(i, strmap)) 2186 if (test_bit(i, info->strmap))
1858 *++s = mod->strtab[i]; 2187 *++s = mod->strtab[i];
1859} 2188}
1860#else 2189#else
1861static inline unsigned long layout_symtab(struct module *mod, 2190static inline void layout_symtab(struct module *mod, struct load_info *info)
1862 Elf_Shdr *sechdrs,
1863 unsigned int symindex,
1864 unsigned int strindex,
1865 const Elf_Ehdr *hdr,
1866 const char *secstrings,
1867 unsigned long *pstroffs,
1868 unsigned long *strmap)
1869{ 2191{
1870 return 0;
1871} 2192}
1872 2193
1873static inline void add_kallsyms(struct module *mod, 2194static void add_kallsyms(struct module *mod, const struct load_info *info)
1874 Elf_Shdr *sechdrs,
1875 unsigned int shnum,
1876 unsigned int symindex,
1877 unsigned int strindex,
1878 unsigned long symoffs,
1879 unsigned long stroffs,
1880 const char *secstrings,
1881 const unsigned long *strmap)
1882{ 2195{
1883} 2196}
1884#endif /* CONFIG_KALLSYMS */ 2197#endif /* CONFIG_KALLSYMS */
1885 2198
1886static void dynamic_debug_setup(struct _ddebug *debug, unsigned int num) 2199static void dynamic_debug_setup(struct _ddebug *debug, unsigned int num)
1887{ 2200{
2201 if (!debug)
2202 return;
1888#ifdef CONFIG_DYNAMIC_DEBUG 2203#ifdef CONFIG_DYNAMIC_DEBUG
1889 if (ddebug_add_module(debug, num, debug->modname)) 2204 if (ddebug_add_module(debug, num, debug->modname))
1890 printk(KERN_ERR "dynamic debug error adding module: %s\n", 2205 printk(KERN_ERR "dynamic debug error adding module: %s\n",
@@ -1892,77 +2207,70 @@ static void dynamic_debug_setup(struct _ddebug *debug, unsigned int num)
1892#endif 2207#endif
1893} 2208}
1894 2209
2210static void dynamic_debug_remove(struct _ddebug *debug)
2211{
2212 if (debug)
2213 ddebug_remove_module(debug->modname);
2214}
2215
1895static void *module_alloc_update_bounds(unsigned long size) 2216static void *module_alloc_update_bounds(unsigned long size)
1896{ 2217{
1897 void *ret = module_alloc(size); 2218 void *ret = module_alloc(size);
1898 2219
1899 if (ret) { 2220 if (ret) {
2221 mutex_lock(&module_mutex);
1900 /* Update module bounds. */ 2222 /* Update module bounds. */
1901 if ((unsigned long)ret < module_addr_min) 2223 if ((unsigned long)ret < module_addr_min)
1902 module_addr_min = (unsigned long)ret; 2224 module_addr_min = (unsigned long)ret;
1903 if ((unsigned long)ret + size > module_addr_max) 2225 if ((unsigned long)ret + size > module_addr_max)
1904 module_addr_max = (unsigned long)ret + size; 2226 module_addr_max = (unsigned long)ret + size;
2227 mutex_unlock(&module_mutex);
1905 } 2228 }
1906 return ret; 2229 return ret;
1907} 2230}
1908 2231
1909#ifdef CONFIG_DEBUG_KMEMLEAK 2232#ifdef CONFIG_DEBUG_KMEMLEAK
1910static void kmemleak_load_module(struct module *mod, Elf_Ehdr *hdr, 2233static void kmemleak_load_module(const struct module *mod,
1911 Elf_Shdr *sechdrs, char *secstrings) 2234 const struct load_info *info)
1912{ 2235{
1913 unsigned int i; 2236 unsigned int i;
1914 2237
1915 /* only scan the sections containing data */ 2238 /* only scan the sections containing data */
1916 kmemleak_scan_area(mod, sizeof(struct module), GFP_KERNEL); 2239 kmemleak_scan_area(mod, sizeof(struct module), GFP_KERNEL);
1917 2240
1918 for (i = 1; i < hdr->e_shnum; i++) { 2241 for (i = 1; i < info->hdr->e_shnum; i++) {
1919 if (!(sechdrs[i].sh_flags & SHF_ALLOC)) 2242 const char *name = info->secstrings + info->sechdrs[i].sh_name;
2243 if (!(info->sechdrs[i].sh_flags & SHF_ALLOC))
1920 continue; 2244 continue;
1921 if (strncmp(secstrings + sechdrs[i].sh_name, ".data", 5) != 0 2245 if (!strstarts(name, ".data") && !strstarts(name, ".bss"))
1922 && strncmp(secstrings + sechdrs[i].sh_name, ".bss", 4) != 0)
1923 continue; 2246 continue;
1924 2247
1925 kmemleak_scan_area((void *)sechdrs[i].sh_addr, 2248 kmemleak_scan_area((void *)info->sechdrs[i].sh_addr,
1926 sechdrs[i].sh_size, GFP_KERNEL); 2249 info->sechdrs[i].sh_size, GFP_KERNEL);
1927 } 2250 }
1928} 2251}
1929#else 2252#else
1930static inline void kmemleak_load_module(struct module *mod, Elf_Ehdr *hdr, 2253static inline void kmemleak_load_module(const struct module *mod,
1931 Elf_Shdr *sechdrs, char *secstrings) 2254 const struct load_info *info)
1932{ 2255{
1933} 2256}
1934#endif 2257#endif
1935 2258
1936/* Allocate and load the module: note that size of section 0 is always 2259/* Sets info->hdr and info->len. */
1937 zero, and we rely on this for optional sections. */ 2260static int copy_and_check(struct load_info *info,
1938static noinline struct module *load_module(void __user *umod, 2261 const void __user *umod, unsigned long len,
1939 unsigned long len, 2262 const char __user *uargs)
1940 const char __user *uargs)
1941{ 2263{
2264 int err;
1942 Elf_Ehdr *hdr; 2265 Elf_Ehdr *hdr;
1943 Elf_Shdr *sechdrs;
1944 char *secstrings, *args, *modmagic, *strtab = NULL;
1945 char *staging;
1946 unsigned int i;
1947 unsigned int symindex = 0;
1948 unsigned int strindex = 0;
1949 unsigned int modindex, versindex, infoindex, pcpuindex;
1950 struct module *mod;
1951 long err = 0;
1952 void *percpu = NULL, *ptr = NULL; /* Stops spurious gcc warning */
1953 unsigned long symoffs, stroffs, *strmap;
1954
1955 mm_segment_t old_fs;
1956 2266
1957 DEBUGP("load_module: umod=%p, len=%lu, uargs=%p\n",
1958 umod, len, uargs);
1959 if (len < sizeof(*hdr)) 2267 if (len < sizeof(*hdr))
1960 return ERR_PTR(-ENOEXEC); 2268 return -ENOEXEC;
1961 2269
1962 /* Suck in entire file: we'll want most of it. */ 2270 /* Suck in entire file: we'll want most of it. */
1963 /* vmalloc barfs on "unusual" numbers. Check here */ 2271 /* vmalloc barfs on "unusual" numbers. Check here */
1964 if (len > 64 * 1024 * 1024 || (hdr = vmalloc(len)) == NULL) 2272 if (len > 64 * 1024 * 1024 || (hdr = vmalloc(len)) == NULL)
1965 return ERR_PTR(-ENOMEM); 2273 return -ENOMEM;
1966 2274
1967 if (copy_from_user(hdr, umod, len) != 0) { 2275 if (copy_from_user(hdr, umod, len) != 0) {
1968 err = -EFAULT; 2276 err = -EFAULT;
@@ -1970,142 +2278,242 @@ static noinline struct module *load_module(void __user *umod,
1970 } 2278 }
1971 2279
1972 /* Sanity checks against insmoding binaries or wrong arch, 2280 /* Sanity checks against insmoding binaries or wrong arch,
1973 weird elf version */ 2281 weird elf version */
1974 if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0 2282 if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0
1975 || hdr->e_type != ET_REL 2283 || hdr->e_type != ET_REL
1976 || !elf_check_arch(hdr) 2284 || !elf_check_arch(hdr)
1977 || hdr->e_shentsize != sizeof(*sechdrs)) { 2285 || hdr->e_shentsize != sizeof(Elf_Shdr)) {
1978 err = -ENOEXEC; 2286 err = -ENOEXEC;
1979 goto free_hdr; 2287 goto free_hdr;
1980 } 2288 }
1981 2289
1982 if (len < hdr->e_shoff + hdr->e_shnum * sizeof(Elf_Shdr)) 2290 if (len < hdr->e_shoff + hdr->e_shnum * sizeof(Elf_Shdr)) {
1983 goto truncated; 2291 err = -ENOEXEC;
2292 goto free_hdr;
2293 }
2294
2295 info->hdr = hdr;
2296 info->len = len;
2297 return 0;
2298
2299free_hdr:
2300 vfree(hdr);
2301 return err;
2302}
2303
2304static void free_copy(struct load_info *info)
2305{
2306 vfree(info->hdr);
2307}
2308
2309static int rewrite_section_headers(struct load_info *info)
2310{
2311 unsigned int i;
1984 2312
1985 /* Convenience variables */ 2313 /* This should always be true, but let's be sure. */
1986 sechdrs = (void *)hdr + hdr->e_shoff; 2314 info->sechdrs[0].sh_addr = 0;
1987 secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
1988 sechdrs[0].sh_addr = 0;
1989 2315
1990 for (i = 1; i < hdr->e_shnum; i++) { 2316 for (i = 1; i < info->hdr->e_shnum; i++) {
1991 if (sechdrs[i].sh_type != SHT_NOBITS 2317 Elf_Shdr *shdr = &info->sechdrs[i];
1992 && len < sechdrs[i].sh_offset + sechdrs[i].sh_size) 2318 if (shdr->sh_type != SHT_NOBITS
1993 goto truncated; 2319 && info->len < shdr->sh_offset + shdr->sh_size) {
2320 printk(KERN_ERR "Module len %lu truncated\n",
2321 info->len);
2322 return -ENOEXEC;
2323 }
1994 2324
1995 /* Mark all sections sh_addr with their address in the 2325 /* Mark all sections sh_addr with their address in the
1996 temporary image. */ 2326 temporary image. */
1997 sechdrs[i].sh_addr = (size_t)hdr + sechdrs[i].sh_offset; 2327 shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset;
1998 2328
1999 /* Internal symbols and strings. */
2000 if (sechdrs[i].sh_type == SHT_SYMTAB) {
2001 symindex = i;
2002 strindex = sechdrs[i].sh_link;
2003 strtab = (char *)hdr + sechdrs[strindex].sh_offset;
2004 }
2005#ifndef CONFIG_MODULE_UNLOAD 2329#ifndef CONFIG_MODULE_UNLOAD
2006 /* Don't load .exit sections */ 2330 /* Don't load .exit sections */
2007 if (strstarts(secstrings+sechdrs[i].sh_name, ".exit")) 2331 if (strstarts(info->secstrings+shdr->sh_name, ".exit"))
2008 sechdrs[i].sh_flags &= ~(unsigned long)SHF_ALLOC; 2332 shdr->sh_flags &= ~(unsigned long)SHF_ALLOC;
2009#endif 2333#endif
2010 } 2334 }
2011 2335
2012 modindex = find_sec(hdr, sechdrs, secstrings, 2336 /* Track but don't keep modinfo and version sections. */
2013 ".gnu.linkonce.this_module"); 2337 info->index.vers = find_sec(info, "__versions");
2014 if (!modindex) { 2338 info->index.info = find_sec(info, ".modinfo");
2339 info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC;
2340 info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC;
2341 return 0;
2342}
2343
2344/*
2345 * Set up our basic convenience variables (pointers to section headers,
2346 * search for module section index etc), and do some basic section
2347 * verification.
2348 *
2349 * Return the temporary module pointer (we'll replace it with the final
2350 * one when we move the module sections around).
2351 */
2352static struct module *setup_load_info(struct load_info *info)
2353{
2354 unsigned int i;
2355 int err;
2356 struct module *mod;
2357
2358 /* Set up the convenience variables */
2359 info->sechdrs = (void *)info->hdr + info->hdr->e_shoff;
2360 info->secstrings = (void *)info->hdr
2361 + info->sechdrs[info->hdr->e_shstrndx].sh_offset;
2362
2363 err = rewrite_section_headers(info);
2364 if (err)
2365 return ERR_PTR(err);
2366
2367 /* Find internal symbols and strings. */
2368 for (i = 1; i < info->hdr->e_shnum; i++) {
2369 if (info->sechdrs[i].sh_type == SHT_SYMTAB) {
2370 info->index.sym = i;
2371 info->index.str = info->sechdrs[i].sh_link;
2372 info->strtab = (char *)info->hdr
2373 + info->sechdrs[info->index.str].sh_offset;
2374 break;
2375 }
2376 }
2377
2378 info->index.mod = find_sec(info, ".gnu.linkonce.this_module");
2379 if (!info->index.mod) {
2015 printk(KERN_WARNING "No module found in object\n"); 2380 printk(KERN_WARNING "No module found in object\n");
2016 err = -ENOEXEC; 2381 return ERR_PTR(-ENOEXEC);
2017 goto free_hdr;
2018 } 2382 }
2019 /* This is temporary: point mod into copy of data. */ 2383 /* This is temporary: point mod into copy of data. */
2020 mod = (void *)sechdrs[modindex].sh_addr; 2384 mod = (void *)info->sechdrs[info->index.mod].sh_addr;
2021 2385
2022 if (symindex == 0) { 2386 if (info->index.sym == 0) {
2023 printk(KERN_WARNING "%s: module has no symbols (stripped?)\n", 2387 printk(KERN_WARNING "%s: module has no symbols (stripped?)\n",
2024 mod->name); 2388 mod->name);
2025 err = -ENOEXEC; 2389 return ERR_PTR(-ENOEXEC);
2026 goto free_hdr;
2027 } 2390 }
2028 2391
2029 versindex = find_sec(hdr, sechdrs, secstrings, "__versions"); 2392 info->index.pcpu = find_pcpusec(info);
2030 infoindex = find_sec(hdr, sechdrs, secstrings, ".modinfo");
2031 pcpuindex = find_pcpusec(hdr, sechdrs, secstrings);
2032
2033 /* Don't keep modinfo and version sections. */
2034 sechdrs[infoindex].sh_flags &= ~(unsigned long)SHF_ALLOC;
2035 sechdrs[versindex].sh_flags &= ~(unsigned long)SHF_ALLOC;
2036 2393
2037 /* Check module struct version now, before we try to use module. */ 2394 /* Check module struct version now, before we try to use module. */
2038 if (!check_modstruct_version(sechdrs, versindex, mod)) { 2395 if (!check_modstruct_version(info->sechdrs, info->index.vers, mod))
2039 err = -ENOEXEC; 2396 return ERR_PTR(-ENOEXEC);
2040 goto free_hdr; 2397
2041 } 2398 return mod;
2399}
2400
2401static int check_modinfo(struct module *mod, struct load_info *info)
2402{
2403 const char *modmagic = get_modinfo(info, "vermagic");
2404 int err;
2042 2405
2043 modmagic = get_modinfo(sechdrs, infoindex, "vermagic");
2044 /* This is allowed: modprobe --force will invalidate it. */ 2406 /* This is allowed: modprobe --force will invalidate it. */
2045 if (!modmagic) { 2407 if (!modmagic) {
2046 err = try_to_force_load(mod, "bad vermagic"); 2408 err = try_to_force_load(mod, "bad vermagic");
2047 if (err) 2409 if (err)
2048 goto free_hdr; 2410 return err;
2049 } else if (!same_magic(modmagic, vermagic, versindex)) { 2411 } else if (!same_magic(modmagic, vermagic, info->index.vers)) {
2050 printk(KERN_ERR "%s: version magic '%s' should be '%s'\n", 2412 printk(KERN_ERR "%s: version magic '%s' should be '%s'\n",
2051 mod->name, modmagic, vermagic); 2413 mod->name, modmagic, vermagic);
2052 err = -ENOEXEC; 2414 return -ENOEXEC;
2053 goto free_hdr;
2054 } 2415 }
2055 2416
2056 staging = get_modinfo(sechdrs, infoindex, "staging"); 2417 if (get_modinfo(info, "staging")) {
2057 if (staging) {
2058 add_taint_module(mod, TAINT_CRAP); 2418 add_taint_module(mod, TAINT_CRAP);
2059 printk(KERN_WARNING "%s: module is from the staging directory," 2419 printk(KERN_WARNING "%s: module is from the staging directory,"
2060 " the quality is unknown, you have been warned.\n", 2420 " the quality is unknown, you have been warned.\n",
2061 mod->name); 2421 mod->name);
2062 } 2422 }
2063 2423
2064 /* Now copy in args */ 2424 /* Set up license info based on the info section */
2065 args = strndup_user(uargs, ~0UL >> 1); 2425 set_license(mod, get_modinfo(info, "license"));
2066 if (IS_ERR(args)) {
2067 err = PTR_ERR(args);
2068 goto free_hdr;
2069 }
2070 2426
2071 strmap = kzalloc(BITS_TO_LONGS(sechdrs[strindex].sh_size) 2427 return 0;
2072 * sizeof(long), GFP_KERNEL); 2428}
2073 if (!strmap) {
2074 err = -ENOMEM;
2075 goto free_mod;
2076 }
2077 2429
2078 if (find_module(mod->name)) { 2430static void find_module_sections(struct module *mod, struct load_info *info)
2079 err = -EEXIST; 2431{
2080 goto free_mod; 2432 mod->kp = section_objs(info, "__param",
2081 } 2433 sizeof(*mod->kp), &mod->num_kp);
2434 mod->syms = section_objs(info, "__ksymtab",
2435 sizeof(*mod->syms), &mod->num_syms);
2436 mod->crcs = section_addr(info, "__kcrctab");
2437 mod->gpl_syms = section_objs(info, "__ksymtab_gpl",
2438 sizeof(*mod->gpl_syms),
2439 &mod->num_gpl_syms);
2440 mod->gpl_crcs = section_addr(info, "__kcrctab_gpl");
2441 mod->gpl_future_syms = section_objs(info,
2442 "__ksymtab_gpl_future",
2443 sizeof(*mod->gpl_future_syms),
2444 &mod->num_gpl_future_syms);
2445 mod->gpl_future_crcs = section_addr(info, "__kcrctab_gpl_future");
2082 2446
2083 mod->state = MODULE_STATE_COMING; 2447#ifdef CONFIG_UNUSED_SYMBOLS
2448 mod->unused_syms = section_objs(info, "__ksymtab_unused",
2449 sizeof(*mod->unused_syms),
2450 &mod->num_unused_syms);
2451 mod->unused_crcs = section_addr(info, "__kcrctab_unused");
2452 mod->unused_gpl_syms = section_objs(info, "__ksymtab_unused_gpl",
2453 sizeof(*mod->unused_gpl_syms),
2454 &mod->num_unused_gpl_syms);
2455 mod->unused_gpl_crcs = section_addr(info, "__kcrctab_unused_gpl");
2456#endif
2457#ifdef CONFIG_CONSTRUCTORS
2458 mod->ctors = section_objs(info, ".ctors",
2459 sizeof(*mod->ctors), &mod->num_ctors);
2460#endif
2084 2461
2085 /* Allow arches to frob section contents and sizes. */ 2462#ifdef CONFIG_TRACEPOINTS
2086 err = module_frob_arch_sections(hdr, sechdrs, secstrings, mod); 2463 mod->tracepoints = section_objs(info, "__tracepoints",
2087 if (err < 0) 2464 sizeof(*mod->tracepoints),
2088 goto free_mod; 2465 &mod->num_tracepoints);
2466#endif
2467#ifdef HAVE_JUMP_LABEL
2468 mod->jump_entries = section_objs(info, "__jump_table",
2469 sizeof(*mod->jump_entries),
2470 &mod->num_jump_entries);
2471#endif
2472#ifdef CONFIG_EVENT_TRACING
2473 mod->trace_events = section_objs(info, "_ftrace_events",
2474 sizeof(*mod->trace_events),
2475 &mod->num_trace_events);
2476 /*
2477 * This section contains pointers to allocated objects in the trace
2478 * code and not scanning it leads to false positives.
2479 */
2480 kmemleak_scan_area(mod->trace_events, sizeof(*mod->trace_events) *
2481 mod->num_trace_events, GFP_KERNEL);
2482#endif
2483#ifdef CONFIG_TRACING
2484 mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt",
2485 sizeof(*mod->trace_bprintk_fmt_start),
2486 &mod->num_trace_bprintk_fmt);
2487 /*
2488 * This section contains pointers to allocated objects in the trace
2489 * code and not scanning it leads to false positives.
2490 */
2491 kmemleak_scan_area(mod->trace_bprintk_fmt_start,
2492 sizeof(*mod->trace_bprintk_fmt_start) *
2493 mod->num_trace_bprintk_fmt, GFP_KERNEL);
2494#endif
2495#ifdef CONFIG_FTRACE_MCOUNT_RECORD
2496 /* sechdrs[0].sh_size is always zero */
2497 mod->ftrace_callsites = section_objs(info, "__mcount_loc",
2498 sizeof(*mod->ftrace_callsites),
2499 &mod->num_ftrace_callsites);
2500#endif
2089 2501
2090 if (pcpuindex) { 2502 mod->extable = section_objs(info, "__ex_table",
2091 /* We have a special allocation for this section. */ 2503 sizeof(*mod->extable), &mod->num_exentries);
2092 percpu = percpu_modalloc(sechdrs[pcpuindex].sh_size,
2093 sechdrs[pcpuindex].sh_addralign,
2094 mod->name);
2095 if (!percpu) {
2096 err = -ENOMEM;
2097 goto free_mod;
2098 }
2099 sechdrs[pcpuindex].sh_flags &= ~(unsigned long)SHF_ALLOC;
2100 mod->percpu = percpu;
2101 }
2102 2504
2103 /* Determine total sizes, and put offsets in sh_entsize. For now 2505 if (section_addr(info, "__obsparm"))
2104 this is done generically; there doesn't appear to be any 2506 printk(KERN_WARNING "%s: Ignoring obsolete parameters\n",
2105 special cases for the architectures. */ 2507 mod->name);
2106 layout_sections(mod, hdr, sechdrs, secstrings); 2508
2107 symoffs = layout_symtab(mod, sechdrs, symindex, strindex, hdr, 2509 info->debug = section_objs(info, "__verbose",
2108 secstrings, &stroffs, strmap); 2510 sizeof(*info->debug), &info->num_debug);
2511}
2512
2513static int move_module(struct module *mod, struct load_info *info)
2514{
2515 int i;
2516 void *ptr;
2109 2517
2110 /* Do the allocs. */ 2518 /* Do the allocs. */
2111 ptr = module_alloc_update_bounds(mod->core_size); 2519 ptr = module_alloc_update_bounds(mod->core_size);
@@ -2115,10 +2523,9 @@ static noinline struct module *load_module(void __user *umod,
2115 * leak. 2523 * leak.
2116 */ 2524 */
2117 kmemleak_not_leak(ptr); 2525 kmemleak_not_leak(ptr);
2118 if (!ptr) { 2526 if (!ptr)
2119 err = -ENOMEM; 2527 return -ENOMEM;
2120 goto free_percpu; 2528
2121 }
2122 memset(ptr, 0, mod->core_size); 2529 memset(ptr, 0, mod->core_size);
2123 mod->module_core = ptr; 2530 mod->module_core = ptr;
2124 2531
@@ -2131,56 +2538,40 @@ static noinline struct module *load_module(void __user *umod,
2131 */ 2538 */
2132 kmemleak_ignore(ptr); 2539 kmemleak_ignore(ptr);
2133 if (!ptr && mod->init_size) { 2540 if (!ptr && mod->init_size) {
2134 err = -ENOMEM; 2541 module_free(mod, mod->module_core);
2135 goto free_core; 2542 return -ENOMEM;
2136 } 2543 }
2137 memset(ptr, 0, mod->init_size); 2544 memset(ptr, 0, mod->init_size);
2138 mod->module_init = ptr; 2545 mod->module_init = ptr;
2139 2546
2140 /* Transfer each section which specifies SHF_ALLOC */ 2547 /* Transfer each section which specifies SHF_ALLOC */
2141 DEBUGP("final section addresses:\n"); 2548 DEBUGP("final section addresses:\n");
2142 for (i = 0; i < hdr->e_shnum; i++) { 2549 for (i = 0; i < info->hdr->e_shnum; i++) {
2143 void *dest; 2550 void *dest;
2551 Elf_Shdr *shdr = &info->sechdrs[i];
2144 2552
2145 if (!(sechdrs[i].sh_flags & SHF_ALLOC)) 2553 if (!(shdr->sh_flags & SHF_ALLOC))
2146 continue; 2554 continue;
2147 2555
2148 if (sechdrs[i].sh_entsize & INIT_OFFSET_MASK) 2556 if (shdr->sh_entsize & INIT_OFFSET_MASK)
2149 dest = mod->module_init 2557 dest = mod->module_init
2150 + (sechdrs[i].sh_entsize & ~INIT_OFFSET_MASK); 2558 + (shdr->sh_entsize & ~INIT_OFFSET_MASK);
2151 else 2559 else
2152 dest = mod->module_core + sechdrs[i].sh_entsize; 2560 dest = mod->module_core + shdr->sh_entsize;
2153 2561
2154 if (sechdrs[i].sh_type != SHT_NOBITS) 2562 if (shdr->sh_type != SHT_NOBITS)
2155 memcpy(dest, (void *)sechdrs[i].sh_addr, 2563 memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
2156 sechdrs[i].sh_size);
2157 /* Update sh_addr to point to copy in image. */ 2564 /* Update sh_addr to point to copy in image. */
2158 sechdrs[i].sh_addr = (unsigned long)dest; 2565 shdr->sh_addr = (unsigned long)dest;
2159 DEBUGP("\t0x%lx %s\n", sechdrs[i].sh_addr, secstrings + sechdrs[i].sh_name); 2566 DEBUGP("\t0x%lx %s\n",
2160 } 2567 shdr->sh_addr, info->secstrings + shdr->sh_name);
2161 /* Module has been moved. */
2162 mod = (void *)sechdrs[modindex].sh_addr;
2163 kmemleak_load_module(mod, hdr, sechdrs, secstrings);
2164
2165#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP)
2166 mod->refptr = percpu_modalloc(sizeof(local_t), __alignof__(local_t),
2167 mod->name);
2168 if (!mod->refptr) {
2169 err = -ENOMEM;
2170 goto free_init;
2171 } 2568 }
2172#endif
2173 /* Now we've moved module, initialize linked lists, etc. */
2174 module_unload_init(mod);
2175 2569
2176 /* add kobject, so we can reference it. */ 2570 return 0;
2177 err = mod_sysfs_init(mod); 2571}
2178 if (err)
2179 goto free_unload;
2180
2181 /* Set up license info based on the info section */
2182 set_license(mod, get_modinfo(sechdrs, infoindex, "license"));
2183 2572
2573static int check_module_license_and_versions(struct module *mod)
2574{
2184 /* 2575 /*
2185 * ndiswrapper is under GPL by itself, but loads proprietary modules. 2576 * ndiswrapper is under GPL by itself, but loads proprietary modules.
2186 * Don't use add_taint_module(), as it would prevent ndiswrapper from 2577 * Don't use add_taint_module(), as it would prevent ndiswrapper from
@@ -2193,77 +2584,6 @@ static noinline struct module *load_module(void __user *umod,
2193 if (strcmp(mod->name, "driverloader") == 0) 2584 if (strcmp(mod->name, "driverloader") == 0)
2194 add_taint_module(mod, TAINT_PROPRIETARY_MODULE); 2585 add_taint_module(mod, TAINT_PROPRIETARY_MODULE);
2195 2586
2196 /* Set up MODINFO_ATTR fields */
2197 setup_modinfo(mod, sechdrs, infoindex);
2198
2199 /* Fix up syms, so that st_value is a pointer to location. */
2200 err = simplify_symbols(sechdrs, symindex, strtab, versindex, pcpuindex,
2201 mod);
2202 if (err < 0)
2203 goto cleanup;
2204
2205 /* Now we've got everything in the final locations, we can
2206 * find optional sections. */
2207 mod->kp = section_objs(hdr, sechdrs, secstrings, "__param",
2208 sizeof(*mod->kp), &mod->num_kp);
2209 mod->syms = section_objs(hdr, sechdrs, secstrings, "__ksymtab",
2210 sizeof(*mod->syms), &mod->num_syms);
2211 mod->crcs = section_addr(hdr, sechdrs, secstrings, "__kcrctab");
2212 mod->gpl_syms = section_objs(hdr, sechdrs, secstrings, "__ksymtab_gpl",
2213 sizeof(*mod->gpl_syms),
2214 &mod->num_gpl_syms);
2215 mod->gpl_crcs = section_addr(hdr, sechdrs, secstrings, "__kcrctab_gpl");
2216 mod->gpl_future_syms = section_objs(hdr, sechdrs, secstrings,
2217 "__ksymtab_gpl_future",
2218 sizeof(*mod->gpl_future_syms),
2219 &mod->num_gpl_future_syms);
2220 mod->gpl_future_crcs = section_addr(hdr, sechdrs, secstrings,
2221 "__kcrctab_gpl_future");
2222
2223#ifdef CONFIG_UNUSED_SYMBOLS
2224 mod->unused_syms = section_objs(hdr, sechdrs, secstrings,
2225 "__ksymtab_unused",
2226 sizeof(*mod->unused_syms),
2227 &mod->num_unused_syms);
2228 mod->unused_crcs = section_addr(hdr, sechdrs, secstrings,
2229 "__kcrctab_unused");
2230 mod->unused_gpl_syms = section_objs(hdr, sechdrs, secstrings,
2231 "__ksymtab_unused_gpl",
2232 sizeof(*mod->unused_gpl_syms),
2233 &mod->num_unused_gpl_syms);
2234 mod->unused_gpl_crcs = section_addr(hdr, sechdrs, secstrings,
2235 "__kcrctab_unused_gpl");
2236#endif
2237#ifdef CONFIG_CONSTRUCTORS
2238 mod->ctors = section_objs(hdr, sechdrs, secstrings, ".ctors",
2239 sizeof(*mod->ctors), &mod->num_ctors);
2240#endif
2241
2242#ifdef CONFIG_TRACEPOINTS
2243 mod->tracepoints = section_objs(hdr, sechdrs, secstrings,
2244 "__tracepoints",
2245 sizeof(*mod->tracepoints),
2246 &mod->num_tracepoints);
2247#endif
2248#ifdef CONFIG_EVENT_TRACING
2249 mod->trace_events = section_objs(hdr, sechdrs, secstrings,
2250 "_ftrace_events",
2251 sizeof(*mod->trace_events),
2252 &mod->num_trace_events);
2253 /*
2254 * This section contains pointers to allocated objects in the trace
2255 * code and not scanning it leads to false positives.
2256 */
2257 kmemleak_scan_area(mod->trace_events, sizeof(*mod->trace_events) *
2258 mod->num_trace_events, GFP_KERNEL);
2259#endif
2260#ifdef CONFIG_FTRACE_MCOUNT_RECORD
2261 /* sechdrs[0].sh_size is always zero */
2262 mod->ftrace_callsites = section_objs(hdr, sechdrs, secstrings,
2263 "__mcount_loc",
2264 sizeof(*mod->ftrace_callsites),
2265 &mod->num_ftrace_callsites);
2266#endif
2267#ifdef CONFIG_MODVERSIONS 2587#ifdef CONFIG_MODVERSIONS
2268 if ((mod->num_syms && !mod->crcs) 2588 if ((mod->num_syms && !mod->crcs)
2269 || (mod->num_gpl_syms && !mod->gpl_crcs) 2589 || (mod->num_gpl_syms && !mod->gpl_crcs)
@@ -2273,67 +2593,16 @@ static noinline struct module *load_module(void __user *umod,
2273 || (mod->num_unused_gpl_syms && !mod->unused_gpl_crcs) 2593 || (mod->num_unused_gpl_syms && !mod->unused_gpl_crcs)
2274#endif 2594#endif
2275 ) { 2595 ) {
2276 err = try_to_force_load(mod, 2596 return try_to_force_load(mod,
2277 "no versions for exported symbols"); 2597 "no versions for exported symbols");
2278 if (err)
2279 goto cleanup;
2280 } 2598 }
2281#endif 2599#endif
2600 return 0;
2601}
2282 2602
2283 /* Now do relocations. */ 2603static void flush_module_icache(const struct module *mod)
2284 for (i = 1; i < hdr->e_shnum; i++) { 2604{
2285 const char *strtab = (char *)sechdrs[strindex].sh_addr; 2605 mm_segment_t old_fs;
2286 unsigned int info = sechdrs[i].sh_info;
2287
2288 /* Not a valid relocation section? */
2289 if (info >= hdr->e_shnum)
2290 continue;
2291
2292 /* Don't bother with non-allocated sections */
2293 if (!(sechdrs[info].sh_flags & SHF_ALLOC))
2294 continue;
2295
2296 if (sechdrs[i].sh_type == SHT_REL)
2297 err = apply_relocate(sechdrs, strtab, symindex, i,mod);
2298 else if (sechdrs[i].sh_type == SHT_RELA)
2299 err = apply_relocate_add(sechdrs, strtab, symindex, i,
2300 mod);
2301 if (err < 0)
2302 goto cleanup;
2303 }
2304
2305 /* Find duplicate symbols */
2306 err = verify_export_symbols(mod);
2307 if (err < 0)
2308 goto cleanup;
2309
2310 /* Set up and sort exception table */
2311 mod->extable = section_objs(hdr, sechdrs, secstrings, "__ex_table",
2312 sizeof(*mod->extable), &mod->num_exentries);
2313 sort_extable(mod->extable, mod->extable + mod->num_exentries);
2314
2315 /* Finally, copy percpu area over. */
2316 percpu_modcopy(mod->percpu, (void *)sechdrs[pcpuindex].sh_addr,
2317 sechdrs[pcpuindex].sh_size);
2318
2319 add_kallsyms(mod, sechdrs, hdr->e_shnum, symindex, strindex,
2320 symoffs, stroffs, secstrings, strmap);
2321 kfree(strmap);
2322 strmap = NULL;
2323
2324 if (!mod->taints) {
2325 struct _ddebug *debug;
2326 unsigned int num_debug;
2327
2328 debug = section_objs(hdr, sechdrs, secstrings, "__verbose",
2329 sizeof(*debug), &num_debug);
2330 if (debug)
2331 dynamic_debug_setup(debug, num_debug);
2332 }
2333
2334 err = module_finalize(hdr, sechdrs, mod);
2335 if (err < 0)
2336 goto cleanup;
2337 2606
2338 /* flush the icache in correct context */ 2607 /* flush the icache in correct context */
2339 old_fs = get_fs(); 2608 old_fs = get_fs();
@@ -2352,11 +2621,160 @@ static noinline struct module *load_module(void __user *umod,
2352 (unsigned long)mod->module_core + mod->core_size); 2621 (unsigned long)mod->module_core + mod->core_size);
2353 2622
2354 set_fs(old_fs); 2623 set_fs(old_fs);
2624}
2355 2625
2356 mod->args = args; 2626static struct module *layout_and_allocate(struct load_info *info)
2357 if (section_addr(hdr, sechdrs, secstrings, "__obsparm")) 2627{
2358 printk(KERN_WARNING "%s: Ignoring obsolete parameters\n", 2628 /* Module within temporary copy. */
2359 mod->name); 2629 struct module *mod;
2630 Elf_Shdr *pcpusec;
2631 int err;
2632
2633 mod = setup_load_info(info);
2634 if (IS_ERR(mod))
2635 return mod;
2636
2637 err = check_modinfo(mod, info);
2638 if (err)
2639 return ERR_PTR(err);
2640
2641 /* Allow arches to frob section contents and sizes. */
2642 err = module_frob_arch_sections(info->hdr, info->sechdrs,
2643 info->secstrings, mod);
2644 if (err < 0)
2645 goto out;
2646
2647 pcpusec = &info->sechdrs[info->index.pcpu];
2648 if (pcpusec->sh_size) {
2649 /* We have a special allocation for this section. */
2650 err = percpu_modalloc(mod,
2651 pcpusec->sh_size, pcpusec->sh_addralign);
2652 if (err)
2653 goto out;
2654 pcpusec->sh_flags &= ~(unsigned long)SHF_ALLOC;
2655 }
2656
2657 /* Determine total sizes, and put offsets in sh_entsize. For now
2658 this is done generically; there doesn't appear to be any
2659 special cases for the architectures. */
2660 layout_sections(mod, info);
2661
2662 info->strmap = kzalloc(BITS_TO_LONGS(info->sechdrs[info->index.str].sh_size)
2663 * sizeof(long), GFP_KERNEL);
2664 if (!info->strmap) {
2665 err = -ENOMEM;
2666 goto free_percpu;
2667 }
2668 layout_symtab(mod, info);
2669
2670 /* Allocate and move to the final place */
2671 err = move_module(mod, info);
2672 if (err)
2673 goto free_strmap;
2674
2675 /* Module has been copied to its final place now: return it. */
2676 mod = (void *)info->sechdrs[info->index.mod].sh_addr;
2677 kmemleak_load_module(mod, info);
2678 return mod;
2679
2680free_strmap:
2681 kfree(info->strmap);
2682free_percpu:
2683 percpu_modfree(mod);
2684out:
2685 return ERR_PTR(err);
2686}
2687
2688/* mod is no longer valid after this! */
2689static void module_deallocate(struct module *mod, struct load_info *info)
2690{
2691 kfree(info->strmap);
2692 percpu_modfree(mod);
2693 module_free(mod, mod->module_init);
2694 module_free(mod, mod->module_core);
2695}
2696
2697static int post_relocation(struct module *mod, const struct load_info *info)
2698{
2699 /* Sort exception table now relocations are done. */
2700 sort_extable(mod->extable, mod->extable + mod->num_exentries);
2701
2702 /* Copy relocated percpu area over. */
2703 percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
2704 info->sechdrs[info->index.pcpu].sh_size);
2705
2706 /* Setup kallsyms-specific fields. */
2707 add_kallsyms(mod, info);
2708
2709 /* Arch-specific module finalizing. */
2710 return module_finalize(info->hdr, info->sechdrs, mod);
2711}
2712
2713/* Allocate and load the module: note that size of section 0 is always
2714 zero, and we rely on this for optional sections. */
2715static struct module *load_module(void __user *umod,
2716 unsigned long len,
2717 const char __user *uargs)
2718{
2719 struct load_info info = { NULL, };
2720 struct module *mod;
2721 long err;
2722
2723 DEBUGP("load_module: umod=%p, len=%lu, uargs=%p\n",
2724 umod, len, uargs);
2725
2726 /* Copy in the blobs from userspace, check they are vaguely sane. */
2727 err = copy_and_check(&info, umod, len, uargs);
2728 if (err)
2729 return ERR_PTR(err);
2730
2731 /* Figure out module layout, and allocate all the memory. */
2732 mod = layout_and_allocate(&info);
2733 if (IS_ERR(mod)) {
2734 err = PTR_ERR(mod);
2735 goto free_copy;
2736 }
2737
2738 /* Now module is in final location, initialize linked lists, etc. */
2739 err = module_unload_init(mod);
2740 if (err)
2741 goto free_module;
2742
2743 /* Now we've got everything in the final locations, we can
2744 * find optional sections. */
2745 find_module_sections(mod, &info);
2746
2747 err = check_module_license_and_versions(mod);
2748 if (err)
2749 goto free_unload;
2750
2751 /* Set up MODINFO_ATTR fields */
2752 setup_modinfo(mod, &info);
2753
2754 /* Fix up syms, so that st_value is a pointer to location. */
2755 err = simplify_symbols(mod, &info);
2756 if (err < 0)
2757 goto free_modinfo;
2758
2759 err = apply_relocations(mod, &info);
2760 if (err < 0)
2761 goto free_modinfo;
2762
2763 err = post_relocation(mod, &info);
2764 if (err < 0)
2765 goto free_modinfo;
2766
2767 flush_module_icache(mod);
2768
2769 /* Now copy in args */
2770 mod->args = strndup_user(uargs, ~0UL >> 1);
2771 if (IS_ERR(mod->args)) {
2772 err = PTR_ERR(mod->args);
2773 goto free_arch_cleanup;
2774 }
2775
2776 /* Mark state as coming so strong_try_module_get() ignores us. */
2777 mod->state = MODULE_STATE_COMING;
2360 2778
2361 /* Now sew it into the lists so we can get lockdep and oops 2779 /* Now sew it into the lists so we can get lockdep and oops
2362 * info during argument parsing. Noone should access us, since 2780 * info during argument parsing. Noone should access us, since
@@ -2365,59 +2783,67 @@ static noinline struct module *load_module(void __user *umod,
2365 * function to insert in a way safe to concurrent readers. 2783 * function to insert in a way safe to concurrent readers.
2366 * The mutex protects against concurrent writers. 2784 * The mutex protects against concurrent writers.
2367 */ 2785 */
2786 mutex_lock(&module_mutex);
2787 if (find_module(mod->name)) {
2788 err = -EEXIST;
2789 goto unlock;
2790 }
2791
2792 /* This has to be done once we're sure module name is unique. */
2793 if (!mod->taints)
2794 dynamic_debug_setup(info.debug, info.num_debug);
2795
2796 /* Find duplicate symbols */
2797 err = verify_export_symbols(mod);
2798 if (err < 0)
2799 goto ddebug;
2800
2801 module_bug_finalize(info.hdr, info.sechdrs, mod);
2368 list_add_rcu(&mod->list, &modules); 2802 list_add_rcu(&mod->list, &modules);
2803 mutex_unlock(&module_mutex);
2369 2804
2805 /* Module is ready to execute: parsing args may do that. */
2370 err = parse_args(mod->name, mod->args, mod->kp, mod->num_kp, NULL); 2806 err = parse_args(mod->name, mod->args, mod->kp, mod->num_kp, NULL);
2371 if (err < 0) 2807 if (err < 0)
2372 goto unlink; 2808 goto unlink;
2373 2809
2374 err = mod_sysfs_setup(mod, mod->kp, mod->num_kp); 2810 /* Link in to syfs. */
2811 err = mod_sysfs_setup(mod, &info, mod->kp, mod->num_kp);
2375 if (err < 0) 2812 if (err < 0)
2376 goto unlink; 2813 goto unlink;
2377 add_sect_attrs(mod, hdr->e_shnum, secstrings, sechdrs);
2378 add_notes_attrs(mod, hdr->e_shnum, secstrings, sechdrs);
2379 2814
2380 /* Get rid of temporary copy */ 2815 /* Get rid of temporary copy and strmap. */
2381 vfree(hdr); 2816 kfree(info.strmap);
2382 2817 free_copy(&info);
2383 trace_module_load(mod);
2384 2818
2385 /* Done! */ 2819 /* Done! */
2820 trace_module_load(mod);
2386 return mod; 2821 return mod;
2387 2822
2388 unlink: 2823 unlink:
2824 mutex_lock(&module_mutex);
2389 /* Unlink carefully: kallsyms could be walking list. */ 2825 /* Unlink carefully: kallsyms could be walking list. */
2390 list_del_rcu(&mod->list); 2826 list_del_rcu(&mod->list);
2827 module_bug_cleanup(mod);
2828
2829 ddebug:
2830 if (!mod->taints)
2831 dynamic_debug_remove(info.debug);
2832 unlock:
2833 mutex_unlock(&module_mutex);
2391 synchronize_sched(); 2834 synchronize_sched();
2835 kfree(mod->args);
2836 free_arch_cleanup:
2392 module_arch_cleanup(mod); 2837 module_arch_cleanup(mod);
2393 cleanup: 2838 free_modinfo:
2394 free_modinfo(mod); 2839 free_modinfo(mod);
2395 kobject_del(&mod->mkobj.kobj);
2396 kobject_put(&mod->mkobj.kobj);
2397 free_unload: 2840 free_unload:
2398 module_unload_free(mod); 2841 module_unload_free(mod);
2399#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP) 2842 free_module:
2400 percpu_modfree(mod->refptr); 2843 module_deallocate(mod, &info);
2401 free_init: 2844 free_copy:
2402#endif 2845 free_copy(&info);
2403 module_free(mod, mod->module_init);
2404 free_core:
2405 module_free(mod, mod->module_core);
2406 /* mod will be freed with core. Don't access it beyond this line! */
2407 free_percpu:
2408 if (percpu)
2409 percpu_modfree(percpu);
2410 free_mod:
2411 kfree(args);
2412 kfree(strmap);
2413 free_hdr:
2414 vfree(hdr);
2415 return ERR_PTR(err); 2846 return ERR_PTR(err);
2416
2417 truncated:
2418 printk(KERN_ERR "Module len %lu truncated\n", len);
2419 err = -ENOEXEC;
2420 goto free_hdr;
2421} 2847}
2422 2848
2423/* Call module constructors. */ 2849/* Call module constructors. */
@@ -2442,23 +2868,26 @@ SYSCALL_DEFINE3(init_module, void __user *, umod,
2442 if (!capable(CAP_SYS_MODULE) || modules_disabled) 2868 if (!capable(CAP_SYS_MODULE) || modules_disabled)
2443 return -EPERM; 2869 return -EPERM;
2444 2870
2445 /* Only one module load at a time, please */
2446 if (mutex_lock_interruptible(&module_mutex) != 0)
2447 return -EINTR;
2448
2449 /* Do all the hard work */ 2871 /* Do all the hard work */
2450 mod = load_module(umod, len, uargs); 2872 mod = load_module(umod, len, uargs);
2451 if (IS_ERR(mod)) { 2873 if (IS_ERR(mod))
2452 mutex_unlock(&module_mutex);
2453 return PTR_ERR(mod); 2874 return PTR_ERR(mod);
2454 }
2455
2456 /* Drop lock so they can recurse */
2457 mutex_unlock(&module_mutex);
2458 2875
2459 blocking_notifier_call_chain(&module_notify_list, 2876 blocking_notifier_call_chain(&module_notify_list,
2460 MODULE_STATE_COMING, mod); 2877 MODULE_STATE_COMING, mod);
2461 2878
2879 /* Set RO and NX regions for core */
2880 set_section_ro_nx(mod->module_core,
2881 mod->core_text_size,
2882 mod->core_ro_size,
2883 mod->core_size);
2884
2885 /* Set RO and NX regions for init */
2886 set_section_ro_nx(mod->module_init,
2887 mod->init_text_size,
2888 mod->init_ro_size,
2889 mod->init_size);
2890
2462 do_mod_ctors(mod); 2891 do_mod_ctors(mod);
2463 /* Start the module */ 2892 /* Start the module */
2464 if (mod->init != NULL) 2893 if (mod->init != NULL)
@@ -2471,9 +2900,7 @@ SYSCALL_DEFINE3(init_module, void __user *, umod,
2471 module_put(mod); 2900 module_put(mod);
2472 blocking_notifier_call_chain(&module_notify_list, 2901 blocking_notifier_call_chain(&module_notify_list,
2473 MODULE_STATE_GOING, mod); 2902 MODULE_STATE_GOING, mod);
2474 mutex_lock(&module_mutex);
2475 free_module(mod); 2903 free_module(mod);
2476 mutex_unlock(&module_mutex);
2477 wake_up(&module_wq); 2904 wake_up(&module_wq);
2478 return ret; 2905 return ret;
2479 } 2906 }
@@ -2504,6 +2931,7 @@ SYSCALL_DEFINE3(init_module, void __user *, umod,
2504 mod->symtab = mod->core_symtab; 2931 mod->symtab = mod->core_symtab;
2505 mod->strtab = mod->core_strtab; 2932 mod->strtab = mod->core_strtab;
2506#endif 2933#endif
2934 unset_section_ro_nx(mod, mod->module_init);
2507 module_free(mod, mod->module_init); 2935 module_free(mod, mod->module_init);
2508 mod->module_init = NULL; 2936 mod->module_init = NULL;
2509 mod->init_size = 0; 2937 mod->init_size = 0;
generated by cgit v1.2.2 (git 2.25.0) at 2025-05-07 08:35:40 -0400