aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/nommu-mmap.txt18
-rw-r--r--arch/arm/include/asm/mmu.h1
-rw-r--r--arch/blackfin/include/asm/mmu.h1
-rw-r--r--arch/blackfin/kernel/ptrace.c6
-rw-r--r--arch/blackfin/kernel/traps.c11
-rw-r--r--arch/frv/kernel/ptrace.c11
-rw-r--r--arch/h8300/include/asm/mmu.h1
-rw-r--r--arch/m68knommu/include/asm/mmu.h1
-rw-r--r--arch/sh/include/asm/mmu.h1
-rw-r--r--fs/binfmt_elf_fdpic.c27
-rw-r--r--fs/proc/internal.h2
-rw-r--r--fs/proc/meminfo.c6
-rw-r--r--fs/proc/nommu.c71
-rw-r--r--fs/proc/task_nommu.c108
-rw-r--r--include/asm-frv/mmu.h1
-rw-r--r--include/asm-m32r/mmu.h1
-rw-r--r--include/linux/mm.h18
-rw-r--r--include/linux/mm_types.h18
-rw-r--r--ipc/shm.c12
-rw-r--r--kernel/fork.c4
-rw-r--r--lib/Kconfig.debug7
-rw-r--r--mm/mmap.c10
-rw-r--r--mm/nommu.c960
23 files changed, 860 insertions, 436 deletions
diff --git a/Documentation/nommu-mmap.txt b/Documentation/nommu-mmap.txt
index 7714f57caad..02b89dcf38a 100644
--- a/Documentation/nommu-mmap.txt
+++ b/Documentation/nommu-mmap.txt
@@ -109,12 +109,18 @@ and it's also much more restricted in the latter case:
109FURTHER NOTES ON NO-MMU MMAP 109FURTHER NOTES ON NO-MMU MMAP
110============================ 110============================
111 111
112 (*) A request for a private mapping of less than a page in size may not return 112 (*) A request for a private mapping of a file may return a buffer that is not
113 a page-aligned buffer. This is because the kernel calls kmalloc() to 113 page-aligned. This is because XIP may take place, and the data may not be
114 allocate the buffer, not get_free_page(). 114 paged aligned in the backing store.
115 115
116 (*) A list of all the mappings on the system is visible through /proc/maps in 116 (*) A request for an anonymous mapping will always be page aligned. If
117 no-MMU mode. 117 possible the size of the request should be a power of two otherwise some
118 of the space may be wasted as the kernel must allocate a power-of-2
119 granule but will only discard the excess if appropriately configured as
120 this has an effect on fragmentation.
121
122 (*) A list of all the private copy and anonymous mappings on the system is
123 visible through /proc/maps in no-MMU mode.
118 124
119 (*) A list of all the mappings in use by a process is visible through 125 (*) A list of all the mappings in use by a process is visible through
120 /proc/<pid>/maps in no-MMU mode. 126 /proc/<pid>/maps in no-MMU mode.
diff --git a/arch/arm/include/asm/mmu.h b/arch/arm/include/asm/mmu.h
index 53099d4ee42..b561584d04a 100644
--- a/arch/arm/include/asm/mmu.h
+++ b/arch/arm/include/asm/mmu.h
@@ -24,7 +24,6 @@ typedef struct {
24 * modified for 2.6 by Hyok S. Choi <hyok.choi@samsung.com> 24 * modified for 2.6 by Hyok S. Choi <hyok.choi@samsung.com>
25 */ 25 */
26typedef struct { 26typedef struct {
27 struct vm_list_struct *vmlist;
28 unsigned long end_brk; 27 unsigned long end_brk;
29} mm_context_t; 28} mm_context_t;
30 29
diff --git a/arch/blackfin/include/asm/mmu.h b/arch/blackfin/include/asm/mmu.h
index 757e43906ed..dbfd686360e 100644
--- a/arch/blackfin/include/asm/mmu.h
+++ b/arch/blackfin/include/asm/mmu.h
@@ -10,7 +10,6 @@ struct sram_list_struct {
10}; 10};
11 11
12typedef struct { 12typedef struct {
13 struct vm_list_struct *vmlist;
14 unsigned long end_brk; 13 unsigned long end_brk;
15 unsigned long stack_start; 14 unsigned long stack_start;
16 15
diff --git a/arch/blackfin/kernel/ptrace.c b/arch/blackfin/kernel/ptrace.c
index d2d38853663..594e325b40e 100644
--- a/arch/blackfin/kernel/ptrace.c
+++ b/arch/blackfin/kernel/ptrace.c
@@ -160,15 +160,15 @@ put_reg(struct task_struct *task, int regno, unsigned long data)
160static inline int is_user_addr_valid(struct task_struct *child, 160static inline int is_user_addr_valid(struct task_struct *child,
161 unsigned long start, unsigned long len) 161 unsigned long start, unsigned long len)
162{ 162{
163 struct vm_list_struct *vml; 163 struct vm_area_struct *vma;
164 struct sram_list_struct *sraml; 164 struct sram_list_struct *sraml;
165 165
166 /* overflow */ 166 /* overflow */
167 if (start + len < start) 167 if (start + len < start)
168 return -EIO; 168 return -EIO;
169 169
170 for (vml = child->mm->context.vmlist; vml; vml = vml->next) 170 vma = find_vma(child->mm, start);
171 if (start >= vml->vma->vm_start && start + len < vml->vma->vm_end) 171 if (vma && start >= vma->vm_start && start + len <= vma->vm_end)
172 return 0; 172 return 0;
173 173
174 for (sraml = child->mm->context.sram_list; sraml; sraml = sraml->next) 174 for (sraml = child->mm->context.sram_list; sraml; sraml = sraml->next)
diff --git a/arch/blackfin/kernel/traps.c b/arch/blackfin/kernel/traps.c
index 17d8e417289..5b0667da8d0 100644
--- a/arch/blackfin/kernel/traps.c
+++ b/arch/blackfin/kernel/traps.c
@@ -32,6 +32,7 @@
32#include <linux/module.h> 32#include <linux/module.h>
33#include <linux/kallsyms.h> 33#include <linux/kallsyms.h>
34#include <linux/fs.h> 34#include <linux/fs.h>
35#include <linux/rbtree.h>
35#include <asm/traps.h> 36#include <asm/traps.h>
36#include <asm/cacheflush.h> 37#include <asm/cacheflush.h>
37#include <asm/cplb.h> 38#include <asm/cplb.h>
@@ -83,6 +84,7 @@ static void decode_address(char *buf, unsigned long address)
83 struct mm_struct *mm; 84 struct mm_struct *mm;
84 unsigned long flags, offset; 85 unsigned long flags, offset;
85 unsigned char in_atomic = (bfin_read_IPEND() & 0x10) || in_atomic(); 86 unsigned char in_atomic = (bfin_read_IPEND() & 0x10) || in_atomic();
87 struct rb_node *n;
86 88
87#ifdef CONFIG_KALLSYMS 89#ifdef CONFIG_KALLSYMS
88 unsigned long symsize; 90 unsigned long symsize;
@@ -128,9 +130,10 @@ static void decode_address(char *buf, unsigned long address)
128 if (!mm) 130 if (!mm)
129 continue; 131 continue;
130 132
131 vml = mm->context.vmlist; 133 for (n = rb_first(&mm->mm_rb); n; n = rb_next(n)) {
132 while (vml) { 134 struct vm_area_struct *vma;
133 struct vm_area_struct *vma = vml->vma; 135
136 vma = rb_entry(n, struct vm_area_struct, vm_rb);
134 137
135 if (address >= vma->vm_start && address < vma->vm_end) { 138 if (address >= vma->vm_start && address < vma->vm_end) {
136 char _tmpbuf[256]; 139 char _tmpbuf[256];
@@ -176,8 +179,6 @@ static void decode_address(char *buf, unsigned long address)
176 179
177 goto done; 180 goto done;
178 } 181 }
179
180 vml = vml->next;
181 } 182 }
182 if (!in_atomic) 183 if (!in_atomic)
183 mmput(mm); 184 mmput(mm);
diff --git a/arch/frv/kernel/ptrace.c b/arch/frv/kernel/ptrace.c
index 709e9bdc612..5e7d401d21e 100644
--- a/arch/frv/kernel/ptrace.c
+++ b/arch/frv/kernel/ptrace.c
@@ -69,7 +69,8 @@ static inline int put_reg(struct task_struct *task, int regno,
69} 69}
70 70
71/* 71/*
72 * check that an address falls within the bounds of the target process's memory mappings 72 * check that an address falls within the bounds of the target process's memory
73 * mappings
73 */ 74 */
74static inline int is_user_addr_valid(struct task_struct *child, 75static inline int is_user_addr_valid(struct task_struct *child,
75 unsigned long start, unsigned long len) 76 unsigned long start, unsigned long len)
@@ -79,11 +80,11 @@ static inline int is_user_addr_valid(struct task_struct *child,
79 return -EIO; 80 return -EIO;
80 return 0; 81 return 0;
81#else 82#else
82 struct vm_list_struct *vml; 83 struct vm_area_struct *vma;
83 84
84 for (vml = child->mm->context.vmlist; vml; vml = vml->next) 85 vma = find_vma(child->mm, start);
85 if (start >= vml->vma->vm_start && start + len <= vml->vma->vm_end) 86 if (vma && start >= vma->vm_start && start + len <= vma->vm_end)
86 return 0; 87 return 0;
87 88
88 return -EIO; 89 return -EIO;
89#endif 90#endif
diff --git a/arch/h8300/include/asm/mmu.h b/arch/h8300/include/asm/mmu.h
index 2ce06ea4610..31309969df7 100644
--- a/arch/h8300/include/asm/mmu.h
+++ b/arch/h8300/include/asm/mmu.h
@@ -4,7 +4,6 @@
4/* Copyright (C) 2002, David McCullough <davidm@snapgear.com> */ 4/* Copyright (C) 2002, David McCullough <davidm@snapgear.com> */
5 5
6typedef struct { 6typedef struct {
7 struct vm_list_struct *vmlist;
8 unsigned long end_brk; 7 unsigned long end_brk;
9} mm_context_t; 8} mm_context_t;
10 9
diff --git a/arch/m68knommu/include/asm/mmu.h b/arch/m68knommu/include/asm/mmu.h
index 5fa6b68353b..e2da1e6f09f 100644
--- a/arch/m68knommu/include/asm/mmu.h
+++ b/arch/m68knommu/include/asm/mmu.h
@@ -4,7 +4,6 @@
4/* Copyright (C) 2002, David McCullough <davidm@snapgear.com> */ 4/* Copyright (C) 2002, David McCullough <davidm@snapgear.com> */
5 5
6typedef struct { 6typedef struct {
7 struct vm_list_struct *vmlist;
8 unsigned long end_brk; 7 unsigned long end_brk;
9} mm_context_t; 8} mm_context_t;
10 9
diff --git a/arch/sh/include/asm/mmu.h b/arch/sh/include/asm/mmu.h
index fdcb93bc6d1..6c43625bb1a 100644
--- a/arch/sh/include/asm/mmu.h
+++ b/arch/sh/include/asm/mmu.h
@@ -9,7 +9,6 @@ typedef struct {
9 mm_context_id_t id; 9 mm_context_id_t id;
10 void *vdso; 10 void *vdso;
11#else 11#else
12 struct vm_list_struct *vmlist;
13 unsigned long end_brk; 12 unsigned long end_brk;
14#endif 13#endif
15#ifdef CONFIG_BINFMT_ELF_FDPIC 14#ifdef CONFIG_BINFMT_ELF_FDPIC
diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c
index aa5b43205e3..22baf1b1349 100644
--- a/fs/binfmt_elf_fdpic.c
+++ b/fs/binfmt_elf_fdpic.c
@@ -1567,11 +1567,9 @@ end_coredump:
1567static int elf_fdpic_dump_segments(struct file *file, size_t *size, 1567static int elf_fdpic_dump_segments(struct file *file, size_t *size,
1568 unsigned long *limit, unsigned long mm_flags) 1568 unsigned long *limit, unsigned long mm_flags)
1569{ 1569{
1570 struct vm_list_struct *vml; 1570 struct vm_area_struct *vma;
1571
1572 for (vml = current->mm->context.vmlist; vml; vml = vml->next) {
1573 struct vm_area_struct *vma = vml->vma;
1574 1571
1572 for (vma = current->mm->mmap; vma; vma = vma->vm_next) {
1575 if (!maydump(vma, mm_flags)) 1573 if (!maydump(vma, mm_flags))
1576 continue; 1574 continue;
1577 1575
@@ -1617,9 +1615,6 @@ static int elf_fdpic_core_dump(long signr, struct pt_regs *regs,
1617 elf_fpxregset_t *xfpu = NULL; 1615 elf_fpxregset_t *xfpu = NULL;
1618#endif 1616#endif
1619 int thread_status_size = 0; 1617 int thread_status_size = 0;
1620#ifndef CONFIG_MMU
1621 struct vm_list_struct *vml;
1622#endif
1623 elf_addr_t *auxv; 1618 elf_addr_t *auxv;
1624 unsigned long mm_flags; 1619 unsigned long mm_flags;
1625 1620
@@ -1685,13 +1680,7 @@ static int elf_fdpic_core_dump(long signr, struct pt_regs *regs,
1685 fill_prstatus(prstatus, current, signr); 1680 fill_prstatus(prstatus, current, signr);
1686 elf_core_copy_regs(&prstatus->pr_reg, regs); 1681 elf_core_copy_regs(&prstatus->pr_reg, regs);
1687 1682
1688#ifdef CONFIG_MMU
1689 segs = current->mm->map_count; 1683 segs = current->mm->map_count;
1690#else
1691 segs = 0;
1692 for (vml = current->mm->context.vmlist; vml; vml = vml->next)
1693 segs++;
1694#endif
1695#ifdef ELF_CORE_EXTRA_PHDRS 1684#ifdef ELF_CORE_EXTRA_PHDRS
1696 segs += ELF_CORE_EXTRA_PHDRS; 1685 segs += ELF_CORE_EXTRA_PHDRS;
1697#endif 1686#endif
@@ -1766,20 +1755,10 @@ static int elf_fdpic_core_dump(long signr, struct pt_regs *regs,
1766 mm_flags = current->mm->flags; 1755 mm_flags = current->mm->flags;
1767 1756
1768 /* write program headers for segments dump */ 1757 /* write program headers for segments dump */
1769 for ( 1758 for (vma = current->mm->mmap; vma; vma = vma->vm_next) {
1770#ifdef CONFIG_MMU
1771 vma = current->mm->mmap; vma; vma = vma->vm_next
1772#else
1773 vml = current->mm->context.vmlist; vml; vml = vml->next
1774#endif
1775 ) {
1776 struct elf_phdr phdr; 1759 struct elf_phdr phdr;
1777 size_t sz; 1760 size_t sz;
1778 1761
1779#ifndef CONFIG_MMU
1780 vma = vml->vma;
1781#endif
1782
1783 sz = vma->vm_end - vma->vm_start; 1762 sz = vma->vm_end - vma->vm_start;
1784 1763
1785 phdr.p_type = PT_LOAD; 1764 phdr.p_type = PT_LOAD;
diff --git a/fs/proc/internal.h b/fs/proc/internal.h
index 3e8aeb8b61c..cd53ff83849 100644
--- a/fs/proc/internal.h
+++ b/fs/proc/internal.h
@@ -41,8 +41,6 @@ do { \
41 (vmi)->used = 0; \ 41 (vmi)->used = 0; \
42 (vmi)->largest_chunk = 0; \ 42 (vmi)->largest_chunk = 0; \
43} while(0) 43} while(0)
44
45extern int nommu_vma_show(struct seq_file *, struct vm_area_struct *);
46#endif 44#endif
47 45
48extern int proc_tid_stat(struct seq_file *m, struct pid_namespace *ns, 46extern int proc_tid_stat(struct seq_file *m, struct pid_namespace *ns,
diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c
index b1675c4e66d..43d23948384 100644
--- a/fs/proc/meminfo.c
+++ b/fs/proc/meminfo.c
@@ -74,6 +74,9 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
74 "LowTotal: %8lu kB\n" 74 "LowTotal: %8lu kB\n"
75 "LowFree: %8lu kB\n" 75 "LowFree: %8lu kB\n"
76#endif 76#endif
77#ifndef CONFIG_MMU
78 "MmapCopy: %8lu kB\n"
79#endif
77 "SwapTotal: %8lu kB\n" 80 "SwapTotal: %8lu kB\n"
78 "SwapFree: %8lu kB\n" 81 "SwapFree: %8lu kB\n"
79 "Dirty: %8lu kB\n" 82 "Dirty: %8lu kB\n"
@@ -116,6 +119,9 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
116 K(i.totalram-i.totalhigh), 119 K(i.totalram-i.totalhigh),
117 K(i.freeram-i.freehigh), 120 K(i.freeram-i.freehigh),
118#endif 121#endif
122#ifndef CONFIG_MMU
123 K((unsigned long) atomic_read(&mmap_pages_allocated)),
124#endif
119 K(i.totalswap), 125 K(i.totalswap),
120 K(i.freeswap), 126 K(i.freeswap),
121 K(global_page_state(NR_FILE_DIRTY)), 127 K(global_page_state(NR_FILE_DIRTY)),
diff --git a/fs/proc/nommu.c b/fs/proc/nommu.c
index 3f87d263294..b446d7ad0b0 100644
--- a/fs/proc/nommu.c
+++ b/fs/proc/nommu.c
@@ -33,33 +33,33 @@
33#include "internal.h" 33#include "internal.h"
34 34
35/* 35/*
36 * display a single VMA to a sequenced file 36 * display a single region to a sequenced file
37 */ 37 */
38int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma) 38static int nommu_region_show(struct seq_file *m, struct vm_region *region)
39{ 39{
40 unsigned long ino = 0; 40 unsigned long ino = 0;
41 struct file *file; 41 struct file *file;
42 dev_t dev = 0; 42 dev_t dev = 0;
43 int flags, len; 43 int flags, len;
44 44
45 flags = vma->vm_flags; 45 flags = region->vm_flags;
46 file = vma->vm_file; 46 file = region->vm_file;
47 47
48 if (file) { 48 if (file) {
49 struct inode *inode = vma->vm_file->f_path.dentry->d_inode; 49 struct inode *inode = region->vm_file->f_path.dentry->d_inode;
50 dev = inode->i_sb->s_dev; 50 dev = inode->i_sb->s_dev;
51 ino = inode->i_ino; 51 ino = inode->i_ino;
52 } 52 }
53 53
54 seq_printf(m, 54 seq_printf(m,
55 "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu %n", 55 "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu %n",
56 vma->vm_start, 56 region->vm_start,
57 vma->vm_end, 57 region->vm_end,
58 flags & VM_READ ? 'r' : '-', 58 flags & VM_READ ? 'r' : '-',
59 flags & VM_WRITE ? 'w' : '-', 59 flags & VM_WRITE ? 'w' : '-',
60 flags & VM_EXEC ? 'x' : '-', 60 flags & VM_EXEC ? 'x' : '-',
61 flags & VM_MAYSHARE ? flags & VM_SHARED ? 'S' : 's' : 'p', 61 flags & VM_MAYSHARE ? flags & VM_SHARED ? 'S' : 's' : 'p',
62 ((loff_t)vma->vm_pgoff) << PAGE_SHIFT, 62 ((loff_t)region->vm_pgoff) << PAGE_SHIFT,
63 MAJOR(dev), MINOR(dev), ino, &len); 63 MAJOR(dev), MINOR(dev), ino, &len);
64 64
65 if (file) { 65 if (file) {
@@ -75,61 +75,54 @@ int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma)
75} 75}
76 76
77/* 77/*
78 * display a list of all the VMAs the kernel knows about 78 * display a list of all the REGIONs the kernel knows about
79 * - nommu kernals have a single flat list 79 * - nommu kernals have a single flat list
80 */ 80 */
81static int nommu_vma_list_show(struct seq_file *m, void *v) 81static int nommu_region_list_show(struct seq_file *m, void *_p)
82{ 82{
83 struct vm_area_struct *vma; 83 struct rb_node *p = _p;
84 84
85 vma = rb_entry((struct rb_node *) v, struct vm_area_struct, vm_rb); 85 return nommu_region_show(m, rb_entry(p, struct vm_region, vm_rb));
86 return nommu_vma_show(m, vma);
87} 86}
88 87
89static void *nommu_vma_list_start(struct seq_file *m, loff_t *_pos) 88static void *nommu_region_list_start(struct seq_file *m, loff_t *_pos)
90{ 89{
91 struct rb_node *_rb; 90 struct rb_node *p;
92 loff_t pos = *_pos; 91 loff_t pos = *_pos;
93 void *next = NULL;
94 92
95 down_read(&nommu_vma_sem); 93 down_read(&nommu_region_sem);
96 94
97 for (_rb = rb_first(&nommu_vma_tree); _rb; _rb = rb_next(_rb)) { 95 for (p = rb_first(&nommu_region_tree); p; p = rb_next(p))
98 if (pos == 0) { 96 if (pos-- == 0)
99 next = _rb; 97 return p;
100 break; 98 return NULL;
101 }
102 pos--;
103 }
104
105 return next;
106} 99}
107 100
108static void nommu_vma_list_stop(struct seq_file *m, void *v) 101static void nommu_region_list_stop(struct seq_file *m, void *v)
109{ 102{
110 up_read(&nommu_vma_sem); 103 up_read(&nommu_region_sem);
111} 104}
112 105
113static void *nommu_vma_list_next(struct seq_file *m, void *v, loff_t *pos) 106static void *nommu_region_list_next(struct seq_file *m, void *v, loff_t *pos)
114{ 107{
115 (*pos)++; 108 (*pos)++;
116 return rb_next((struct rb_node *) v); 109 return rb_next((struct rb_node *) v);
117} 110}
118 111
119static const struct seq_operations proc_nommu_vma_list_seqop = { 112static struct seq_operations proc_nommu_region_list_seqop = {
120 .start = nommu_vma_list_start, 113 .start = nommu_region_list_start,
121 .next = nommu_vma_list_next, 114 .next = nommu_region_list_next,
122 .stop = nommu_vma_list_stop, 115 .stop = nommu_region_list_stop,
123 .show = nommu_vma_list_show 116 .show = nommu_region_list_show
124}; 117};
125 118
126static int proc_nommu_vma_list_open(struct inode *inode, struct file *file) 119static int proc_nommu_region_list_open(struct inode *inode, struct file *file)
127{ 120{
128 return seq_open(file, &proc_nommu_vma_list_seqop); 121 return seq_open(file, &proc_nommu_region_list_seqop);
129} 122}
130 123
131static const struct file_operations proc_nommu_vma_list_operations = { 124static const struct file_operations proc_nommu_region_list_operations = {
132 .open = proc_nommu_vma_list_open, 125 .open = proc_nommu_region_list_open,
133 .read = seq_read, 126 .read = seq_read,
134 .llseek = seq_lseek, 127 .llseek = seq_lseek,
135 .release = seq_release, 128 .release = seq_release,
@@ -137,7 +130,7 @@ static const struct file_operations proc_nommu_vma_list_operations = {
137 130
138static int __init proc_nommu_init(void) 131static int __init proc_nommu_init(void)
139{ 132{
140 proc_create("maps", S_IRUGO, NULL, &proc_nommu_vma_list_operations); 133 proc_create("maps", S_IRUGO, NULL, &proc_nommu_region_list_operations);
141 return 0; 134 return 0;
142} 135}
143 136
diff --git a/fs/proc/task_nommu.c b/fs/proc/task_nommu.c
index d4a8be32b90..ca4a48d0d31 100644
--- a/fs/proc/task_nommu.c
+++ b/fs/proc/task_nommu.c
@@ -15,25 +15,25 @@
15 */ 15 */
16void task_mem(struct seq_file *m, struct mm_struct *mm) 16void task_mem(struct seq_file *m, struct mm_struct *mm)
17{ 17{
18 struct vm_list_struct *vml; 18 struct vm_area_struct *vma;
19 struct rb_node *p;
19 unsigned long bytes = 0, sbytes = 0, slack = 0; 20 unsigned long bytes = 0, sbytes = 0, slack = 0;
20 21
21 down_read(&mm->mmap_sem); 22 down_read(&mm->mmap_sem);
22 for (vml = mm->context.vmlist; vml; vml = vml->next) { 23 for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) {
23 if (!vml->vma) 24 vma = rb_entry(p, struct vm_area_struct, vm_rb);
24 continue;
25 25
26 bytes += kobjsize(vml); 26 bytes += kobjsize(vma);
27 if (atomic_read(&mm->mm_count) > 1 || 27 if (atomic_read(&mm->mm_count) > 1 ||
28 atomic_read(&vml->vma->vm_usage) > 1 28 vma->vm_region ||
29 ) { 29 vma->vm_flags & VM_MAYSHARE) {
30 sbytes += kobjsize((void *) vml->vma->vm_start); 30 sbytes += kobjsize((void *) vma->vm_start);
31 sbytes += kobjsize(vml->vma); 31 if (vma->vm_region)
32 sbytes += kobjsize(vma->vm_region);
32 } else { 33 } else {
33 bytes += kobjsize((void *) vml->vma->vm_start); 34 bytes += kobjsize((void *) vma->vm_start);
34 bytes += kobjsize(vml->vma); 35 slack += kobjsize((void *) vma->vm_start) -
35 slack += kobjsize((void *) vml->vma->vm_start) - 36 (vma->vm_end - vma->vm_start);
36 (vml->vma->vm_end - vml->vma->vm_start);
37 } 37 }
38 } 38 }
39 39
@@ -70,13 +70,14 @@ void task_mem(struct seq_file *m, struct mm_struct *mm)
70 70
71unsigned long task_vsize(struct mm_struct *mm) 71unsigned long task_vsize(struct mm_struct *mm)
72{ 72{
73 struct vm_list_struct *tbp; 73 struct vm_area_struct *vma;
74 struct rb_node *p;
74 unsigned long vsize = 0; 75 unsigned long vsize = 0;
75 76
76 down_read(&mm->mmap_sem); 77 down_read(&mm->mmap_sem);
77 for (tbp = mm->context.vmlist; tbp; tbp = tbp->next) { 78 for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) {
78 if (tbp->vma) 79 vma = rb_entry(p, struct vm_area_struct, vm_rb);
79 vsize += kobjsize((void *) tbp->vma->vm_start); 80 vsize += vma->vm_region->vm_end - vma->vm_region->vm_start;
80 } 81 }
81 up_read(&mm->mmap_sem); 82 up_read(&mm->mmap_sem);
82 return vsize; 83 return vsize;
@@ -85,16 +86,15 @@ unsigned long task_vsize(struct mm_struct *mm)
85int task_statm(struct mm_struct *mm, int *shared, int *text, 86int task_statm(struct mm_struct *mm, int *shared, int *text,
86 int *data, int *resident) 87 int *data, int *resident)
87{ 88{
88 struct vm_list_struct *tbp; 89 struct vm_area_struct *vma;
90 struct rb_node *p;
89 int size = kobjsize(mm); 91 int size = kobjsize(mm);
90 92
91 down_read(&mm->mmap_sem); 93 down_read(&mm->mmap_sem);
92 for (tbp = mm->context.vmlist; tbp; tbp = tbp->next) { 94 for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) {
93 size += kobjsize(tbp); 95 vma = rb_entry(p, struct vm_area_struct, vm_rb);
94 if (tbp->vma) { 96 size += kobjsize(vma);
95 size += kobjsize(tbp->vma); 97 size += kobjsize((void *) vma->vm_start);
96 size += kobjsize((void *) tbp->vma->vm_start);
97 }
98 } 98 }
99 99
100 size += (*text = mm->end_code - mm->start_code); 100 size += (*text = mm->end_code - mm->start_code);
@@ -105,20 +105,62 @@ int task_statm(struct mm_struct *mm, int *shared, int *text,
105} 105}
106 106
107/* 107/*
108 * display a single VMA to a sequenced file
109 */
110static int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma)
111{
112 unsigned long ino = 0;
113 struct file *file;
114 dev_t dev = 0;
115 int flags, len;
116
117 flags = vma->vm_flags;
118 file = vma->vm_file;
119
120 if (file) {
121 struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
122 dev = inode->i_sb->s_dev;
123 ino = inode->i_ino;
124 }
125
126 seq_printf(m,
127 "%08lx-%08lx %c%c%c%c %08lx %02x:%02x %lu %n",
128 vma->vm_start,
129 vma->vm_end,
130 flags & VM_READ ? 'r' : '-',
131 flags & VM_WRITE ? 'w' : '-',
132 flags & VM_EXEC ? 'x' : '-',
133 flags & VM_MAYSHARE ? flags & VM_SHARED ? 'S' : 's' : 'p',
134 vma->vm_pgoff << PAGE_SHIFT,
135 MAJOR(dev), MINOR(dev), ino, &len);
136
137 if (file) {
138 len = 25 + sizeof(void *) * 6 - len;
139 if (len < 1)
140 len = 1;
141 seq_printf(m, "%*c", len, ' ');
142 seq_path(m, &file->f_path, "");
143 }
144
145 seq_putc(m, '\n');
146 return 0;
147}
148
149/*
108 * display mapping lines for a particular process's /proc/pid/maps 150 * display mapping lines for a particular process's /proc/pid/maps
109 */ 151 */
110static int show_map(struct seq_file *m, void *_vml) 152static int show_map(struct seq_file *m, void *_p)
111{ 153{
112 struct vm_list_struct *vml = _vml; 154 struct rb_node *p = _p;
113 155
114 return nommu_vma_show(m, vml->vma); 156 return nommu_vma_show(m, rb_entry(p, struct vm_area_struct, vm_rb));
115} 157}
116 158
117static void *m_start(struct seq_file *m, loff_t *pos) 159static void *m_start(struct seq_file *m, loff_t *pos)
118{ 160{
119 struct proc_maps_private *priv = m->private; 161 struct proc_maps_private *priv = m->private;
120 struct vm_list_struct *vml;
121 struct mm_struct *mm; 162 struct mm_struct *mm;
163 struct rb_node *p;
122 loff_t n = *pos; 164 loff_t n = *pos;
123 165
124 /* pin the task and mm whilst we play with them */ 166 /* pin the task and mm whilst we play with them */
@@ -134,9 +176,9 @@ static void *m_start(struct seq_file *m, loff_t *pos)
134 } 176 }
135 177
136 /* start from the Nth VMA */ 178 /* start from the Nth VMA */
137 for (vml = mm->context.vmlist; vml; vml = vml->next) 179 for (p = rb_first(&mm->mm_rb); p; p = rb_next(p))
138 if (n-- == 0) 180 if (n-- == 0)
139 return vml; 181 return p;
140 return NULL; 182 return NULL;
141} 183}
142 184
@@ -152,12 +194,12 @@ static void m_stop(struct seq_file *m, void *_vml)
152 } 194 }
153} 195}
154 196
155static void *m_next(struct seq_file *m, void *_vml, loff_t *pos) 197static void *m_next(struct seq_file *m, void *_p, loff_t *pos)
156{ 198{
157 struct vm_list_struct *vml = _vml; 199 struct rb_node *p = _p;
158 200
159 (*pos)++; 201 (*pos)++;
160 return vml ? vml->next : NULL; 202 return p ? rb_next(p) : NULL;
161} 203}
162 204
163static const struct seq_operations proc_pid_maps_ops = { 205static const struct seq_operations proc_pid_maps_ops = {
diff --git a/include/asm-frv/mmu.h b/include/asm-frv/mmu.h
index 22c03714fb1..86ca0e86e7d 100644
--- a/include/asm-frv/mmu.h
+++ b/include/asm-frv/mmu.h
@@ -22,7 +22,6 @@ typedef struct {
22 unsigned long dtlb_ptd_mapping; /* [DAMR5] PTD mapping for dtlb cached PGE */ 22 unsigned long dtlb_ptd_mapping; /* [DAMR5] PTD mapping for dtlb cached PGE */
23 23
24#else 24#else
25 struct vm_list_struct *vmlist;
26 unsigned long end_brk; 25 unsigned long end_brk;
27 26
28#endif 27#endif
diff --git a/include/asm-m32r/mmu.h b/include/asm-m32r/mmu.h
index d9bd724479c..150cb92bb66 100644
--- a/include/asm-m32r/mmu.h
+++ b/include/asm-m32r/mmu.h
@@ -4,7 +4,6 @@
4#if !defined(CONFIG_MMU) 4#if !defined(CONFIG_MMU)
5 5
6typedef struct { 6typedef struct {
7 struct vm_list_struct *vmlist;
8 unsigned long end_brk; 7 unsigned long end_brk;
9} mm_context_t; 8} mm_context_t;
10 9
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 4a3d28c8644..b91a73fd1bc 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -56,19 +56,9 @@ extern unsigned long mmap_min_addr;
56 56
57extern struct kmem_cache *vm_area_cachep; 57extern struct kmem_cache *vm_area_cachep;
58 58
59/*
60 * This struct defines the per-mm list of VMAs for uClinux. If CONFIG_MMU is
61 * disabled, then there's a single shared list of VMAs maintained by the
62 * system, and mm's subscribe to these individually
63 */
64struct vm_list_struct {
65 struct vm_list_struct *next;
66 struct vm_area_struct *vma;
67};
68
69#ifndef CONFIG_MMU 59#ifndef CONFIG_MMU
70extern struct rb_root nommu_vma_tree; 60extern struct rb_root nommu_region_tree;
71extern struct rw_semaphore nommu_vma_sem; 61extern struct rw_semaphore nommu_region_sem;
72 62
73extern unsigned int kobjsize(const void *objp); 63extern unsigned int kobjsize(const void *objp);
74#endif 64#endif
@@ -1061,6 +1051,7 @@ extern void memmap_init_zone(unsigned long, int, unsigned long,
1061 unsigned long, enum memmap_context); 1051 unsigned long, enum memmap_context);
1062extern void setup_per_zone_pages_min(void); 1052extern void setup_per_zone_pages_min(void);
1063extern void mem_init(void); 1053extern void mem_init(void);
1054extern void __init mmap_init(void);
1064extern void show_mem(void); 1055extern void show_mem(void);
1065extern void si_meminfo(struct sysinfo * val); 1056extern void si_meminfo(struct sysinfo * val);
1066extern void si_meminfo_node(struct sysinfo *val, int nid); 1057extern void si_meminfo_node(struct sysinfo *val, int nid);
@@ -1072,6 +1063,9 @@ extern void setup_per_cpu_pageset(void);
1072static inline void setup_per_cpu_pageset(void) {} 1063static inline void setup_per_cpu_pageset(void) {}
1073#endif 1064#endif
1074 1065
1066/* nommu.c */
1067extern atomic_t mmap_pages_allocated;
1068
1075/* prio_tree.c */ 1069/* prio_tree.c */
1076void vma_prio_tree_add(struct vm_area_struct *, struct vm_area_struct *old); 1070void vma_prio_tree_add(struct vm_area_struct *, struct vm_area_struct *old);
1077void vma_prio_tree_insert(struct vm_area_struct *, struct prio_tree_root *); 1071void vma_prio_tree_insert(struct vm_area_struct *, struct prio_tree_root *);
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 9cfc9b627fd..1c1e0d3a171 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -97,6 +97,22 @@ struct page {
97}; 97};
98 98
99/* 99/*
100 * A region containing a mapping of a non-memory backed file under NOMMU
101 * conditions. These are held in a global tree and are pinned by the VMAs that
102 * map parts of them.
103 */
104struct vm_region {
105 struct rb_node vm_rb; /* link in global region tree */
106 unsigned long vm_flags; /* VMA vm_flags */
107 unsigned long vm_start; /* start address of region */
108 unsigned long vm_end; /* region initialised to here */
109 unsigned long vm_pgoff; /* the offset in vm_file corresponding to vm_start */
110 struct file *vm_file; /* the backing file or NULL */
111
112 atomic_t vm_usage; /* region usage count */
113};
114
115/*
100 * This struct defines a memory VMM memory area. There is one of these 116 * This struct defines a memory VMM memory area. There is one of these
101 * per VM-area/task. A VM area is any part of the process virtual memory 117 * per VM-area/task. A VM area is any part of the process virtual memory
102 * space that has a special rule for the page-fault handlers (ie a shared 118 * space that has a special rule for the page-fault handlers (ie a shared
@@ -152,7 +168,7 @@ struct vm_area_struct {
152 unsigned long vm_truncate_count;/* truncate_count or restart_addr */ 168 unsigned long vm_truncate_count;/* truncate_count or restart_addr */
153 169
154#ifndef CONFIG_MMU 170#ifndef CONFIG_MMU
155 atomic_t vm_usage; /* refcount (VMAs shared if !MMU) */ 171 struct vm_region *vm_region; /* NOMMU mapping region */
156#endif 172#endif
157#ifdef CONFIG_NUMA 173#ifdef CONFIG_NUMA
158 struct mempolicy *vm_policy; /* NUMA policy for the VMA */ 174 struct mempolicy *vm_policy; /* NUMA policy for the VMA */
diff --git a/ipc/shm.c b/ipc/shm.c
index b125b560240..d0ab5527bf4 100644
--- a/ipc/shm.c
+++ b/ipc/shm.c
@@ -990,6 +990,7 @@ asmlinkage long sys_shmdt(char __user *shmaddr)
990 */ 990 */
991 vma = find_vma(mm, addr); 991 vma = find_vma(mm, addr);
992 992
993#ifdef CONFIG_MMU
993 while (vma) { 994 while (vma) {
994 next = vma->vm_next; 995 next = vma->vm_next;
995 996
@@ -1034,6 +1035,17 @@ asmlinkage long sys_shmdt(char __user *shmaddr)
1034 vma = next; 1035 vma = next;
1035 } 1036 }
1036 1037
1038#else /* CONFIG_MMU */
1039 /* under NOMMU conditions, the exact address to be destroyed must be
1040 * given */
1041 retval = -EINVAL;
1042 if (vma->vm_start == addr && vma->vm_ops == &shm_vm_ops) {
1043 do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start);
1044 retval = 0;
1045 }
1046
1047#endif
1048
1037 up_write(&mm->mmap_sem); 1049 up_write(&mm->mmap_sem);
1038 return retval; 1050 return retval;
1039} 1051}
diff --git a/kernel/fork.c b/kernel/fork.c
index 7b8f2a78be3..0bce4a43bb3 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1481,12 +1481,10 @@ void __init proc_caches_init(void)
1481 fs_cachep = kmem_cache_create("fs_cache", 1481 fs_cachep = kmem_cache_create("fs_cache",
1482 sizeof(struct fs_struct), 0, 1482 sizeof(struct fs_struct), 0,
1483 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); 1483 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
1484 vm_area_cachep = kmem_cache_create("vm_area_struct",
1485 sizeof(struct vm_area_struct), 0,
1486 SLAB_PANIC, NULL);
1487 mm_cachep = kmem_cache_create("mm_struct", 1484 mm_cachep = kmem_cache_create("mm_struct",
1488 sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN, 1485 sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN,
1489 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); 1486 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
1487 mmap_init();
1490} 1488}
1491 1489
1492/* 1490/*
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 2e75478e9c6..d0a32aab03f 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -512,6 +512,13 @@ config DEBUG_VIRTUAL
512 512
513 If unsure, say N. 513 If unsure, say N.
514 514
515config DEBUG_NOMMU_REGIONS
516 bool "Debug the global anon/private NOMMU mapping region tree"
517 depends on DEBUG_KERNEL && !MMU
518 help
519 This option causes the global tree of anonymous and private mapping
520 regions to be regularly checked for invalid topology.
521
515config DEBUG_WRITECOUNT 522config DEBUG_WRITECOUNT
516 bool "Debug filesystem writers count" 523 bool "Debug filesystem writers count"
517 depends on DEBUG_KERNEL 524 depends on DEBUG_KERNEL
diff --git a/mm/mmap.c b/mm/mmap.c
index a910c045cfd..749623196cb 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -2472,3 +2472,13 @@ void mm_drop_all_locks(struct mm_struct *mm)
2472 2472
2473 mutex_unlock(&mm_all_locks_mutex); 2473 mutex_unlock(&mm_all_locks_mutex);
2474} 2474}
2475
2476/*
2477 * initialise the VMA slab
2478 */
2479void __init mmap_init(void)
2480{
2481 vm_area_cachep = kmem_cache_create("vm_area_struct",
2482 sizeof(struct vm_area_struct), 0,
2483 SLAB_PANIC, NULL);
2484}
diff --git a/mm/nommu.c b/mm/nommu.c
index 23f355bbe26..0d363dfcf10 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -6,7 +6,7 @@
6 * 6 *
7 * See Documentation/nommu-mmap.txt 7 * See Documentation/nommu-mmap.txt
8 * 8 *
9 * Copyright (c) 2004-2005 David Howells <dhowells@redhat.com> 9 * Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>
10 * Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com> 10 * Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>
11 * Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org> 11 * Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org>
12 * Copyright (c) 2002 Greg Ungerer <gerg@snapgear.com> 12 * Copyright (c) 2002 Greg Ungerer <gerg@snapgear.com>
@@ -33,6 +33,28 @@
33#include <asm/uaccess.h> 33#include <asm/uaccess.h>
34#include <asm/tlb.h> 34#include <asm/tlb.h>
35#include <asm/tlbflush.h> 35#include <asm/tlbflush.h>
36#include "internal.h"
37
38static inline __attribute__((format(printf, 1, 2)))
39void no_printk(const char *fmt, ...)
40{
41}
42
43#if 0
44#define kenter(FMT, ...) \
45 printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__)
46#define kleave(FMT, ...) \
47 printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__)
48#define kdebug(FMT, ...) \
49 printk(KERN_DEBUG "xxx" FMT"yyy\n", ##__VA_ARGS__)
50#else
51#define kenter(FMT, ...) \
52 no_printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__)
53#define kleave(FMT, ...) \
54 no_printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__)
55#define kdebug(FMT, ...) \
56 no_printk(KERN_DEBUG FMT"\n", ##__VA_ARGS__)
57#endif
36 58
37#include "internal.h" 59#include "internal.h"
38 60
@@ -46,12 +68,15 @@ int sysctl_overcommit_ratio = 50; /* default is 50% */
46int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT; 68int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT;
47int heap_stack_gap = 0; 69int heap_stack_gap = 0;
48 70
71atomic_t mmap_pages_allocated;
72
49EXPORT_SYMBOL(mem_map); 73EXPORT_SYMBOL(mem_map);
50EXPORT_SYMBOL(num_physpages); 74EXPORT_SYMBOL(num_physpages);
51 75
52/* list of shareable VMAs */ 76/* list of mapped, potentially shareable regions */
53struct rb_root nommu_vma_tree = RB_ROOT; 77static struct kmem_cache *vm_region_jar;
54DECLARE_RWSEM(nommu_vma_sem); 78struct rb_root nommu_region_tree = RB_ROOT;
79DECLARE_RWSEM(nommu_region_sem);
55 80
56struct vm_operations_struct generic_file_vm_ops = { 81struct vm_operations_struct generic_file_vm_ops = {
57}; 82};
@@ -400,129 +425,174 @@ asmlinkage unsigned long sys_brk(unsigned long brk)
400 return mm->brk = brk; 425 return mm->brk = brk;
401} 426}
402 427
403#ifdef DEBUG 428/*
404static void show_process_blocks(void) 429 * initialise the VMA and region record slabs
430 */
431void __init mmap_init(void)
405{ 432{
406 struct vm_list_struct *vml; 433 vm_region_jar = kmem_cache_create("vm_region_jar",
407 434 sizeof(struct vm_region), 0,
408 printk("Process blocks %d:", current->pid); 435 SLAB_PANIC, NULL);
409 436 vm_area_cachep = kmem_cache_create("vm_area_struct",
410 for (vml = &current->mm->context.vmlist; vml; vml = vml->next) { 437 sizeof(struct vm_area_struct), 0,
411 printk(" %p: %p", vml, vml->vma); 438 SLAB_PANIC, NULL);
412 if (vml->vma)
413 printk(" (%d @%lx #%d)",
414 kobjsize((void *) vml->vma->vm_start),
415 vml->vma->vm_start,
416 atomic_read(&vml->vma->vm_usage));
417 printk(vml->next ? " ->" : ".\n");
418 }
419} 439}
420#endif /* DEBUG */
421 440
422/* 441/*
423 * add a VMA into a process's mm_struct in the appropriate place in the list 442 * validate the region tree
424 * - should be called with mm->mmap_sem held writelocked 443 * - the caller must hold the region lock
425 */ 444 */
426static void add_vma_to_mm(struct mm_struct *mm, struct vm_list_struct *vml) 445#ifdef CONFIG_DEBUG_NOMMU_REGIONS
446static noinline void validate_nommu_regions(void)
427{ 447{
428 struct vm_list_struct **ppv; 448 struct vm_region *region, *last;
449 struct rb_node *p, *lastp;
429 450
430 for (ppv = &current->mm->context.vmlist; *ppv; ppv = &(*ppv)->next) 451 lastp = rb_first(&nommu_region_tree);
431 if ((*ppv)->vma->vm_start > vml->vma->vm_start) 452 if (!lastp)
432 break; 453 return;
454
455 last = rb_entry(lastp, struct vm_region, vm_rb);
456 if (unlikely(last->vm_end <= last->vm_start))
457 BUG();
458
459 while ((p = rb_next(lastp))) {
460 region = rb_entry(p, struct vm_region, vm_rb);
461 last = rb_entry(lastp, struct vm_region, vm_rb);
462
463 if (unlikely(region->vm_end <= region->vm_start))
464 BUG();
465 if (unlikely(region->vm_start < last->vm_end))
466 BUG();
433 467
434 vml->next = *ppv; 468 lastp = p;
435 *ppv = vml; 469 }
436} 470}
471#else
472#define validate_nommu_regions() do {} while(0)
473#endif
437 474
438/* 475/*
439 * look up the first VMA in which addr resides, NULL if none 476 * add a region into the global tree
440 * - should be called with mm->mmap_sem at least held readlocked
441 */ 477 */
442struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) 478static void add_nommu_region(struct vm_region *region)
443{ 479{
444 struct vm_list_struct *loop, *vml; 480 struct vm_region *pregion;
481 struct rb_node **p, *parent;
445 482
446 /* search the vm_start ordered list */ 483 validate_nommu_regions();
447 vml = NULL; 484
448 for (loop = mm->context.vmlist; loop; loop = loop->next) { 485 BUG_ON(region->vm_start & ~PAGE_MASK);
449 if (loop->vma->vm_start > addr) 486
450 break; 487 parent = NULL;
451 vml = loop; 488 p = &nommu_region_tree.rb_node;
489 while (*p) {
490 parent = *p;
491 pregion = rb_entry(parent, struct vm_region, vm_rb);
492 if (region->vm_start < pregion->vm_start)
493 p = &(*p)->rb_left;
494 else if (region->vm_start > pregion->vm_start)
495 p = &(*p)->rb_right;
496 else if (pregion == region)
497 return;
498 else
499 BUG();
452 } 500 }
453 501
454 if (vml && vml->vma->vm_end > addr) 502 rb_link_node(&region->vm_rb, parent, p);
455 return vml->vma; 503 rb_insert_color(&region->vm_rb, &nommu_region_tree);
456 504
457 return NULL; 505 validate_nommu_regions();
458} 506}
459EXPORT_SYMBOL(find_vma);
460 507
461/* 508/*
462 * find a VMA 509 * delete a region from the global tree
463 * - we don't extend stack VMAs under NOMMU conditions
464 */ 510 */
465struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr) 511static void delete_nommu_region(struct vm_region *region)
466{ 512{
467 return find_vma(mm, addr); 513 BUG_ON(!nommu_region_tree.rb_node);
468}
469 514
470int expand_stack(struct vm_area_struct *vma, unsigned long address) 515 validate_nommu_regions();
471{ 516 rb_erase(&region->vm_rb, &nommu_region_tree);
472 return -ENOMEM; 517 validate_nommu_regions();
473} 518}
474 519
475/* 520/*
476 * look up the first VMA exactly that exactly matches addr 521 * free a contiguous series of pages
477 * - should be called with mm->mmap_sem at least held readlocked
478 */ 522 */
479static inline struct vm_area_struct *find_vma_exact(struct mm_struct *mm, 523static void free_page_series(unsigned long from, unsigned long to)
480 unsigned long addr)
481{ 524{
482 struct vm_list_struct *vml; 525 for (; from < to; from += PAGE_SIZE) {
483 526 struct page *page = virt_to_page(from);
484 /* search the vm_start ordered list */ 527
485 for (vml = mm->context.vmlist; vml; vml = vml->next) { 528 kdebug("- free %lx", from);
486 if (vml->vma->vm_start == addr) 529 atomic_dec(&mmap_pages_allocated);
487 return vml->vma; 530 if (page_count(page) != 1)
488 if (vml->vma->vm_start > addr) 531 kdebug("free page %p [%d]", page, page_count(page));
489 break; 532 put_page(page);
490 } 533 }
491
492 return NULL;
493} 534}
494 535
495/* 536/*
496 * find a VMA in the global tree 537 * release a reference to a region
538 * - the caller must hold the region semaphore, which this releases
539 * - the region may not have been added to the tree yet, in which case vm_end
540 * will equal vm_start
497 */ 541 */
498static inline struct vm_area_struct *find_nommu_vma(unsigned long start) 542static void __put_nommu_region(struct vm_region *region)
543 __releases(nommu_region_sem)
499{ 544{
500 struct vm_area_struct *vma; 545 kenter("%p{%d}", region, atomic_read(&region->vm_usage));
501 struct rb_node *n = nommu_vma_tree.rb_node;
502 546
503 while (n) { 547 BUG_ON(!nommu_region_tree.rb_node);
504 vma = rb_entry(n, struct vm_area_struct, vm_rb);
505 548
506 if (start < vma->vm_start) 549 if (atomic_dec_and_test(&region->vm_usage)) {
507 n = n->rb_left; 550 if (region->vm_end > region->vm_start)
508 else if (start > vma->vm_start) 551 delete_nommu_region(region);
509 n = n->rb_right; 552 up_write(&nommu_region_sem);
510 else 553
511 return vma; 554 if (region->vm_file)
555 fput(region->vm_file);
556
557 /* IO memory and memory shared directly out of the pagecache
558 * from ramfs/tmpfs mustn't be released here */
559 if (region->vm_flags & VM_MAPPED_COPY) {
560 kdebug("free series");
561 free_page_series(region->vm_start, region->vm_end);
562 }
563 kmem_cache_free(vm_region_jar, region);
564 } else {
565 up_write(&nommu_region_sem);
512 } 566 }
567}
513 568
514 return NULL; 569/*
570 * release a reference to a region
571 */
572static void put_nommu_region(struct vm_region *region)
573{
574 down_write(&nommu_region_sem);
575 __put_nommu_region(region);
515} 576}
516 577
517/* 578/*
518 * add a VMA in the global tree 579 * add a VMA into a process's mm_struct in the appropriate place in the list
580 * and tree and add to the address space's page tree also if not an anonymous
581 * page
582 * - should be called with mm->mmap_sem held writelocked
519 */ 583 */
520static void add_nommu_vma(struct vm_area_struct *vma) 584static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
521{ 585{
522 struct vm_area_struct *pvma; 586 struct vm_area_struct *pvma, **pp;
523 struct address_space *mapping; 587 struct address_space *mapping;
524 struct rb_node **p = &nommu_vma_tree.rb_node; 588 struct rb_node **p, *parent;
525 struct rb_node *parent = NULL; 589
590 kenter(",%p", vma);
591
592 BUG_ON(!vma->vm_region);
593
594 mm->map_count++;
595 vma->vm_mm = mm;
526 596
527 /* add the VMA to the mapping */ 597 /* add the VMA to the mapping */
528 if (vma->vm_file) { 598 if (vma->vm_file) {
@@ -533,42 +603,62 @@ static void add_nommu_vma(struct vm_area_struct *vma)
533 flush_dcache_mmap_unlock(mapping); 603 flush_dcache_mmap_unlock(mapping);
534 } 604 }
535 605
536 /* add the VMA to the master list */ 606 /* add the VMA to the tree */
607 parent = NULL;
608 p = &mm->mm_rb.rb_node;
537 while (*p) { 609 while (*p) {
538 parent = *p; 610 parent = *p;
539 pvma = rb_entry(parent, struct vm_area_struct, vm_rb); 611 pvma = rb_entry(parent, struct vm_area_struct, vm_rb);
540 612
541 if (vma->vm_start < pvma->vm_start) { 613 /* sort by: start addr, end addr, VMA struct addr in that order
614 * (the latter is necessary as we may get identical VMAs) */
615 if (vma->vm_start < pvma->vm_start)
542 p = &(*p)->rb_left; 616 p = &(*p)->rb_left;
543 } 617 else if (vma->vm_start > pvma->vm_start)
544 else if (vma->vm_start > pvma->vm_start) {
545 p = &(*p)->rb_right; 618 p = &(*p)->rb_right;
546 } 619 else if (vma->vm_end < pvma->vm_end)
547 else { 620 p = &(*p)->rb_left;
548 /* mappings are at the same address - this can only 621 else if (vma->vm_end > pvma->vm_end)
549 * happen for shared-mem chardevs and shared file 622 p = &(*p)->rb_right;
550 * mappings backed by ramfs/tmpfs */ 623 else if (vma < pvma)
551 BUG_ON(!(pvma->vm_flags & VM_SHARED)); 624 p = &(*p)->rb_left;
552 625 else if (vma > pvma)
553 if (vma < pvma) 626 p = &(*p)->rb_right;
554 p = &(*p)->rb_left; 627 else
555 else if (vma > pvma) 628 BUG();
556 p = &(*p)->rb_right;
557 else
558 BUG();
559 }
560 } 629 }
561 630
562 rb_link_node(&vma->vm_rb, parent, p); 631 rb_link_node(&vma->vm_rb, parent, p);
563 rb_insert_color(&vma->vm_rb, &nommu_vma_tree); 632 rb_insert_color(&vma->vm_rb, &mm->mm_rb);
633
634 /* add VMA to the VMA list also */
635 for (pp = &mm->mmap; (pvma = *pp); pp = &(*pp)->vm_next) {
636 if (pvma->vm_start > vma->vm_start)
637 break;
638 if (pvma->vm_start < vma->vm_start)
639 continue;
640 if (pvma->vm_end < vma->vm_end)
641 break;
642 }
643
644 vma->vm_next = *pp;
645 *pp = vma;
564} 646}
565 647
566/* 648/*
567 * delete a VMA from the global list 649 * delete a VMA from its owning mm_struct and address space
568 */ 650 */
569static void delete_nommu_vma(struct vm_area_struct *vma) 651static void delete_vma_from_mm(struct vm_area_struct *vma)
570{ 652{
653 struct vm_area_struct **pp;
571 struct address_space *mapping; 654 struct address_space *mapping;
655 struct mm_struct *mm = vma->vm_mm;
656
657 kenter("%p", vma);
658
659 mm->map_count--;
660 if (mm->mmap_cache == vma)
661 mm->mmap_cache = NULL;
572 662
573 /* remove the VMA from the mapping */ 663 /* remove the VMA from the mapping */
574 if (vma->vm_file) { 664 if (vma->vm_file) {
@@ -579,8 +669,115 @@ static void delete_nommu_vma(struct vm_area_struct *vma)
579 flush_dcache_mmap_unlock(mapping); 669 flush_dcache_mmap_unlock(mapping);
580 } 670 }
581 671
582 /* remove from the master list */ 672 /* remove from the MM's tree and list */
583 rb_erase(&vma->vm_rb, &nommu_vma_tree); 673 rb_erase(&vma->vm_rb, &mm->mm_rb);
674 for (pp = &mm->mmap; *pp; pp = &(*pp)->vm_next) {
675 if (*pp == vma) {
676 *pp = vma->vm_next;
677 break;
678 }
679 }
680
681 vma->vm_mm = NULL;
682}
683
684/*
685 * destroy a VMA record
686 */
687static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma)
688{
689 kenter("%p", vma);
690 if (vma->vm_ops && vma->vm_ops->close)
691 vma->vm_ops->close(vma);
692 if (vma->vm_file) {
693 fput(vma->vm_file);
694 if (vma->vm_flags & VM_EXECUTABLE)
695 removed_exe_file_vma(mm);
696 }
697 put_nommu_region(vma->vm_region);
698 kmem_cache_free(vm_area_cachep, vma);
699}
700
701/*
702 * look up the first VMA in which addr resides, NULL if none
703 * - should be called with mm->mmap_sem at least held readlocked
704 */
705struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
706{
707 struct vm_area_struct *vma;
708 struct rb_node *n = mm->mm_rb.rb_node;
709
710 /* check the cache first */
711 vma = mm->mmap_cache;
712 if (vma && vma->vm_start <= addr && vma->vm_end > addr)
713 return vma;
714
715 /* trawl the tree (there may be multiple mappings in which addr
716 * resides) */
717 for (n = rb_first(&mm->mm_rb); n; n = rb_next(n)) {
718 vma = rb_entry(n, struct vm_area_struct, vm_rb);
719 if (vma->vm_start > addr)
720 return NULL;
721 if (vma->vm_end > addr) {
722 mm->mmap_cache = vma;
723 return vma;
724 }
725 }
726
727 return NULL;
728}
729EXPORT_SYMBOL(find_vma);
730
731/*
732 * find a VMA
733 * - we don't extend stack VMAs under NOMMU conditions
734 */
735struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
736{
737 return find_vma(mm, addr);
738}
739
740/*
741 * expand a stack to a given address
742 * - not supported under NOMMU conditions
743 */
744int expand_stack(struct vm_area_struct *vma, unsigned long address)
745{
746 return -ENOMEM;
747}
748
749/*
750 * look up the first VMA exactly that exactly matches addr
751 * - should be called with mm->mmap_sem at least held readlocked
752 */
753static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
754 unsigned long addr,
755 unsigned long len)
756{
757 struct vm_area_struct *vma;
758 struct rb_node *n = mm->mm_rb.rb_node;
759 unsigned long end = addr + len;
760
761 /* check the cache first */
762 vma = mm->mmap_cache;
763 if (vma && vma->vm_start == addr && vma->vm_end == end)
764 return vma;
765
766 /* trawl the tree (there may be multiple mappings in which addr
767 * resides) */
768 for (n = rb_first(&mm->mm_rb); n; n = rb_next(n)) {
769 vma = rb_entry(n, struct vm_area_struct, vm_rb);
770 if (vma->vm_start < addr)
771 continue;
772 if (vma->vm_start > addr)
773 return NULL;
774 if (vma->vm_end == end) {
775 mm->mmap_cache = vma;
776 return vma;
777 }
778 }
779
780 return NULL;
584} 781}
585 782
586/* 783/*
@@ -595,7 +792,7 @@ static int validate_mmap_request(struct file *file,
595 unsigned long pgoff, 792 unsigned long pgoff,
596 unsigned long *_capabilities) 793 unsigned long *_capabilities)
597{ 794{
598 unsigned long capabilities; 795 unsigned long capabilities, rlen;
599 unsigned long reqprot = prot; 796 unsigned long reqprot = prot;
600 int ret; 797 int ret;
601 798
@@ -615,12 +812,12 @@ static int validate_mmap_request(struct file *file,
615 return -EINVAL; 812 return -EINVAL;
616 813
617 /* Careful about overflows.. */ 814 /* Careful about overflows.. */
618 len = PAGE_ALIGN(len); 815 rlen = PAGE_ALIGN(len);
619 if (!len || len > TASK_SIZE) 816 if (!rlen || rlen > TASK_SIZE)
620 return -ENOMEM; 817 return -ENOMEM;
621 818
622 /* offset overflow? */ 819 /* offset overflow? */
623 if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) 820 if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff)
624 return -EOVERFLOW; 821 return -EOVERFLOW;
625 822
626 if (file) { 823 if (file) {
@@ -794,9 +991,10 @@ static unsigned long determine_vm_flags(struct file *file,
794} 991}
795 992
796/* 993/*
797 * set up a shared mapping on a file 994 * set up a shared mapping on a file (the driver or filesystem provides and
995 * pins the storage)
798 */ 996 */
799static int do_mmap_shared_file(struct vm_area_struct *vma, unsigned long len) 997static int do_mmap_shared_file(struct vm_area_struct *vma)
800{ 998{
801 int ret; 999 int ret;
802 1000
@@ -814,10 +1012,14 @@ static int do_mmap_shared_file(struct vm_area_struct *vma, unsigned long len)
814/* 1012/*
815 * set up a private mapping or an anonymous shared mapping 1013 * set up a private mapping or an anonymous shared mapping
816 */ 1014 */
817static int do_mmap_private(struct vm_area_struct *vma, unsigned long len) 1015static int do_mmap_private(struct vm_area_struct *vma,
1016 struct vm_region *region,
1017 unsigned long len)
818{ 1018{
1019 struct page *pages;
1020 unsigned long total, point, n, rlen;
819 void *base; 1021 void *base;
820 int ret; 1022 int ret, order;
821 1023
822 /* invoke the file's mapping function so that it can keep track of 1024 /* invoke the file's mapping function so that it can keep track of
823 * shared mappings on devices or memory 1025 * shared mappings on devices or memory
@@ -836,23 +1038,46 @@ static int do_mmap_private(struct vm_area_struct *vma, unsigned long len)
836 * make a private copy of the data and map that instead */ 1038 * make a private copy of the data and map that instead */
837 } 1039 }
838 1040
1041 rlen = PAGE_ALIGN(len);
1042
839 /* allocate some memory to hold the mapping 1043 /* allocate some memory to hold the mapping
840 * - note that this may not return a page-aligned address if the object 1044 * - note that this may not return a page-aligned address if the object
841 * we're allocating is smaller than a page 1045 * we're allocating is smaller than a page
842 */ 1046 */
843 base = kmalloc(len, GFP_KERNEL|__GFP_COMP); 1047 order = get_order(rlen);
844 if (!base) 1048 kdebug("alloc order %d for %lx", order, len);
1049
1050 pages = alloc_pages(GFP_KERNEL, order);
1051 if (!pages)
845 goto enomem; 1052 goto enomem;
846 1053
847 vma->vm_start = (unsigned long) base; 1054 /* we allocated a power-of-2 sized page set, so we need to trim off the
848 vma->vm_end = vma->vm_start + len; 1055 * excess */
849 vma->vm_flags |= VM_MAPPED_COPY; 1056 total = 1 << order;
1057 atomic_add(total, &mmap_pages_allocated);
1058
1059 point = rlen >> PAGE_SHIFT;
1060 while (total > point) {
1061 order = ilog2(total - point);
1062 n = 1 << order;
1063 kdebug("shave %lu/%lu @%lu", n, total - point, total);
1064 atomic_sub(n, &mmap_pages_allocated);
1065 total -= n;
1066 set_page_refcounted(pages + total);
1067 __free_pages(pages + total, order);
1068 }
1069
1070 total = rlen >> PAGE_SHIFT;
1071 for (point = 1; point < total; point++)
1072 set_page_refcounted(&pages[point]);
850 1073
851#ifdef WARN_ON_SLACK 1074 base = page_address(pages);
852 if (len + WARN_ON_SLACK <= kobjsize(result)) 1075 region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
853 printk("Allocation of %lu bytes from process %d has %lu bytes of slack\n", 1076 region->vm_start = (unsigned long) base;
854 len, current->pid, kobjsize(result) - len); 1077 region->vm_end = region->vm_start + rlen;
855#endif 1078
1079 vma->vm_start = region->vm_start;
1080 vma->vm_end = region->vm_start + len;
856 1081
857 if (vma->vm_file) { 1082 if (vma->vm_file) {
858 /* read the contents of a file into the copy */ 1083 /* read the contents of a file into the copy */
@@ -864,26 +1089,27 @@ static int do_mmap_private(struct vm_area_struct *vma, unsigned long len)
864 1089
865 old_fs = get_fs(); 1090 old_fs = get_fs();
866 set_fs(KERNEL_DS); 1091 set_fs(KERNEL_DS);
867 ret = vma->vm_file->f_op->read(vma->vm_file, base, len, &fpos); 1092 ret = vma->vm_file->f_op->read(vma->vm_file, base, rlen, &fpos);
868 set_fs(old_fs); 1093 set_fs(old_fs);
869 1094
870 if (ret < 0) 1095 if (ret < 0)
871 goto error_free; 1096 goto error_free;
872 1097
873 /* clear the last little bit */ 1098 /* clear the last little bit */
874 if (ret < len) 1099 if (ret < rlen)
875 memset(base + ret, 0, len - ret); 1100 memset(base + ret, 0, rlen - ret);
876 1101
877 } else { 1102 } else {
878 /* if it's an anonymous mapping, then just clear it */ 1103 /* if it's an anonymous mapping, then just clear it */
879 memset(base, 0, len); 1104 memset(base, 0, rlen);
880 } 1105 }
881 1106
882 return 0; 1107 return 0;
883 1108
884error_free: 1109error_free:
885 kfree(base); 1110 free_page_series(region->vm_start, region->vm_end);
886 vma->vm_start = 0; 1111 region->vm_start = vma->vm_start = 0;
1112 region->vm_end = vma->vm_end = 0;
887 return ret; 1113 return ret;
888 1114
889enomem: 1115enomem:
@@ -903,13 +1129,14 @@ unsigned long do_mmap_pgoff(struct file *file,
903 unsigned long flags, 1129 unsigned long flags,
904 unsigned long pgoff) 1130 unsigned long pgoff)
905{ 1131{
906 struct vm_list_struct *vml = NULL; 1132 struct vm_area_struct *vma;
907 struct vm_area_struct *vma = NULL; 1133 struct vm_region *region;
908 struct rb_node *rb; 1134 struct rb_node *rb;
909 unsigned long capabilities, vm_flags; 1135 unsigned long capabilities, vm_flags, result;
910 void *result;
911 int ret; 1136 int ret;
912 1137
1138 kenter(",%lx,%lx,%lx,%lx,%lx", addr, len, prot, flags, pgoff);
1139
913 if (!(flags & MAP_FIXED)) 1140 if (!(flags & MAP_FIXED))
914 addr = round_hint_to_min(addr); 1141 addr = round_hint_to_min(addr);
915 1142
@@ -917,73 +1144,120 @@ unsigned long do_mmap_pgoff(struct file *file,
917 * mapping */ 1144 * mapping */
918 ret = validate_mmap_request(file, addr, len, prot, flags, pgoff, 1145 ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,
919 &capabilities); 1146 &capabilities);
920 if (ret < 0) 1147 if (ret < 0) {
1148 kleave(" = %d [val]", ret);
921 return ret; 1149 return ret;
1150 }
922 1151
923 /* we've determined that we can make the mapping, now translate what we 1152 /* we've determined that we can make the mapping, now translate what we
924 * now know into VMA flags */ 1153 * now know into VMA flags */
925 vm_flags = determine_vm_flags(file, prot, flags, capabilities); 1154 vm_flags = determine_vm_flags(file, prot, flags, capabilities);
926 1155
927 /* we're going to need to record the mapping if it works */ 1156 /* we're going to need to record the mapping */
928 vml = kzalloc(sizeof(struct vm_list_struct), GFP_KERNEL); 1157 region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
929 if (!vml) 1158 if (!region)
930 goto error_getting_vml; 1159 goto error_getting_region;
1160
1161 vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
1162 if (!vma)
1163 goto error_getting_vma;
1164
1165 atomic_set(&region->vm_usage, 1);
1166 region->vm_flags = vm_flags;
1167 region->vm_pgoff = pgoff;
931 1168
932 down_write(&nommu_vma_sem); 1169 INIT_LIST_HEAD(&vma->anon_vma_node);
1170 vma->vm_flags = vm_flags;
1171 vma->vm_pgoff = pgoff;
933 1172
934 /* if we want to share, we need to check for VMAs created by other 1173 if (file) {
1174 region->vm_file = file;
1175 get_file(file);
1176 vma->vm_file = file;
1177 get_file(file);
1178 if (vm_flags & VM_EXECUTABLE) {
1179 added_exe_file_vma(current->mm);
1180 vma->vm_mm = current->mm;
1181 }
1182 }
1183
1184 down_write(&nommu_region_sem);
1185
1186 /* if we want to share, we need to check for regions created by other
935 * mmap() calls that overlap with our proposed mapping 1187 * mmap() calls that overlap with our proposed mapping
936 * - we can only share with an exact match on most regular files 1188 * - we can only share with a superset match on most regular files
937 * - shared mappings on character devices and memory backed files are 1189 * - shared mappings on character devices and memory backed files are
938 * permitted to overlap inexactly as far as we are concerned for in 1190 * permitted to overlap inexactly as far as we are concerned for in
939 * these cases, sharing is handled in the driver or filesystem rather 1191 * these cases, sharing is handled in the driver or filesystem rather
940 * than here 1192 * than here
941 */ 1193 */
942 if (vm_flags & VM_MAYSHARE) { 1194 if (vm_flags & VM_MAYSHARE) {
943 unsigned long pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; 1195 struct vm_region *pregion;
944 unsigned long vmpglen; 1196 unsigned long pglen, rpglen, pgend, rpgend, start;
945 1197
946 /* suppress VMA sharing for shared regions */ 1198 pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
947 if (vm_flags & VM_SHARED && 1199 pgend = pgoff + pglen;
948 capabilities & BDI_CAP_MAP_DIRECT)
949 goto dont_share_VMAs;
950 1200
951 for (rb = rb_first(&nommu_vma_tree); rb; rb = rb_next(rb)) { 1201 for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) {
952 vma = rb_entry(rb, struct vm_area_struct, vm_rb); 1202 pregion = rb_entry(rb, struct vm_region, vm_rb);
953 1203
954 if (!(vma->vm_flags & VM_MAYSHARE)) 1204 if (!(pregion->vm_flags & VM_MAYSHARE))
955 continue; 1205 continue;
956 1206
957 /* search for overlapping mappings on the same file */ 1207 /* search for overlapping mappings on the same file */
958 if (vma->vm_file->f_path.dentry->d_inode != file->f_path.dentry->d_inode) 1208 if (pregion->vm_file->f_path.dentry->d_inode !=
1209 file->f_path.dentry->d_inode)
959 continue; 1210 continue;
960 1211
961 if (vma->vm_pgoff >= pgoff + pglen) 1212 if (pregion->vm_pgoff >= pgend)
962 continue; 1213 continue;
963 1214
964 vmpglen = vma->vm_end - vma->vm_start + PAGE_SIZE - 1; 1215 rpglen = pregion->vm_end - pregion->vm_start;
965 vmpglen >>= PAGE_SHIFT; 1216 rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT;
966 if (pgoff >= vma->vm_pgoff + vmpglen) 1217 rpgend = pregion->vm_pgoff + rpglen;
1218 if (pgoff >= rpgend)
967 continue; 1219 continue;
968 1220
969 /* handle inexactly overlapping matches between mappings */ 1221 /* handle inexactly overlapping matches between
970 if (vma->vm_pgoff != pgoff || vmpglen != pglen) { 1222 * mappings */
1223 if ((pregion->vm_pgoff != pgoff || rpglen != pglen) &&
1224 !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) {
1225 /* new mapping is not a subset of the region */
971 if (!(capabilities & BDI_CAP_MAP_DIRECT)) 1226 if (!(capabilities & BDI_CAP_MAP_DIRECT))
972 goto sharing_violation; 1227 goto sharing_violation;
973 continue; 1228 continue;
974 } 1229 }
975 1230
976 /* we've found a VMA we can share */ 1231 /* we've found a region we can share */
977 atomic_inc(&vma->vm_usage); 1232 atomic_inc(&pregion->vm_usage);
978 1233 vma->vm_region = pregion;
979 vml->vma = vma; 1234 start = pregion->vm_start;
980 result = (void *) vma->vm_start; 1235 start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT;
981 goto shared; 1236 vma->vm_start = start;
1237 vma->vm_end = start + len;
1238
1239 if (pregion->vm_flags & VM_MAPPED_COPY) {
1240 kdebug("share copy");
1241 vma->vm_flags |= VM_MAPPED_COPY;
1242 } else {
1243 kdebug("share mmap");
1244 ret = do_mmap_shared_file(vma);
1245 if (ret < 0) {
1246 vma->vm_region = NULL;
1247 vma->vm_start = 0;
1248 vma->vm_end = 0;
1249 atomic_dec(&pregion->vm_usage);
1250 pregion = NULL;
1251 goto error_just_free;
1252 }
1253 }
1254 fput(region->vm_file);
1255 kmem_cache_free(vm_region_jar, region);
1256 region = pregion;
1257 result = start;
1258 goto share;
982 } 1259 }
983 1260
984 dont_share_VMAs:
985 vma = NULL;
986
987 /* obtain the address at which to make a shared mapping 1261 /* obtain the address at which to make a shared mapping
988 * - this is the hook for quasi-memory character devices to 1262 * - this is the hook for quasi-memory character devices to
989 * tell us the location of a shared mapping 1263 * tell us the location of a shared mapping
@@ -994,102 +1268,93 @@ unsigned long do_mmap_pgoff(struct file *file,
994 if (IS_ERR((void *) addr)) { 1268 if (IS_ERR((void *) addr)) {
995 ret = addr; 1269 ret = addr;
996 if (ret != (unsigned long) -ENOSYS) 1270 if (ret != (unsigned long) -ENOSYS)
997 goto error; 1271 goto error_just_free;
998 1272
999 /* the driver refused to tell us where to site 1273 /* the driver refused to tell us where to site
1000 * the mapping so we'll have to attempt to copy 1274 * the mapping so we'll have to attempt to copy
1001 * it */ 1275 * it */
1002 ret = (unsigned long) -ENODEV; 1276 ret = (unsigned long) -ENODEV;
1003 if (!(capabilities & BDI_CAP_MAP_COPY)) 1277 if (!(capabilities & BDI_CAP_MAP_COPY))
1004 goto error; 1278 goto error_just_free;
1005 1279
1006 capabilities &= ~BDI_CAP_MAP_DIRECT; 1280 capabilities &= ~BDI_CAP_MAP_DIRECT;
1281 } else {
1282 vma->vm_start = region->vm_start = addr;
1283 vma->vm_end = region->vm_end = addr + len;
1007 } 1284 }
1008 } 1285 }
1009 } 1286 }
1010 1287
1011 /* we're going to need a VMA struct as well */ 1288 vma->vm_region = region;
1012 vma = kzalloc(sizeof(struct vm_area_struct), GFP_KERNEL);
1013 if (!vma)
1014 goto error_getting_vma;
1015
1016 INIT_LIST_HEAD(&vma->anon_vma_node);
1017 atomic_set(&vma->vm_usage, 1);
1018 if (file) {
1019 get_file(file);
1020 if (vm_flags & VM_EXECUTABLE) {
1021 added_exe_file_vma(current->mm);
1022 vma->vm_mm = current->mm;
1023 }
1024 }
1025 vma->vm_file = file;
1026 vma->vm_flags = vm_flags;
1027 vma->vm_start = addr;
1028 vma->vm_end = addr + len;
1029 vma->vm_pgoff = pgoff;
1030
1031 vml->vma = vma;
1032 1289
1033 /* set up the mapping */ 1290 /* set up the mapping */
1034 if (file && vma->vm_flags & VM_SHARED) 1291 if (file && vma->vm_flags & VM_SHARED)
1035 ret = do_mmap_shared_file(vma, len); 1292 ret = do_mmap_shared_file(vma);
1036 else 1293 else
1037 ret = do_mmap_private(vma, len); 1294 ret = do_mmap_private(vma, region, len);
1038 if (ret < 0) 1295 if (ret < 0)
1039 goto error; 1296 goto error_put_region;
1297
1298 add_nommu_region(region);
1040 1299
1041 /* okay... we have a mapping; now we have to register it */ 1300 /* okay... we have a mapping; now we have to register it */
1042 result = (void *) vma->vm_start; 1301 result = vma->vm_start;
1043 1302
1044 current->mm->total_vm += len >> PAGE_SHIFT; 1303 current->mm->total_vm += len >> PAGE_SHIFT;
1045 1304
1046 add_nommu_vma(vma); 1305share:
1306 add_vma_to_mm(current->mm, vma);
1047 1307
1048 shared: 1308 up_write(&nommu_region_sem);
1049 add_vma_to_mm(current->mm, vml);
1050
1051 up_write(&nommu_vma_sem);
1052 1309
1053 if (prot & PROT_EXEC) 1310 if (prot & PROT_EXEC)
1054 flush_icache_range((unsigned long) result, 1311 flush_icache_range(result, result + len);
1055 (unsigned long) result + len);
1056 1312
1057#ifdef DEBUG 1313 kleave(" = %lx", result);
1058 printk("do_mmap:\n"); 1314 return result;
1059 show_process_blocks();
1060#endif
1061 1315
1062 return (unsigned long) result; 1316error_put_region:
1063 1317 __put_nommu_region(region);
1064 error:
1065 up_write(&nommu_vma_sem);
1066 kfree(vml);
1067 if (vma) { 1318 if (vma) {
1068 if (vma->vm_file) { 1319 if (vma->vm_file) {
1069 fput(vma->vm_file); 1320 fput(vma->vm_file);
1070 if (vma->vm_flags & VM_EXECUTABLE) 1321 if (vma->vm_flags & VM_EXECUTABLE)
1071 removed_exe_file_vma(vma->vm_mm); 1322 removed_exe_file_vma(vma->vm_mm);
1072 } 1323 }
1073 kfree(vma); 1324 kmem_cache_free(vm_area_cachep, vma);
1074 } 1325 }
1326 kleave(" = %d [pr]", ret);
1075 return ret; 1327 return ret;
1076 1328
1077 sharing_violation: 1329error_just_free:
1078 up_write(&nommu_vma_sem); 1330 up_write(&nommu_region_sem);
1079 printk("Attempt to share mismatched mappings\n"); 1331error:
1080 kfree(vml); 1332 fput(region->vm_file);
1081 return -EINVAL; 1333 kmem_cache_free(vm_region_jar, region);
1334 fput(vma->vm_file);
1335 if (vma->vm_flags & VM_EXECUTABLE)
1336 removed_exe_file_vma(vma->vm_mm);
1337 kmem_cache_free(vm_area_cachep, vma);
1338 kleave(" = %d", ret);
1339 return ret;
1340
1341sharing_violation:
1342 up_write(&nommu_region_sem);
1343 printk(KERN_WARNING "Attempt to share mismatched mappings\n");
1344 ret = -EINVAL;
1345 goto error;
1082 1346
1083 error_getting_vma: 1347error_getting_vma:
1084 up_write(&nommu_vma_sem); 1348 kmem_cache_free(vm_region_jar, region);
1085 kfree(vml); 1349 printk(KERN_WARNING "Allocation of vma for %lu byte allocation"
1086 printk("Allocation of vma for %lu byte allocation from process %d failed\n", 1350 " from process %d failed\n",
1087 len, current->pid); 1351 len, current->pid);
1088 show_free_areas(); 1352 show_free_areas();
1089 return -ENOMEM; 1353 return -ENOMEM;
1090 1354
1091 error_getting_vml: 1355error_getting_region:
1092 printk("Allocation of vml for %lu byte allocation from process %d failed\n", 1356 printk(KERN_WARNING "Allocation of vm region for %lu byte allocation"
1357 " from process %d failed\n",
1093 len, current->pid); 1358 len, current->pid);
1094 show_free_areas(); 1359 show_free_areas();
1095 return -ENOMEM; 1360 return -ENOMEM;
@@ -1097,77 +1362,180 @@ unsigned long do_mmap_pgoff(struct file *file,
1097EXPORT_SYMBOL(do_mmap_pgoff); 1362EXPORT_SYMBOL(do_mmap_pgoff);
1098 1363
1099/* 1364/*
1100 * handle mapping disposal for uClinux 1365 * split a vma into two pieces at address 'addr', a new vma is allocated either
1366 * for the first part or the tail.
1101 */ 1367 */
1102static void put_vma(struct mm_struct *mm, struct vm_area_struct *vma) 1368int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
1369 unsigned long addr, int new_below)
1103{ 1370{
1104 if (vma) { 1371 struct vm_area_struct *new;
1105 down_write(&nommu_vma_sem); 1372 struct vm_region *region;
1373 unsigned long npages;
1106 1374
1107 if (atomic_dec_and_test(&vma->vm_usage)) { 1375 kenter("");
1108 delete_nommu_vma(vma);
1109 1376
1110 if (vma->vm_ops && vma->vm_ops->close) 1377 /* we're only permitted to split anonymous regions that have a single
1111 vma->vm_ops->close(vma); 1378 * owner */
1379 if (vma->vm_file ||
1380 atomic_read(&vma->vm_region->vm_usage) != 1)
1381 return -ENOMEM;
1112 1382
1113 /* IO memory and memory shared directly out of the pagecache from 1383 if (mm->map_count >= sysctl_max_map_count)
1114 * ramfs/tmpfs mustn't be released here */ 1384 return -ENOMEM;
1115 if (vma->vm_flags & VM_MAPPED_COPY)
1116 kfree((void *) vma->vm_start);
1117 1385
1118 if (vma->vm_file) { 1386 region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL);
1119 fput(vma->vm_file); 1387 if (!region)
1120 if (vma->vm_flags & VM_EXECUTABLE) 1388 return -ENOMEM;
1121 removed_exe_file_vma(mm); 1389
1122 } 1390 new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
1123 kfree(vma); 1391 if (!new) {
1124 } 1392 kmem_cache_free(vm_region_jar, region);
1393 return -ENOMEM;
1394 }
1395
1396 /* most fields are the same, copy all, and then fixup */
1397 *new = *vma;
1398 *region = *vma->vm_region;
1399 new->vm_region = region;
1400
1401 npages = (addr - vma->vm_start) >> PAGE_SHIFT;
1402
1403 if (new_below) {
1404 region->vm_end = new->vm_end = addr;
1405 } else {
1406 region->vm_start = new->vm_start = addr;
1407 region->vm_pgoff = new->vm_pgoff += npages;
1408 }
1125 1409
1126 up_write(&nommu_vma_sem); 1410 if (new->vm_ops && new->vm_ops->open)
1411 new->vm_ops->open(new);
1412
1413 delete_vma_from_mm(vma);
1414 down_write(&nommu_region_sem);
1415 delete_nommu_region(vma->vm_region);
1416 if (new_below) {
1417 vma->vm_region->vm_start = vma->vm_start = addr;
1418 vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;
1419 } else {
1420 vma->vm_region->vm_end = vma->vm_end = addr;
1127 } 1421 }
1422 add_nommu_region(vma->vm_region);
1423 add_nommu_region(new->vm_region);
1424 up_write(&nommu_region_sem);
1425 add_vma_to_mm(mm, vma);
1426 add_vma_to_mm(mm, new);
1427 return 0;
1128} 1428}
1129 1429
1130/* 1430/*
1131 * release a mapping 1431 * shrink a VMA by removing the specified chunk from either the beginning or
1132 * - under NOMMU conditions the parameters must match exactly to the mapping to 1432 * the end
1133 * be removed
1134 */ 1433 */
1135int do_munmap(struct mm_struct *mm, unsigned long addr, size_t len) 1434static int shrink_vma(struct mm_struct *mm,
1435 struct vm_area_struct *vma,
1436 unsigned long from, unsigned long to)
1136{ 1437{
1137 struct vm_list_struct *vml, **parent; 1438 struct vm_region *region;
1138 unsigned long end = addr + len;
1139 1439
1140#ifdef DEBUG 1440 kenter("");
1141 printk("do_munmap:\n");
1142#endif
1143 1441
1144 for (parent = &mm->context.vmlist; *parent; parent = &(*parent)->next) { 1442 /* adjust the VMA's pointers, which may reposition it in the MM's tree
1145 if ((*parent)->vma->vm_start > addr) 1443 * and list */
1146 break; 1444 delete_vma_from_mm(vma);
1147 if ((*parent)->vma->vm_start == addr && 1445 if (from > vma->vm_start)
1148 ((len == 0) || ((*parent)->vma->vm_end == end))) 1446 vma->vm_end = from;
1149 goto found; 1447 else
1150 } 1448 vma->vm_start = to;
1449 add_vma_to_mm(mm, vma);
1151 1450
1152 printk("munmap of non-mmaped memory by process %d (%s): %p\n", 1451 /* cut the backing region down to size */
1153 current->pid, current->comm, (void *) addr); 1452 region = vma->vm_region;
1154 return -EINVAL; 1453 BUG_ON(atomic_read(&region->vm_usage) != 1);
1155 1454
1156 found: 1455 down_write(&nommu_region_sem);
1157 vml = *parent; 1456 delete_nommu_region(region);
1457 if (from > region->vm_start)
1458 region->vm_end = from;
1459 else
1460 region->vm_start = to;
1461 add_nommu_region(region);
1462 up_write(&nommu_region_sem);
1158 1463
1159 put_vma(mm, vml->vma); 1464 free_page_series(from, to);
1465 return 0;
1466}
1160 1467
1161 *parent = vml->next; 1468/*
1162 kfree(vml); 1469 * release a mapping
1470 * - under NOMMU conditions the chunk to be unmapped must be backed by a single
1471 * VMA, though it need not cover the whole VMA
1472 */
1473int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
1474{
1475 struct vm_area_struct *vma;
1476 struct rb_node *rb;
1477 unsigned long end = start + len;
1478 int ret;
1163 1479
1164 update_hiwater_vm(mm); 1480 kenter(",%lx,%zx", start, len);
1165 mm->total_vm -= len >> PAGE_SHIFT;
1166 1481
1167#ifdef DEBUG 1482 if (len == 0)
1168 show_process_blocks(); 1483 return -EINVAL;
1169#endif 1484
1485 /* find the first potentially overlapping VMA */
1486 vma = find_vma(mm, start);
1487 if (!vma) {
1488 printk(KERN_WARNING
1489 "munmap of memory not mmapped by process %d (%s):"
1490 " 0x%lx-0x%lx\n",
1491 current->pid, current->comm, start, start + len - 1);
1492 return -EINVAL;
1493 }
1170 1494
1495 /* we're allowed to split an anonymous VMA but not a file-backed one */
1496 if (vma->vm_file) {
1497 do {
1498 if (start > vma->vm_start) {
1499 kleave(" = -EINVAL [miss]");
1500 return -EINVAL;
1501 }
1502 if (end == vma->vm_end)
1503 goto erase_whole_vma;
1504 rb = rb_next(&vma->vm_rb);
1505 vma = rb_entry(rb, struct vm_area_struct, vm_rb);
1506 } while (rb);
1507 kleave(" = -EINVAL [split file]");
1508 return -EINVAL;
1509 } else {
1510 /* the chunk must be a subset of the VMA found */
1511 if (start == vma->vm_start && end == vma->vm_end)
1512 goto erase_whole_vma;
1513 if (start < vma->vm_start || end > vma->vm_end) {
1514 kleave(" = -EINVAL [superset]");
1515 return -EINVAL;
1516 }
1517 if (start & ~PAGE_MASK) {
1518 kleave(" = -EINVAL [unaligned start]");
1519 return -EINVAL;
1520 }
1521 if (end != vma->vm_end && end & ~PAGE_MASK) {
1522 kleave(" = -EINVAL [unaligned split]");
1523 return -EINVAL;
1524 }
1525 if (start != vma->vm_start && end != vma->vm_end) {
1526 ret = split_vma(mm, vma, start, 1);
1527 if (ret < 0) {
1528 kleave(" = %d [split]", ret);
1529 return ret;
1530 }
1531 }
1532 return shrink_vma(mm, vma, start, end);
1533 }
1534
1535erase_whole_vma:
1536 delete_vma_from_mm(vma);
1537 delete_vma(mm, vma);
1538 kleave(" = 0");
1171 return 0; 1539 return 0;
1172} 1540}
1173EXPORT_SYMBOL(do_munmap); 1541EXPORT_SYMBOL(do_munmap);
@@ -1184,29 +1552,26 @@ asmlinkage long sys_munmap(unsigned long addr, size_t len)
1184} 1552}
1185 1553
1186/* 1554/*
1187 * Release all mappings 1555 * release all the mappings made in a process's VM space
1188 */ 1556 */
1189void exit_mmap(struct mm_struct * mm) 1557void exit_mmap(struct mm_struct *mm)
1190{ 1558{
1191 struct vm_list_struct *tmp; 1559 struct vm_area_struct *vma;
1192 1560
1193 if (mm) { 1561 if (!mm)
1194#ifdef DEBUG 1562 return;
1195 printk("Exit_mmap:\n");
1196#endif
1197 1563
1198 mm->total_vm = 0; 1564 kenter("");
1199 1565
1200 while ((tmp = mm->context.vmlist)) { 1566 mm->total_vm = 0;
1201 mm->context.vmlist = tmp->next;
1202 put_vma(mm, tmp->vma);
1203 kfree(tmp);
1204 }
1205 1567
1206#ifdef DEBUG 1568 while ((vma = mm->mmap)) {
1207 show_process_blocks(); 1569 mm->mmap = vma->vm_next;
1208#endif 1570 delete_vma_from_mm(vma);
1571 delete_vma(mm, vma);
1209 } 1572 }
1573
1574 kleave("");
1210} 1575}
1211 1576
1212unsigned long do_brk(unsigned long addr, unsigned long len) 1577unsigned long do_brk(unsigned long addr, unsigned long len)
@@ -1219,8 +1584,8 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
1219 * time (controlled by the MREMAP_MAYMOVE flag and available VM space) 1584 * time (controlled by the MREMAP_MAYMOVE flag and available VM space)
1220 * 1585 *
1221 * under NOMMU conditions, we only permit changing a mapping's size, and only 1586 * under NOMMU conditions, we only permit changing a mapping's size, and only
1222 * as long as it stays within the hole allocated by the kmalloc() call in 1587 * as long as it stays within the region allocated by do_mmap_private() and the
1223 * do_mmap_pgoff() and the block is not shareable 1588 * block is not shareable
1224 * 1589 *
1225 * MREMAP_FIXED is not supported under NOMMU conditions 1590 * MREMAP_FIXED is not supported under NOMMU conditions
1226 */ 1591 */
@@ -1231,13 +1596,16 @@ unsigned long do_mremap(unsigned long addr,
1231 struct vm_area_struct *vma; 1596 struct vm_area_struct *vma;
1232 1597
1233 /* insanity checks first */ 1598 /* insanity checks first */
1234 if (new_len == 0) 1599 if (old_len == 0 || new_len == 0)
1235 return (unsigned long) -EINVAL; 1600 return (unsigned long) -EINVAL;
1236 1601
1602 if (addr & ~PAGE_MASK)
1603 return -EINVAL;
1604
1237 if (flags & MREMAP_FIXED && new_addr != addr) 1605 if (flags & MREMAP_FIXED && new_addr != addr)
1238 return (unsigned long) -EINVAL; 1606 return (unsigned long) -EINVAL;
1239 1607
1240 vma = find_vma_exact(current->mm, addr); 1608 vma = find_vma_exact(current->mm, addr, old_len);
1241 if (!vma) 1609 if (!vma)
1242 return (unsigned long) -EINVAL; 1610 return (unsigned long) -EINVAL;
1243 1611
@@ -1247,19 +1615,19 @@ unsigned long do_mremap(unsigned long addr,
1247 if (vma->vm_flags & VM_MAYSHARE) 1615 if (vma->vm_flags & VM_MAYSHARE)
1248 return (unsigned long) -EPERM; 1616 return (unsigned long) -EPERM;
1249 1617
1250 if (new_len > kobjsize((void *) addr)) 1618 if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start)
1251 return (unsigned long) -ENOMEM; 1619 return (unsigned long) -ENOMEM;
1252 1620
1253 /* all checks complete - do it */ 1621 /* all checks complete - do it */
1254 vma->vm_end = vma->vm_start + new_len; 1622 vma->vm_end = vma->vm_start + new_len;
1255
1256 return vma->vm_start; 1623 return vma->vm_start;
1257} 1624}
1258EXPORT_SYMBOL(do_mremap); 1625EXPORT_SYMBOL(do_mremap);
1259 1626
1260asmlinkage unsigned long sys_mremap(unsigned long addr, 1627asmlinkage
1261 unsigned long old_len, unsigned long new_len, 1628unsigned long sys_mremap(unsigned long addr,
1262 unsigned long flags, unsigned long new_addr) 1629 unsigned long old_len, unsigned long new_len,
1630 unsigned long flags, unsigned long new_addr)
1263{ 1631{
1264 unsigned long ret; 1632 unsigned long ret;
1265 1633
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-18 14:27:26 -0400