aboutsummaryrefslogtreecommitdiffstats
path: root/mm/memory.c
diff options
context:
space:
mode:
Diffstat (limited to 'mm/memory.c')
-rw-r--r--mm/memory.c2165
1 files changed, 2165 insertions, 0 deletions
diff --git a/mm/memory.c b/mm/memory.c
new file mode 100644
index 000000000000..fb6e5deb873a
--- /dev/null
+++ b/mm/memory.c
@@ -0,0 +1,2165 @@
1/*
2 * linux/mm/memory.c
3 *
4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
5 */
6
7/*
8 * demand-loading started 01.12.91 - seems it is high on the list of
9 * things wanted, and it should be easy to implement. - Linus
10 */
11
12/*
13 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
14 * pages started 02.12.91, seems to work. - Linus.
15 *
16 * Tested sharing by executing about 30 /bin/sh: under the old kernel it
17 * would have taken more than the 6M I have free, but it worked well as
18 * far as I could see.
19 *
20 * Also corrected some "invalidate()"s - I wasn't doing enough of them.
21 */
22
23/*
24 * Real VM (paging to/from disk) started 18.12.91. Much more work and
25 * thought has to go into this. Oh, well..
26 * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why.
27 * Found it. Everything seems to work now.
28 * 20.12.91 - Ok, making the swap-device changeable like the root.
29 */
30
31/*
32 * 05.04.94 - Multi-page memory management added for v1.1.
33 * Idea by Alex Bligh (alex@cconcepts.co.uk)
34 *
35 * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG
36 * (Gerhard.Wichert@pdb.siemens.de)
37 *
38 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
39 */
40
41#include <linux/kernel_stat.h>
42#include <linux/mm.h>
43#include <linux/hugetlb.h>
44#include <linux/mman.h>
45#include <linux/swap.h>
46#include <linux/highmem.h>
47#include <linux/pagemap.h>
48#include <linux/rmap.h>
49#include <linux/module.h>
50#include <linux/init.h>
51
52#include <asm/pgalloc.h>
53#include <asm/uaccess.h>
54#include <asm/tlb.h>
55#include <asm/tlbflush.h>
56#include <asm/pgtable.h>
57
58#include <linux/swapops.h>
59#include <linux/elf.h>
60
61#ifndef CONFIG_DISCONTIGMEM
62/* use the per-pgdat data instead for discontigmem - mbligh */
63unsigned long max_mapnr;
64struct page *mem_map;
65
66EXPORT_SYMBOL(max_mapnr);
67EXPORT_SYMBOL(mem_map);
68#endif
69
70unsigned long num_physpages;
71/*
72 * A number of key systems in x86 including ioremap() rely on the assumption
73 * that high_memory defines the upper bound on direct map memory, then end
74 * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and
75 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
76 * and ZONE_HIGHMEM.
77 */
78void * high_memory;
79unsigned long vmalloc_earlyreserve;
80
81EXPORT_SYMBOL(num_physpages);
82EXPORT_SYMBOL(high_memory);
83EXPORT_SYMBOL(vmalloc_earlyreserve);
84
85/*
86 * If a p?d_bad entry is found while walking page tables, report
87 * the error, before resetting entry to p?d_none. Usually (but
88 * very seldom) called out from the p?d_none_or_clear_bad macros.
89 */
90
91void pgd_clear_bad(pgd_t *pgd)
92{
93 pgd_ERROR(*pgd);
94 pgd_clear(pgd);
95}
96
97void pud_clear_bad(pud_t *pud)
98{
99 pud_ERROR(*pud);
100 pud_clear(pud);
101}
102
103void pmd_clear_bad(pmd_t *pmd)
104{
105 pmd_ERROR(*pmd);
106 pmd_clear(pmd);
107}
108
109/*
110 * Note: this doesn't free the actual pages themselves. That
111 * has been handled earlier when unmapping all the memory regions.
112 */
113static inline void clear_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
114 unsigned long addr, unsigned long end)
115{
116 if (!((addr | end) & ~PMD_MASK)) {
117 /* Only free fully aligned ranges */
118 struct page *page = pmd_page(*pmd);
119 pmd_clear(pmd);
120 dec_page_state(nr_page_table_pages);
121 tlb->mm->nr_ptes--;
122 pte_free_tlb(tlb, page);
123 }
124}
125
126static inline void clear_pmd_range(struct mmu_gather *tlb, pud_t *pud,
127 unsigned long addr, unsigned long end)
128{
129 pmd_t *pmd;
130 unsigned long next;
131 pmd_t *empty_pmd = NULL;
132
133 pmd = pmd_offset(pud, addr);
134
135 /* Only free fully aligned ranges */
136 if (!((addr | end) & ~PUD_MASK))
137 empty_pmd = pmd;
138 do {
139 next = pmd_addr_end(addr, end);
140 if (pmd_none_or_clear_bad(pmd))
141 continue;
142 clear_pte_range(tlb, pmd, addr, next);
143 } while (pmd++, addr = next, addr != end);
144
145 if (empty_pmd) {
146 pud_clear(pud);
147 pmd_free_tlb(tlb, empty_pmd);
148 }
149}
150
151static inline void clear_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
152 unsigned long addr, unsigned long end)
153{
154 pud_t *pud;
155 unsigned long next;
156 pud_t *empty_pud = NULL;
157
158 pud = pud_offset(pgd, addr);
159
160 /* Only free fully aligned ranges */
161 if (!((addr | end) & ~PGDIR_MASK))
162 empty_pud = pud;
163 do {
164 next = pud_addr_end(addr, end);
165 if (pud_none_or_clear_bad(pud))
166 continue;
167 clear_pmd_range(tlb, pud, addr, next);
168 } while (pud++, addr = next, addr != end);
169
170 if (empty_pud) {
171 pgd_clear(pgd);
172 pud_free_tlb(tlb, empty_pud);
173 }
174}
175
176/*
177 * This function clears user-level page tables of a process.
178 * Unlike other pagetable walks, some memory layouts might give end 0.
179 * Must be called with pagetable lock held.
180 */
181void clear_page_range(struct mmu_gather *tlb,
182 unsigned long addr, unsigned long end)
183{
184 pgd_t *pgd;
185 unsigned long next;
186
187 pgd = pgd_offset(tlb->mm, addr);
188 do {
189 next = pgd_addr_end(addr, end);
190 if (pgd_none_or_clear_bad(pgd))
191 continue;
192 clear_pud_range(tlb, pgd, addr, next);
193 } while (pgd++, addr = next, addr != end);
194}
195
196pte_t fastcall * pte_alloc_map(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
197{
198 if (!pmd_present(*pmd)) {
199 struct page *new;
200
201 spin_unlock(&mm->page_table_lock);
202 new = pte_alloc_one(mm, address);
203 spin_lock(&mm->page_table_lock);
204 if (!new)
205 return NULL;
206 /*
207 * Because we dropped the lock, we should re-check the
208 * entry, as somebody else could have populated it..
209 */
210 if (pmd_present(*pmd)) {
211 pte_free(new);
212 goto out;
213 }
214 mm->nr_ptes++;
215 inc_page_state(nr_page_table_pages);
216 pmd_populate(mm, pmd, new);
217 }
218out:
219 return pte_offset_map(pmd, address);
220}
221
222pte_t fastcall * pte_alloc_kernel(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
223{
224 if (!pmd_present(*pmd)) {
225 pte_t *new;
226
227 spin_unlock(&mm->page_table_lock);
228 new = pte_alloc_one_kernel(mm, address);
229 spin_lock(&mm->page_table_lock);
230 if (!new)
231 return NULL;
232
233 /*
234 * Because we dropped the lock, we should re-check the
235 * entry, as somebody else could have populated it..
236 */
237 if (pmd_present(*pmd)) {
238 pte_free_kernel(new);
239 goto out;
240 }
241 pmd_populate_kernel(mm, pmd, new);
242 }
243out:
244 return pte_offset_kernel(pmd, address);
245}
246
247/*
248 * copy one vm_area from one task to the other. Assumes the page tables
249 * already present in the new task to be cleared in the whole range
250 * covered by this vma.
251 *
252 * dst->page_table_lock is held on entry and exit,
253 * but may be dropped within p[mg]d_alloc() and pte_alloc_map().
254 */
255
256static inline void
257copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
258 pte_t *dst_pte, pte_t *src_pte, unsigned long vm_flags,
259 unsigned long addr)
260{
261 pte_t pte = *src_pte;
262 struct page *page;
263 unsigned long pfn;
264
265 /* pte contains position in swap or file, so copy. */
266 if (unlikely(!pte_present(pte))) {
267 if (!pte_file(pte)) {
268 swap_duplicate(pte_to_swp_entry(pte));
269 /* make sure dst_mm is on swapoff's mmlist. */
270 if (unlikely(list_empty(&dst_mm->mmlist))) {
271 spin_lock(&mmlist_lock);
272 list_add(&dst_mm->mmlist, &src_mm->mmlist);
273 spin_unlock(&mmlist_lock);
274 }
275 }
276 set_pte_at(dst_mm, addr, dst_pte, pte);
277 return;
278 }
279
280 pfn = pte_pfn(pte);
281 /* the pte points outside of valid memory, the
282 * mapping is assumed to be good, meaningful
283 * and not mapped via rmap - duplicate the
284 * mapping as is.
285 */
286 page = NULL;
287 if (pfn_valid(pfn))
288 page = pfn_to_page(pfn);
289
290 if (!page || PageReserved(page)) {
291 set_pte_at(dst_mm, addr, dst_pte, pte);
292 return;
293 }
294
295 /*
296 * If it's a COW mapping, write protect it both
297 * in the parent and the child
298 */
299 if ((vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE) {
300 ptep_set_wrprotect(src_mm, addr, src_pte);
301 pte = *src_pte;
302 }
303
304 /*
305 * If it's a shared mapping, mark it clean in
306 * the child
307 */
308 if (vm_flags & VM_SHARED)
309 pte = pte_mkclean(pte);
310 pte = pte_mkold(pte);
311 get_page(page);
312 inc_mm_counter(dst_mm, rss);
313 if (PageAnon(page))
314 inc_mm_counter(dst_mm, anon_rss);
315 set_pte_at(dst_mm, addr, dst_pte, pte);
316 page_dup_rmap(page);
317}
318
319static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
320 pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,
321 unsigned long addr, unsigned long end)
322{
323 pte_t *src_pte, *dst_pte;
324 unsigned long vm_flags = vma->vm_flags;
325 int progress;
326
327again:
328 dst_pte = pte_alloc_map(dst_mm, dst_pmd, addr);
329 if (!dst_pte)
330 return -ENOMEM;
331 src_pte = pte_offset_map_nested(src_pmd, addr);
332
333 progress = 0;
334 spin_lock(&src_mm->page_table_lock);
335 do {
336 /*
337 * We are holding two locks at this point - either of them
338 * could generate latencies in another task on another CPU.
339 */
340 if (progress >= 32 && (need_resched() ||
341 need_lockbreak(&src_mm->page_table_lock) ||
342 need_lockbreak(&dst_mm->page_table_lock)))
343 break;
344 if (pte_none(*src_pte)) {
345 progress++;
346 continue;
347 }
348 copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, vm_flags, addr);
349 progress += 8;
350 } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
351 spin_unlock(&src_mm->page_table_lock);
352
353 pte_unmap_nested(src_pte - 1);
354 pte_unmap(dst_pte - 1);
355 cond_resched_lock(&dst_mm->page_table_lock);
356 if (addr != end)
357 goto again;
358 return 0;
359}
360
361static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
362 pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma,
363 unsigned long addr, unsigned long end)
364{
365 pmd_t *src_pmd, *dst_pmd;
366 unsigned long next;
367
368 dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
369 if (!dst_pmd)
370 return -ENOMEM;
371 src_pmd = pmd_offset(src_pud, addr);
372 do {
373 next = pmd_addr_end(addr, end);
374 if (pmd_none_or_clear_bad(src_pmd))
375 continue;
376 if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd,
377 vma, addr, next))
378 return -ENOMEM;
379 } while (dst_pmd++, src_pmd++, addr = next, addr != end);
380 return 0;
381}
382
383static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
384 pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
385 unsigned long addr, unsigned long end)
386{
387 pud_t *src_pud, *dst_pud;
388 unsigned long next;
389
390 dst_pud = pud_alloc(dst_mm, dst_pgd, addr);
391 if (!dst_pud)
392 return -ENOMEM;
393 src_pud = pud_offset(src_pgd, addr);
394 do {
395 next = pud_addr_end(addr, end);
396 if (pud_none_or_clear_bad(src_pud))
397 continue;
398 if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud,
399 vma, addr, next))
400 return -ENOMEM;
401 } while (dst_pud++, src_pud++, addr = next, addr != end);
402 return 0;
403}
404
405int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
406 struct vm_area_struct *vma)
407{
408 pgd_t *src_pgd, *dst_pgd;
409 unsigned long next;
410 unsigned long addr = vma->vm_start;
411 unsigned long end = vma->vm_end;
412
413 if (is_vm_hugetlb_page(vma))
414 return copy_hugetlb_page_range(dst_mm, src_mm, vma);
415
416 dst_pgd = pgd_offset(dst_mm, addr);
417 src_pgd = pgd_offset(src_mm, addr);
418 do {
419 next = pgd_addr_end(addr, end);
420 if (pgd_none_or_clear_bad(src_pgd))
421 continue;
422 if (copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd,
423 vma, addr, next))
424 return -ENOMEM;
425 } while (dst_pgd++, src_pgd++, addr = next, addr != end);
426 return 0;
427}
428
429static void zap_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
430 unsigned long addr, unsigned long end,
431 struct zap_details *details)
432{
433 pte_t *pte;
434
435 pte = pte_offset_map(pmd, addr);
436 do {
437 pte_t ptent = *pte;
438 if (pte_none(ptent))
439 continue;
440 if (pte_present(ptent)) {
441 struct page *page = NULL;
442 unsigned long pfn = pte_pfn(ptent);
443 if (pfn_valid(pfn)) {
444 page = pfn_to_page(pfn);
445 if (PageReserved(page))
446 page = NULL;
447 }
448 if (unlikely(details) && page) {
449 /*
450 * unmap_shared_mapping_pages() wants to
451 * invalidate cache without truncating:
452 * unmap shared but keep private pages.
453 */
454 if (details->check_mapping &&
455 details->check_mapping != page->mapping)
456 continue;
457 /*
458 * Each page->index must be checked when
459 * invalidating or truncating nonlinear.
460 */
461 if (details->nonlinear_vma &&
462 (page->index < details->first_index ||
463 page->index > details->last_index))
464 continue;
465 }
466 ptent = ptep_get_and_clear(tlb->mm, addr, pte);
467 tlb_remove_tlb_entry(tlb, pte, addr);
468 if (unlikely(!page))
469 continue;
470 if (unlikely(details) && details->nonlinear_vma
471 && linear_page_index(details->nonlinear_vma,
472 addr) != page->index)
473 set_pte_at(tlb->mm, addr, pte,
474 pgoff_to_pte(page->index));
475 if (pte_dirty(ptent))
476 set_page_dirty(page);
477 if (PageAnon(page))
478 dec_mm_counter(tlb->mm, anon_rss);
479 else if (pte_young(ptent))
480 mark_page_accessed(page);
481 tlb->freed++;
482 page_remove_rmap(page);
483 tlb_remove_page(tlb, page);
484 continue;
485 }
486 /*
487 * If details->check_mapping, we leave swap entries;
488 * if details->nonlinear_vma, we leave file entries.
489 */
490 if (unlikely(details))
491 continue;
492 if (!pte_file(ptent))
493 free_swap_and_cache(pte_to_swp_entry(ptent));
494 pte_clear(tlb->mm, addr, pte);
495 } while (pte++, addr += PAGE_SIZE, addr != end);
496 pte_unmap(pte - 1);
497}
498
499static inline void zap_pmd_range(struct mmu_gather *tlb, pud_t *pud,
500 unsigned long addr, unsigned long end,
501 struct zap_details *details)
502{
503 pmd_t *pmd;
504 unsigned long next;
505
506 pmd = pmd_offset(pud, addr);
507 do {
508 next = pmd_addr_end(addr, end);
509 if (pmd_none_or_clear_bad(pmd))
510 continue;
511 zap_pte_range(tlb, pmd, addr, next, details);
512 } while (pmd++, addr = next, addr != end);
513}
514
515static inline void zap_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
516 unsigned long addr, unsigned long end,
517 struct zap_details *details)
518{
519 pud_t *pud;
520 unsigned long next;
521
522 pud = pud_offset(pgd, addr);
523 do {
524 next = pud_addr_end(addr, end);
525 if (pud_none_or_clear_bad(pud))
526 continue;
527 zap_pmd_range(tlb, pud, addr, next, details);
528 } while (pud++, addr = next, addr != end);
529}
530
531static void unmap_page_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
532 unsigned long addr, unsigned long end,
533 struct zap_details *details)
534{
535 pgd_t *pgd;
536 unsigned long next;
537
538 if (details && !details->check_mapping && !details->nonlinear_vma)
539 details = NULL;
540
541 BUG_ON(addr >= end);
542 tlb_start_vma(tlb, vma);
543 pgd = pgd_offset(vma->vm_mm, addr);
544 do {
545 next = pgd_addr_end(addr, end);
546 if (pgd_none_or_clear_bad(pgd))
547 continue;
548 zap_pud_range(tlb, pgd, addr, next, details);
549 } while (pgd++, addr = next, addr != end);
550 tlb_end_vma(tlb, vma);
551}
552
553#ifdef CONFIG_PREEMPT
554# define ZAP_BLOCK_SIZE (8 * PAGE_SIZE)
555#else
556/* No preempt: go for improved straight-line efficiency */
557# define ZAP_BLOCK_SIZE (1024 * PAGE_SIZE)
558#endif
559
560/**
561 * unmap_vmas - unmap a range of memory covered by a list of vma's
562 * @tlbp: address of the caller's struct mmu_gather
563 * @mm: the controlling mm_struct
564 * @vma: the starting vma
565 * @start_addr: virtual address at which to start unmapping
566 * @end_addr: virtual address at which to end unmapping
567 * @nr_accounted: Place number of unmapped pages in vm-accountable vma's here
568 * @details: details of nonlinear truncation or shared cache invalidation
569 *
570 * Returns the number of vma's which were covered by the unmapping.
571 *
572 * Unmap all pages in the vma list. Called under page_table_lock.
573 *
574 * We aim to not hold page_table_lock for too long (for scheduling latency
575 * reasons). So zap pages in ZAP_BLOCK_SIZE bytecounts. This means we need to
576 * return the ending mmu_gather to the caller.
577 *
578 * Only addresses between `start' and `end' will be unmapped.
579 *
580 * The VMA list must be sorted in ascending virtual address order.
581 *
582 * unmap_vmas() assumes that the caller will flush the whole unmapped address
583 * range after unmap_vmas() returns. So the only responsibility here is to
584 * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
585 * drops the lock and schedules.
586 */
587int unmap_vmas(struct mmu_gather **tlbp, struct mm_struct *mm,
588 struct vm_area_struct *vma, unsigned long start_addr,
589 unsigned long end_addr, unsigned long *nr_accounted,
590 struct zap_details *details)
591{
592 unsigned long zap_bytes = ZAP_BLOCK_SIZE;
593 unsigned long tlb_start = 0; /* For tlb_finish_mmu */
594 int tlb_start_valid = 0;
595 int ret = 0;
596 spinlock_t *i_mmap_lock = details? details->i_mmap_lock: NULL;
597 int fullmm = tlb_is_full_mm(*tlbp);
598
599 for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) {
600 unsigned long start;
601 unsigned long end;
602
603 start = max(vma->vm_start, start_addr);
604 if (start >= vma->vm_end)
605 continue;
606 end = min(vma->vm_end, end_addr);
607 if (end <= vma->vm_start)
608 continue;
609
610 if (vma->vm_flags & VM_ACCOUNT)
611 *nr_accounted += (end - start) >> PAGE_SHIFT;
612
613 ret++;
614 while (start != end) {
615 unsigned long block;
616
617 if (!tlb_start_valid) {
618 tlb_start = start;
619 tlb_start_valid = 1;
620 }
621
622 if (is_vm_hugetlb_page(vma)) {
623 block = end - start;
624 unmap_hugepage_range(vma, start, end);
625 } else {
626 block = min(zap_bytes, end - start);
627 unmap_page_range(*tlbp, vma, start,
628 start + block, details);
629 }
630
631 start += block;
632 zap_bytes -= block;
633 if ((long)zap_bytes > 0)
634 continue;
635
636 tlb_finish_mmu(*tlbp, tlb_start, start);
637
638 if (need_resched() ||
639 need_lockbreak(&mm->page_table_lock) ||
640 (i_mmap_lock && need_lockbreak(i_mmap_lock))) {
641 if (i_mmap_lock) {
642 /* must reset count of rss freed */
643 *tlbp = tlb_gather_mmu(mm, fullmm);
644 details->break_addr = start;
645 goto out;
646 }
647 spin_unlock(&mm->page_table_lock);
648 cond_resched();
649 spin_lock(&mm->page_table_lock);
650 }
651
652 *tlbp = tlb_gather_mmu(mm, fullmm);
653 tlb_start_valid = 0;
654 zap_bytes = ZAP_BLOCK_SIZE;
655 }
656 }
657out:
658 return ret;
659}
660
661/**
662 * zap_page_range - remove user pages in a given range
663 * @vma: vm_area_struct holding the applicable pages
664 * @address: starting address of pages to zap
665 * @size: number of bytes to zap
666 * @details: details of nonlinear truncation or shared cache invalidation
667 */
668void zap_page_range(struct vm_area_struct *vma, unsigned long address,
669 unsigned long size, struct zap_details *details)
670{
671 struct mm_struct *mm = vma->vm_mm;
672 struct mmu_gather *tlb;
673 unsigned long end = address + size;
674 unsigned long nr_accounted = 0;
675
676 if (is_vm_hugetlb_page(vma)) {
677 zap_hugepage_range(vma, address, size);
678 return;
679 }
680
681 lru_add_drain();
682 spin_lock(&mm->page_table_lock);
683 tlb = tlb_gather_mmu(mm, 0);
684 unmap_vmas(&tlb, mm, vma, address, end, &nr_accounted, details);
685 tlb_finish_mmu(tlb, address, end);
686 spin_unlock(&mm->page_table_lock);
687}
688
689/*
690 * Do a quick page-table lookup for a single page.
691 * mm->page_table_lock must be held.
692 */
693static struct page *
694__follow_page(struct mm_struct *mm, unsigned long address, int read, int write)
695{
696 pgd_t *pgd;
697 pud_t *pud;
698 pmd_t *pmd;
699 pte_t *ptep, pte;
700 unsigned long pfn;
701 struct page *page;
702
703 page = follow_huge_addr(mm, address, write);
704 if (! IS_ERR(page))
705 return page;
706
707 pgd = pgd_offset(mm, address);
708 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
709 goto out;
710
711 pud = pud_offset(pgd, address);
712 if (pud_none(*pud) || unlikely(pud_bad(*pud)))
713 goto out;
714
715 pmd = pmd_offset(pud, address);
716 if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
717 goto out;
718 if (pmd_huge(*pmd))
719 return follow_huge_pmd(mm, address, pmd, write);
720
721 ptep = pte_offset_map(pmd, address);
722 if (!ptep)
723 goto out;
724
725 pte = *ptep;
726 pte_unmap(ptep);
727 if (pte_present(pte)) {
728 if (write && !pte_write(pte))
729 goto out;
730 if (read && !pte_read(pte))
731 goto out;
732 pfn = pte_pfn(pte);
733 if (pfn_valid(pfn)) {
734 page = pfn_to_page(pfn);
735 if (write && !pte_dirty(pte) && !PageDirty(page))
736 set_page_dirty(page);
737 mark_page_accessed(page);
738 return page;
739 }
740 }
741
742out:
743 return NULL;
744}
745
746struct page *
747follow_page(struct mm_struct *mm, unsigned long address, int write)
748{
749 return __follow_page(mm, address, /*read*/0, write);
750}
751
752int
753check_user_page_readable(struct mm_struct *mm, unsigned long address)
754{
755 return __follow_page(mm, address, /*read*/1, /*write*/0) != NULL;
756}
757
758EXPORT_SYMBOL(check_user_page_readable);
759
760/*
761 * Given a physical address, is there a useful struct page pointing to
762 * it? This may become more complex in the future if we start dealing
763 * with IO-aperture pages for direct-IO.
764 */
765
766static inline struct page *get_page_map(struct page *page)
767{
768 if (!pfn_valid(page_to_pfn(page)))
769 return NULL;
770 return page;
771}
772
773
774static inline int
775untouched_anonymous_page(struct mm_struct* mm, struct vm_area_struct *vma,
776 unsigned long address)
777{
778 pgd_t *pgd;
779 pud_t *pud;
780 pmd_t *pmd;
781
782 /* Check if the vma is for an anonymous mapping. */
783 if (vma->vm_ops && vma->vm_ops->nopage)
784 return 0;
785
786 /* Check if page directory entry exists. */
787 pgd = pgd_offset(mm, address);
788 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
789 return 1;
790
791 pud = pud_offset(pgd, address);
792 if (pud_none(*pud) || unlikely(pud_bad(*pud)))
793 return 1;
794
795 /* Check if page middle directory entry exists. */
796 pmd = pmd_offset(pud, address);
797 if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
798 return 1;
799
800 /* There is a pte slot for 'address' in 'mm'. */
801 return 0;
802}
803
804
805int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
806 unsigned long start, int len, int write, int force,
807 struct page **pages, struct vm_area_struct **vmas)
808{
809 int i;
810 unsigned int flags;
811
812 /*
813 * Require read or write permissions.
814 * If 'force' is set, we only require the "MAY" flags.
815 */
816 flags = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
817 flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
818 i = 0;
819
820 do {
821 struct vm_area_struct * vma;
822
823 vma = find_extend_vma(mm, start);
824 if (!vma && in_gate_area(tsk, start)) {
825 unsigned long pg = start & PAGE_MASK;
826 struct vm_area_struct *gate_vma = get_gate_vma(tsk);
827 pgd_t *pgd;
828 pud_t *pud;
829 pmd_t *pmd;
830 pte_t *pte;
831 if (write) /* user gate pages are read-only */
832 return i ? : -EFAULT;
833 if (pg > TASK_SIZE)
834 pgd = pgd_offset_k(pg);
835 else
836 pgd = pgd_offset_gate(mm, pg);
837 BUG_ON(pgd_none(*pgd));
838 pud = pud_offset(pgd, pg);
839 BUG_ON(pud_none(*pud));
840 pmd = pmd_offset(pud, pg);
841 BUG_ON(pmd_none(*pmd));
842 pte = pte_offset_map(pmd, pg);
843 BUG_ON(pte_none(*pte));
844 if (pages) {
845 pages[i] = pte_page(*pte);
846 get_page(pages[i]);
847 }
848 pte_unmap(pte);
849 if (vmas)
850 vmas[i] = gate_vma;
851 i++;
852 start += PAGE_SIZE;
853 len--;
854 continue;
855 }
856
857 if (!vma || (vma->vm_flags & VM_IO)
858 || !(flags & vma->vm_flags))
859 return i ? : -EFAULT;
860
861 if (is_vm_hugetlb_page(vma)) {
862 i = follow_hugetlb_page(mm, vma, pages, vmas,
863 &start, &len, i);
864 continue;
865 }
866 spin_lock(&mm->page_table_lock);
867 do {
868 struct page *map;
869 int lookup_write = write;
870
871 cond_resched_lock(&mm->page_table_lock);
872 while (!(map = follow_page(mm, start, lookup_write))) {
873 /*
874 * Shortcut for anonymous pages. We don't want
875 * to force the creation of pages tables for
876 * insanly big anonymously mapped areas that
877 * nobody touched so far. This is important
878 * for doing a core dump for these mappings.
879 */
880 if (!lookup_write &&
881 untouched_anonymous_page(mm,vma,start)) {
882 map = ZERO_PAGE(start);
883 break;
884 }
885 spin_unlock(&mm->page_table_lock);
886 switch (handle_mm_fault(mm,vma,start,write)) {
887 case VM_FAULT_MINOR:
888 tsk->min_flt++;
889 break;
890 case VM_FAULT_MAJOR:
891 tsk->maj_flt++;
892 break;
893 case VM_FAULT_SIGBUS:
894 return i ? i : -EFAULT;
895 case VM_FAULT_OOM:
896 return i ? i : -ENOMEM;
897 default:
898 BUG();
899 }
900 /*
901 * Now that we have performed a write fault
902 * and surely no longer have a shared page we
903 * shouldn't write, we shouldn't ignore an
904 * unwritable page in the page table if
905 * we are forcing write access.
906 */
907 lookup_write = write && !force;
908 spin_lock(&mm->page_table_lock);
909 }
910 if (pages) {
911 pages[i] = get_page_map(map);
912 if (!pages[i]) {
913 spin_unlock(&mm->page_table_lock);
914 while (i--)
915 page_cache_release(pages[i]);
916 i = -EFAULT;
917 goto out;
918 }
919 flush_dcache_page(pages[i]);
920 if (!PageReserved(pages[i]))
921 page_cache_get(pages[i]);
922 }
923 if (vmas)
924 vmas[i] = vma;
925 i++;
926 start += PAGE_SIZE;
927 len--;
928 } while(len && start < vma->vm_end);
929 spin_unlock(&mm->page_table_lock);
930 } while(len);
931out:
932 return i;
933}
934
935EXPORT_SYMBOL(get_user_pages);
936
937static int zeromap_pte_range(struct mm_struct *mm, pmd_t *pmd,
938 unsigned long addr, unsigned long end, pgprot_t prot)
939{
940 pte_t *pte;
941
942 pte = pte_alloc_map(mm, pmd, addr);
943 if (!pte)
944 return -ENOMEM;
945 do {
946 pte_t zero_pte = pte_wrprotect(mk_pte(ZERO_PAGE(addr), prot));
947 BUG_ON(!pte_none(*pte));
948 set_pte_at(mm, addr, pte, zero_pte);
949 } while (pte++, addr += PAGE_SIZE, addr != end);
950 pte_unmap(pte - 1);
951 return 0;
952}
953
954static inline int zeromap_pmd_range(struct mm_struct *mm, pud_t *pud,
955 unsigned long addr, unsigned long end, pgprot_t prot)
956{
957 pmd_t *pmd;
958 unsigned long next;
959
960 pmd = pmd_alloc(mm, pud, addr);
961 if (!pmd)
962 return -ENOMEM;
963 do {
964 next = pmd_addr_end(addr, end);
965 if (zeromap_pte_range(mm, pmd, addr, next, prot))
966 return -ENOMEM;
967 } while (pmd++, addr = next, addr != end);
968 return 0;
969}
970
971static inline int zeromap_pud_range(struct mm_struct *mm, pgd_t *pgd,
972 unsigned long addr, unsigned long end, pgprot_t prot)
973{
974 pud_t *pud;
975 unsigned long next;
976
977 pud = pud_alloc(mm, pgd, addr);
978 if (!pud)
979 return -ENOMEM;
980 do {
981 next = pud_addr_end(addr, end);
982 if (zeromap_pmd_range(mm, pud, addr, next, prot))
983 return -ENOMEM;
984 } while (pud++, addr = next, addr != end);
985 return 0;
986}
987
988int zeromap_page_range(struct vm_area_struct *vma,
989 unsigned long addr, unsigned long size, pgprot_t prot)
990{
991 pgd_t *pgd;
992 unsigned long next;
993 unsigned long end = addr + size;
994 struct mm_struct *mm = vma->vm_mm;
995 int err;
996
997 BUG_ON(addr >= end);
998 pgd = pgd_offset(mm, addr);
999 flush_cache_range(vma, addr, end);
1000 spin_lock(&mm->page_table_lock);
1001 do {
1002 next = pgd_addr_end(addr, end);
1003 err = zeromap_pud_range(mm, pgd, addr, next, prot);
1004 if (err)
1005 break;
1006 } while (pgd++, addr = next, addr != end);
1007 spin_unlock(&mm->page_table_lock);
1008 return err;
1009}
1010
1011/*
1012 * maps a range of physical memory into the requested pages. the old
1013 * mappings are removed. any references to nonexistent pages results
1014 * in null mappings (currently treated as "copy-on-access")
1015 */
1016static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
1017 unsigned long addr, unsigned long end,
1018 unsigned long pfn, pgprot_t prot)
1019{
1020 pte_t *pte;
1021
1022 pte = pte_alloc_map(mm, pmd, addr);
1023 if (!pte)
1024 return -ENOMEM;
1025 do {
1026 BUG_ON(!pte_none(*pte));
1027 if (!pfn_valid(pfn) || PageReserved(pfn_to_page(pfn)))
1028 set_pte_at(mm, addr, pte, pfn_pte(pfn, prot));
1029 pfn++;
1030 } while (pte++, addr += PAGE_SIZE, addr != end);
1031 pte_unmap(pte - 1);
1032 return 0;
1033}
1034
1035static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
1036 unsigned long addr, unsigned long end,
1037 unsigned long pfn, pgprot_t prot)
1038{
1039 pmd_t *pmd;
1040 unsigned long next;
1041
1042 pfn -= addr >> PAGE_SHIFT;
1043 pmd = pmd_alloc(mm, pud, addr);
1044 if (!pmd)
1045 return -ENOMEM;
1046 do {
1047 next = pmd_addr_end(addr, end);
1048 if (remap_pte_range(mm, pmd, addr, next,
1049 pfn + (addr >> PAGE_SHIFT), prot))
1050 return -ENOMEM;
1051 } while (pmd++, addr = next, addr != end);
1052 return 0;
1053}
1054
1055static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd,
1056 unsigned long addr, unsigned long end,
1057 unsigned long pfn, pgprot_t prot)
1058{
1059 pud_t *pud;
1060 unsigned long next;
1061
1062 pfn -= addr >> PAGE_SHIFT;
1063 pud = pud_alloc(mm, pgd, addr);
1064 if (!pud)
1065 return -ENOMEM;
1066 do {
1067 next = pud_addr_end(addr, end);
1068 if (remap_pmd_range(mm, pud, addr, next,
1069 pfn + (addr >> PAGE_SHIFT), prot))
1070 return -ENOMEM;
1071 } while (pud++, addr = next, addr != end);
1072 return 0;
1073}
1074
1075/* Note: this is only safe if the mm semaphore is held when called. */
1076int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
1077 unsigned long pfn, unsigned long size, pgprot_t prot)
1078{
1079 pgd_t *pgd;
1080 unsigned long next;
1081 unsigned long end = addr + size;
1082 struct mm_struct *mm = vma->vm_mm;
1083 int err;
1084
1085 /*
1086 * Physically remapped pages are special. Tell the
1087 * rest of the world about it:
1088 * VM_IO tells people not to look at these pages
1089 * (accesses can have side effects).
1090 * VM_RESERVED tells swapout not to try to touch
1091 * this region.
1092 */
1093 vma->vm_flags |= VM_IO | VM_RESERVED;
1094
1095 BUG_ON(addr >= end);
1096 pfn -= addr >> PAGE_SHIFT;
1097 pgd = pgd_offset(mm, addr);
1098 flush_cache_range(vma, addr, end);
1099 spin_lock(&mm->page_table_lock);
1100 do {
1101 next = pgd_addr_end(addr, end);
1102 err = remap_pud_range(mm, pgd, addr, next,
1103 pfn + (addr >> PAGE_SHIFT), prot);
1104 if (err)
1105 break;
1106 } while (pgd++, addr = next, addr != end);
1107 spin_unlock(&mm->page_table_lock);
1108 return err;
1109}
1110EXPORT_SYMBOL(remap_pfn_range);
1111
1112/*
1113 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
1114 * servicing faults for write access. In the normal case, do always want
1115 * pte_mkwrite. But get_user_pages can cause write faults for mappings
1116 * that do not have writing enabled, when used by access_process_vm.
1117 */
1118static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
1119{
1120 if (likely(vma->vm_flags & VM_WRITE))
1121 pte = pte_mkwrite(pte);
1122 return pte;
1123}
1124
1125/*
1126 * We hold the mm semaphore for reading and vma->vm_mm->page_table_lock
1127 */
1128static inline void break_cow(struct vm_area_struct * vma, struct page * new_page, unsigned long address,
1129 pte_t *page_table)
1130{
1131 pte_t entry;
1132
1133 entry = maybe_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot)),
1134 vma);
1135 ptep_establish(vma, address, page_table, entry);
1136 update_mmu_cache(vma, address, entry);
1137 lazy_mmu_prot_update(entry);
1138}
1139
1140/*
1141 * This routine handles present pages, when users try to write
1142 * to a shared page. It is done by copying the page to a new address
1143 * and decrementing the shared-page counter for the old page.
1144 *
1145 * Goto-purists beware: the only reason for goto's here is that it results
1146 * in better assembly code.. The "default" path will see no jumps at all.
1147 *
1148 * Note that this routine assumes that the protection checks have been
1149 * done by the caller (the low-level page fault routine in most cases).
1150 * Thus we can safely just mark it writable once we've done any necessary
1151 * COW.
1152 *
1153 * We also mark the page dirty at this point even though the page will
1154 * change only once the write actually happens. This avoids a few races,
1155 * and potentially makes it more efficient.
1156 *
1157 * We hold the mm semaphore and the page_table_lock on entry and exit
1158 * with the page_table_lock released.
1159 */
1160static int do_wp_page(struct mm_struct *mm, struct vm_area_struct * vma,
1161 unsigned long address, pte_t *page_table, pmd_t *pmd, pte_t pte)
1162{
1163 struct page *old_page, *new_page;
1164 unsigned long pfn = pte_pfn(pte);
1165 pte_t entry;
1166
1167 if (unlikely(!pfn_valid(pfn))) {
1168 /*
1169 * This should really halt the system so it can be debugged or
1170 * at least the kernel stops what it's doing before it corrupts
1171 * data, but for the moment just pretend this is OOM.
1172 */
1173 pte_unmap(page_table);
1174 printk(KERN_ERR "do_wp_page: bogus page at address %08lx\n",
1175 address);
1176 spin_unlock(&mm->page_table_lock);
1177 return VM_FAULT_OOM;
1178 }
1179 old_page = pfn_to_page(pfn);
1180
1181 if (!TestSetPageLocked(old_page)) {
1182 int reuse = can_share_swap_page(old_page);
1183 unlock_page(old_page);
1184 if (reuse) {
1185 flush_cache_page(vma, address, pfn);
1186 entry = maybe_mkwrite(pte_mkyoung(pte_mkdirty(pte)),
1187 vma);
1188 ptep_set_access_flags(vma, address, page_table, entry, 1);
1189 update_mmu_cache(vma, address, entry);
1190 lazy_mmu_prot_update(entry);
1191 pte_unmap(page_table);
1192 spin_unlock(&mm->page_table_lock);
1193 return VM_FAULT_MINOR;
1194 }
1195 }
1196 pte_unmap(page_table);
1197
1198 /*
1199 * Ok, we need to copy. Oh, well..
1200 */
1201 if (!PageReserved(old_page))
1202 page_cache_get(old_page);
1203 spin_unlock(&mm->page_table_lock);
1204
1205 if (unlikely(anon_vma_prepare(vma)))
1206 goto no_new_page;
1207 if (old_page == ZERO_PAGE(address)) {
1208 new_page = alloc_zeroed_user_highpage(vma, address);
1209 if (!new_page)
1210 goto no_new_page;
1211 } else {
1212 new_page = alloc_page_vma(GFP_HIGHUSER, vma, address);
1213 if (!new_page)
1214 goto no_new_page;
1215 copy_user_highpage(new_page, old_page, address);
1216 }
1217 /*
1218 * Re-check the pte - we dropped the lock
1219 */
1220 spin_lock(&mm->page_table_lock);
1221 page_table = pte_offset_map(pmd, address);
1222 if (likely(pte_same(*page_table, pte))) {
1223 if (PageAnon(old_page))
1224 dec_mm_counter(mm, anon_rss);
1225 if (PageReserved(old_page))
1226 inc_mm_counter(mm, rss);
1227 else
1228 page_remove_rmap(old_page);
1229 flush_cache_page(vma, address, pfn);
1230 break_cow(vma, new_page, address, page_table);
1231 lru_cache_add_active(new_page);
1232 page_add_anon_rmap(new_page, vma, address);
1233
1234 /* Free the old page.. */
1235 new_page = old_page;
1236 }
1237 pte_unmap(page_table);
1238 page_cache_release(new_page);
1239 page_cache_release(old_page);
1240 spin_unlock(&mm->page_table_lock);
1241 return VM_FAULT_MINOR;
1242
1243no_new_page:
1244 page_cache_release(old_page);
1245 return VM_FAULT_OOM;
1246}
1247
1248/*
1249 * Helper functions for unmap_mapping_range().
1250 *
1251 * __ Notes on dropping i_mmap_lock to reduce latency while unmapping __
1252 *
1253 * We have to restart searching the prio_tree whenever we drop the lock,
1254 * since the iterator is only valid while the lock is held, and anyway
1255 * a later vma might be split and reinserted earlier while lock dropped.
1256 *
1257 * The list of nonlinear vmas could be handled more efficiently, using
1258 * a placeholder, but handle it in the same way until a need is shown.
1259 * It is important to search the prio_tree before nonlinear list: a vma
1260 * may become nonlinear and be shifted from prio_tree to nonlinear list
1261 * while the lock is dropped; but never shifted from list to prio_tree.
1262 *
1263 * In order to make forward progress despite restarting the search,
1264 * vm_truncate_count is used to mark a vma as now dealt with, so we can
1265 * quickly skip it next time around. Since the prio_tree search only
1266 * shows us those vmas affected by unmapping the range in question, we
1267 * can't efficiently keep all vmas in step with mapping->truncate_count:
1268 * so instead reset them all whenever it wraps back to 0 (then go to 1).
1269 * mapping->truncate_count and vma->vm_truncate_count are protected by
1270 * i_mmap_lock.
1271 *
1272 * In order to make forward progress despite repeatedly restarting some
1273 * large vma, note the break_addr set by unmap_vmas when it breaks out:
1274 * and restart from that address when we reach that vma again. It might
1275 * have been split or merged, shrunk or extended, but never shifted: so
1276 * restart_addr remains valid so long as it remains in the vma's range.
1277 * unmap_mapping_range forces truncate_count to leap over page-aligned
1278 * values so we can save vma's restart_addr in its truncate_count field.
1279 */
1280#define is_restart_addr(truncate_count) (!((truncate_count) & ~PAGE_MASK))
1281
1282static void reset_vma_truncate_counts(struct address_space *mapping)
1283{
1284 struct vm_area_struct *vma;
1285 struct prio_tree_iter iter;
1286
1287 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, 0, ULONG_MAX)
1288 vma->vm_truncate_count = 0;
1289 list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.vm_set.list)
1290 vma->vm_truncate_count = 0;
1291}
1292
1293static int unmap_mapping_range_vma(struct vm_area_struct *vma,
1294 unsigned long start_addr, unsigned long end_addr,
1295 struct zap_details *details)
1296{
1297 unsigned long restart_addr;
1298 int need_break;
1299
1300again:
1301 restart_addr = vma->vm_truncate_count;
1302 if (is_restart_addr(restart_addr) && start_addr < restart_addr) {
1303 start_addr = restart_addr;
1304 if (start_addr >= end_addr) {
1305 /* Top of vma has been split off since last time */
1306 vma->vm_truncate_count = details->truncate_count;
1307 return 0;
1308 }
1309 }
1310
1311 details->break_addr = end_addr;
1312 zap_page_range(vma, start_addr, end_addr - start_addr, details);
1313
1314 /*
1315 * We cannot rely on the break test in unmap_vmas:
1316 * on the one hand, we don't want to restart our loop
1317 * just because that broke out for the page_table_lock;
1318 * on the other hand, it does no test when vma is small.
1319 */
1320 need_break = need_resched() ||
1321 need_lockbreak(details->i_mmap_lock);
1322
1323 if (details->break_addr >= end_addr) {
1324 /* We have now completed this vma: mark it so */
1325 vma->vm_truncate_count = details->truncate_count;
1326 if (!need_break)
1327 return 0;
1328 } else {
1329 /* Note restart_addr in vma's truncate_count field */
1330 vma->vm_truncate_count = details->break_addr;
1331 if (!need_break)
1332 goto again;
1333 }
1334
1335 spin_unlock(details->i_mmap_lock);
1336 cond_resched();
1337 spin_lock(details->i_mmap_lock);
1338 return -EINTR;
1339}
1340
1341static inline void unmap_mapping_range_tree(struct prio_tree_root *root,
1342 struct zap_details *details)
1343{
1344 struct vm_area_struct *vma;
1345 struct prio_tree_iter iter;
1346 pgoff_t vba, vea, zba, zea;
1347
1348restart:
1349 vma_prio_tree_foreach(vma, &iter, root,
1350 details->first_index, details->last_index) {
1351 /* Skip quickly over those we have already dealt with */
1352 if (vma->vm_truncate_count == details->truncate_count)
1353 continue;
1354
1355 vba = vma->vm_pgoff;
1356 vea = vba + ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) - 1;
1357 /* Assume for now that PAGE_CACHE_SHIFT == PAGE_SHIFT */
1358 zba = details->first_index;
1359 if (zba < vba)
1360 zba = vba;
1361 zea = details->last_index;
1362 if (zea > vea)
1363 zea = vea;
1364
1365 if (unmap_mapping_range_vma(vma,
1366 ((zba - vba) << PAGE_SHIFT) + vma->vm_start,
1367 ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
1368 details) < 0)
1369 goto restart;
1370 }
1371}
1372
1373static inline void unmap_mapping_range_list(struct list_head *head,
1374 struct zap_details *details)
1375{
1376 struct vm_area_struct *vma;
1377
1378 /*
1379 * In nonlinear VMAs there is no correspondence between virtual address
1380 * offset and file offset. So we must perform an exhaustive search
1381 * across *all* the pages in each nonlinear VMA, not just the pages
1382 * whose virtual address lies outside the file truncation point.
1383 */
1384restart:
1385 list_for_each_entry(vma, head, shared.vm_set.list) {
1386 /* Skip quickly over those we have already dealt with */
1387 if (vma->vm_truncate_count == details->truncate_count)
1388 continue;
1389 details->nonlinear_vma = vma;
1390 if (unmap_mapping_range_vma(vma, vma->vm_start,
1391 vma->vm_end, details) < 0)
1392 goto restart;
1393 }
1394}
1395
1396/**
1397 * unmap_mapping_range - unmap the portion of all mmaps
1398 * in the specified address_space corresponding to the specified
1399 * page range in the underlying file.
1400 * @address_space: the address space containing mmaps to be unmapped.
1401 * @holebegin: byte in first page to unmap, relative to the start of
1402 * the underlying file. This will be rounded down to a PAGE_SIZE
1403 * boundary. Note that this is different from vmtruncate(), which
1404 * must keep the partial page. In contrast, we must get rid of
1405 * partial pages.
1406 * @holelen: size of prospective hole in bytes. This will be rounded
1407 * up to a PAGE_SIZE boundary. A holelen of zero truncates to the
1408 * end of the file.
1409 * @even_cows: 1 when truncating a file, unmap even private COWed pages;
1410 * but 0 when invalidating pagecache, don't throw away private data.
1411 */
1412void unmap_mapping_range(struct address_space *mapping,
1413 loff_t const holebegin, loff_t const holelen, int even_cows)
1414{
1415 struct zap_details details;
1416 pgoff_t hba = holebegin >> PAGE_SHIFT;
1417 pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
1418
1419 /* Check for overflow. */
1420 if (sizeof(holelen) > sizeof(hlen)) {
1421 long long holeend =
1422 (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
1423 if (holeend & ~(long long)ULONG_MAX)
1424 hlen = ULONG_MAX - hba + 1;
1425 }
1426
1427 details.check_mapping = even_cows? NULL: mapping;
1428 details.nonlinear_vma = NULL;
1429 details.first_index = hba;
1430 details.last_index = hba + hlen - 1;
1431 if (details.last_index < details.first_index)
1432 details.last_index = ULONG_MAX;
1433 details.i_mmap_lock = &mapping->i_mmap_lock;
1434
1435 spin_lock(&mapping->i_mmap_lock);
1436
1437 /* serialize i_size write against truncate_count write */
1438 smp_wmb();
1439 /* Protect against page faults, and endless unmapping loops */
1440 mapping->truncate_count++;
1441 /*
1442 * For archs where spin_lock has inclusive semantics like ia64
1443 * this smp_mb() will prevent to read pagetable contents
1444 * before the truncate_count increment is visible to
1445 * other cpus.
1446 */
1447 smp_mb();
1448 if (unlikely(is_restart_addr(mapping->truncate_count))) {
1449 if (mapping->truncate_count == 0)
1450 reset_vma_truncate_counts(mapping);
1451 mapping->truncate_count++;
1452 }
1453 details.truncate_count = mapping->truncate_count;
1454
1455 if (unlikely(!prio_tree_empty(&mapping->i_mmap)))
1456 unmap_mapping_range_tree(&mapping->i_mmap, &details);
1457 if (unlikely(!list_empty(&mapping->i_mmap_nonlinear)))
1458 unmap_mapping_range_list(&mapping->i_mmap_nonlinear, &details);
1459 spin_unlock(&mapping->i_mmap_lock);
1460}
1461EXPORT_SYMBOL(unmap_mapping_range);
1462
1463/*
1464 * Handle all mappings that got truncated by a "truncate()"
1465 * system call.
1466 *
1467 * NOTE! We have to be ready to update the memory sharing
1468 * between the file and the memory map for a potential last
1469 * incomplete page. Ugly, but necessary.
1470 */
1471int vmtruncate(struct inode * inode, loff_t offset)
1472{
1473 struct address_space *mapping = inode->i_mapping;
1474 unsigned long limit;
1475
1476 if (inode->i_size < offset)
1477 goto do_expand;
1478 /*
1479 * truncation of in-use swapfiles is disallowed - it would cause
1480 * subsequent swapout to scribble on the now-freed blocks.
1481 */
1482 if (IS_SWAPFILE(inode))
1483 goto out_busy;
1484 i_size_write(inode, offset);
1485 unmap_mapping_range(mapping, offset + PAGE_SIZE - 1, 0, 1);
1486 truncate_inode_pages(mapping, offset);
1487 goto out_truncate;
1488
1489do_expand:
1490 limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
1491 if (limit != RLIM_INFINITY && offset > limit)
1492 goto out_sig;
1493 if (offset > inode->i_sb->s_maxbytes)
1494 goto out_big;
1495 i_size_write(inode, offset);
1496
1497out_truncate:
1498 if (inode->i_op && inode->i_op->truncate)
1499 inode->i_op->truncate(inode);
1500 return 0;
1501out_sig:
1502 send_sig(SIGXFSZ, current, 0);
1503out_big:
1504 return -EFBIG;
1505out_busy:
1506 return -ETXTBSY;
1507}
1508
1509EXPORT_SYMBOL(vmtruncate);
1510
1511/*
1512 * Primitive swap readahead code. We simply read an aligned block of
1513 * (1 << page_cluster) entries in the swap area. This method is chosen
1514 * because it doesn't cost us any seek time. We also make sure to queue
1515 * the 'original' request together with the readahead ones...
1516 *
1517 * This has been extended to use the NUMA policies from the mm triggering
1518 * the readahead.
1519 *
1520 * Caller must hold down_read on the vma->vm_mm if vma is not NULL.
1521 */
1522void swapin_readahead(swp_entry_t entry, unsigned long addr,struct vm_area_struct *vma)
1523{
1524#ifdef CONFIG_NUMA
1525 struct vm_area_struct *next_vma = vma ? vma->vm_next : NULL;
1526#endif
1527 int i, num;
1528 struct page *new_page;
1529 unsigned long offset;
1530
1531 /*
1532 * Get the number of handles we should do readahead io to.
1533 */
1534 num = valid_swaphandles(entry, &offset);
1535 for (i = 0; i < num; offset++, i++) {
1536 /* Ok, do the async read-ahead now */
1537 new_page = read_swap_cache_async(swp_entry(swp_type(entry),
1538 offset), vma, addr);
1539 if (!new_page)
1540 break;
1541 page_cache_release(new_page);
1542#ifdef CONFIG_NUMA
1543 /*
1544 * Find the next applicable VMA for the NUMA policy.
1545 */
1546 addr += PAGE_SIZE;
1547 if (addr == 0)
1548 vma = NULL;
1549 if (vma) {
1550 if (addr >= vma->vm_end) {
1551 vma = next_vma;
1552 next_vma = vma ? vma->vm_next : NULL;
1553 }
1554 if (vma && addr < vma->vm_start)
1555 vma = NULL;
1556 } else {
1557 if (next_vma && addr >= next_vma->vm_start) {
1558 vma = next_vma;
1559 next_vma = vma->vm_next;
1560 }
1561 }
1562#endif
1563 }
1564 lru_add_drain(); /* Push any new pages onto the LRU now */
1565}
1566
1567/*
1568 * We hold the mm semaphore and the page_table_lock on entry and
1569 * should release the pagetable lock on exit..
1570 */
1571static int do_swap_page(struct mm_struct * mm,
1572 struct vm_area_struct * vma, unsigned long address,
1573 pte_t *page_table, pmd_t *pmd, pte_t orig_pte, int write_access)
1574{
1575 struct page *page;
1576 swp_entry_t entry = pte_to_swp_entry(orig_pte);
1577 pte_t pte;
1578 int ret = VM_FAULT_MINOR;
1579
1580 pte_unmap(page_table);
1581 spin_unlock(&mm->page_table_lock);
1582 page = lookup_swap_cache(entry);
1583 if (!page) {
1584 swapin_readahead(entry, address, vma);
1585 page = read_swap_cache_async(entry, vma, address);
1586 if (!page) {
1587 /*
1588 * Back out if somebody else faulted in this pte while
1589 * we released the page table lock.
1590 */
1591 spin_lock(&mm->page_table_lock);
1592 page_table = pte_offset_map(pmd, address);
1593 if (likely(pte_same(*page_table, orig_pte)))
1594 ret = VM_FAULT_OOM;
1595 else
1596 ret = VM_FAULT_MINOR;
1597 pte_unmap(page_table);
1598 spin_unlock(&mm->page_table_lock);
1599 goto out;
1600 }
1601
1602 /* Had to read the page from swap area: Major fault */
1603 ret = VM_FAULT_MAJOR;
1604 inc_page_state(pgmajfault);
1605 grab_swap_token();
1606 }
1607
1608 mark_page_accessed(page);
1609 lock_page(page);
1610
1611 /*
1612 * Back out if somebody else faulted in this pte while we
1613 * released the page table lock.
1614 */
1615 spin_lock(&mm->page_table_lock);
1616 page_table = pte_offset_map(pmd, address);
1617 if (unlikely(!pte_same(*page_table, orig_pte))) {
1618 pte_unmap(page_table);
1619 spin_unlock(&mm->page_table_lock);
1620 unlock_page(page);
1621 page_cache_release(page);
1622 ret = VM_FAULT_MINOR;
1623 goto out;
1624 }
1625
1626 /* The page isn't present yet, go ahead with the fault. */
1627
1628 swap_free(entry);
1629 if (vm_swap_full())
1630 remove_exclusive_swap_page(page);
1631
1632 inc_mm_counter(mm, rss);
1633 pte = mk_pte(page, vma->vm_page_prot);
1634 if (write_access && can_share_swap_page(page)) {
1635 pte = maybe_mkwrite(pte_mkdirty(pte), vma);
1636 write_access = 0;
1637 }
1638 unlock_page(page);
1639
1640 flush_icache_page(vma, page);
1641 set_pte_at(mm, address, page_table, pte);
1642 page_add_anon_rmap(page, vma, address);
1643
1644 if (write_access) {
1645 if (do_wp_page(mm, vma, address,
1646 page_table, pmd, pte) == VM_FAULT_OOM)
1647 ret = VM_FAULT_OOM;
1648 goto out;
1649 }
1650
1651 /* No need to invalidate - it was non-present before */
1652 update_mmu_cache(vma, address, pte);
1653 lazy_mmu_prot_update(pte);
1654 pte_unmap(page_table);
1655 spin_unlock(&mm->page_table_lock);
1656out:
1657 return ret;
1658}
1659
1660/*
1661 * We are called with the MM semaphore and page_table_lock
1662 * spinlock held to protect against concurrent faults in
1663 * multithreaded programs.
1664 */
1665static int
1666do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
1667 pte_t *page_table, pmd_t *pmd, int write_access,
1668 unsigned long addr)
1669{
1670 pte_t entry;
1671 struct page * page = ZERO_PAGE(addr);
1672
1673 /* Read-only mapping of ZERO_PAGE. */
1674 entry = pte_wrprotect(mk_pte(ZERO_PAGE(addr), vma->vm_page_prot));
1675
1676 /* ..except if it's a write access */
1677 if (write_access) {
1678 /* Allocate our own private page. */
1679 pte_unmap(page_table);
1680 spin_unlock(&mm->page_table_lock);
1681
1682 if (unlikely(anon_vma_prepare(vma)))
1683 goto no_mem;
1684 page = alloc_zeroed_user_highpage(vma, addr);
1685 if (!page)
1686 goto no_mem;
1687
1688 spin_lock(&mm->page_table_lock);
1689 page_table = pte_offset_map(pmd, addr);
1690
1691 if (!pte_none(*page_table)) {
1692 pte_unmap(page_table);
1693 page_cache_release(page);
1694 spin_unlock(&mm->page_table_lock);
1695 goto out;
1696 }
1697 inc_mm_counter(mm, rss);
1698 entry = maybe_mkwrite(pte_mkdirty(mk_pte(page,
1699 vma->vm_page_prot)),
1700 vma);
1701 lru_cache_add_active(page);
1702 SetPageReferenced(page);
1703 page_add_anon_rmap(page, vma, addr);
1704 }
1705
1706 set_pte_at(mm, addr, page_table, entry);
1707 pte_unmap(page_table);
1708
1709 /* No need to invalidate - it was non-present before */
1710 update_mmu_cache(vma, addr, entry);
1711 lazy_mmu_prot_update(entry);
1712 spin_unlock(&mm->page_table_lock);
1713out:
1714 return VM_FAULT_MINOR;
1715no_mem:
1716 return VM_FAULT_OOM;
1717}
1718
1719/*
1720 * do_no_page() tries to create a new page mapping. It aggressively
1721 * tries to share with existing pages, but makes a separate copy if
1722 * the "write_access" parameter is true in order to avoid the next
1723 * page fault.
1724 *
1725 * As this is called only for pages that do not currently exist, we
1726 * do not need to flush old virtual caches or the TLB.
1727 *
1728 * This is called with the MM semaphore held and the page table
1729 * spinlock held. Exit with the spinlock released.
1730 */
1731static int
1732do_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
1733 unsigned long address, int write_access, pte_t *page_table, pmd_t *pmd)
1734{
1735 struct page * new_page;
1736 struct address_space *mapping = NULL;
1737 pte_t entry;
1738 unsigned int sequence = 0;
1739 int ret = VM_FAULT_MINOR;
1740 int anon = 0;
1741
1742 if (!vma->vm_ops || !vma->vm_ops->nopage)
1743 return do_anonymous_page(mm, vma, page_table,
1744 pmd, write_access, address);
1745 pte_unmap(page_table);
1746 spin_unlock(&mm->page_table_lock);
1747
1748 if (vma->vm_file) {
1749 mapping = vma->vm_file->f_mapping;
1750 sequence = mapping->truncate_count;
1751 smp_rmb(); /* serializes i_size against truncate_count */
1752 }
1753retry:
1754 cond_resched();
1755 new_page = vma->vm_ops->nopage(vma, address & PAGE_MASK, &ret);
1756 /*
1757 * No smp_rmb is needed here as long as there's a full
1758 * spin_lock/unlock sequence inside the ->nopage callback
1759 * (for the pagecache lookup) that acts as an implicit
1760 * smp_mb() and prevents the i_size read to happen
1761 * after the next truncate_count read.
1762 */
1763
1764 /* no page was available -- either SIGBUS or OOM */
1765 if (new_page == NOPAGE_SIGBUS)
1766 return VM_FAULT_SIGBUS;
1767 if (new_page == NOPAGE_OOM)
1768 return VM_FAULT_OOM;
1769
1770 /*
1771 * Should we do an early C-O-W break?
1772 */
1773 if (write_access && !(vma->vm_flags & VM_SHARED)) {
1774 struct page *page;
1775
1776 if (unlikely(anon_vma_prepare(vma)))
1777 goto oom;
1778 page = alloc_page_vma(GFP_HIGHUSER, vma, address);
1779 if (!page)
1780 goto oom;
1781 copy_user_highpage(page, new_page, address);
1782 page_cache_release(new_page);
1783 new_page = page;
1784 anon = 1;
1785 }
1786
1787 spin_lock(&mm->page_table_lock);
1788 /*
1789 * For a file-backed vma, someone could have truncated or otherwise
1790 * invalidated this page. If unmap_mapping_range got called,
1791 * retry getting the page.
1792 */
1793 if (mapping && unlikely(sequence != mapping->truncate_count)) {
1794 sequence = mapping->truncate_count;
1795 spin_unlock(&mm->page_table_lock);
1796 page_cache_release(new_page);
1797 goto retry;
1798 }
1799 page_table = pte_offset_map(pmd, address);
1800
1801 /*
1802 * This silly early PAGE_DIRTY setting removes a race
1803 * due to the bad i386 page protection. But it's valid
1804 * for other architectures too.
1805 *
1806 * Note that if write_access is true, we either now have
1807 * an exclusive copy of the page, or this is a shared mapping,
1808 * so we can make it writable and dirty to avoid having to
1809 * handle that later.
1810 */
1811 /* Only go through if we didn't race with anybody else... */
1812 if (pte_none(*page_table)) {
1813 if (!PageReserved(new_page))
1814 inc_mm_counter(mm, rss);
1815
1816 flush_icache_page(vma, new_page);
1817 entry = mk_pte(new_page, vma->vm_page_prot);
1818 if (write_access)
1819 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
1820 set_pte_at(mm, address, page_table, entry);
1821 if (anon) {
1822 lru_cache_add_active(new_page);
1823 page_add_anon_rmap(new_page, vma, address);
1824 } else
1825 page_add_file_rmap(new_page);
1826 pte_unmap(page_table);
1827 } else {
1828 /* One of our sibling threads was faster, back out. */
1829 pte_unmap(page_table);
1830 page_cache_release(new_page);
1831 spin_unlock(&mm->page_table_lock);
1832 goto out;
1833 }
1834
1835 /* no need to invalidate: a not-present page shouldn't be cached */
1836 update_mmu_cache(vma, address, entry);
1837 lazy_mmu_prot_update(entry);
1838 spin_unlock(&mm->page_table_lock);
1839out:
1840 return ret;
1841oom:
1842 page_cache_release(new_page);
1843 ret = VM_FAULT_OOM;
1844 goto out;
1845}
1846
1847/*
1848 * Fault of a previously existing named mapping. Repopulate the pte
1849 * from the encoded file_pte if possible. This enables swappable
1850 * nonlinear vmas.
1851 */
1852static int do_file_page(struct mm_struct * mm, struct vm_area_struct * vma,
1853 unsigned long address, int write_access, pte_t *pte, pmd_t *pmd)
1854{
1855 unsigned long pgoff;
1856 int err;
1857
1858 BUG_ON(!vma->vm_ops || !vma->vm_ops->nopage);
1859 /*
1860 * Fall back to the linear mapping if the fs does not support
1861 * ->populate:
1862 */
1863 if (!vma->vm_ops || !vma->vm_ops->populate ||
1864 (write_access && !(vma->vm_flags & VM_SHARED))) {
1865 pte_clear(mm, address, pte);
1866 return do_no_page(mm, vma, address, write_access, pte, pmd);
1867 }
1868
1869 pgoff = pte_to_pgoff(*pte);
1870
1871 pte_unmap(pte);
1872 spin_unlock(&mm->page_table_lock);
1873
1874 err = vma->vm_ops->populate(vma, address & PAGE_MASK, PAGE_SIZE, vma->vm_page_prot, pgoff, 0);
1875 if (err == -ENOMEM)
1876 return VM_FAULT_OOM;
1877 if (err)
1878 return VM_FAULT_SIGBUS;
1879 return VM_FAULT_MAJOR;
1880}
1881
1882/*
1883 * These routines also need to handle stuff like marking pages dirty
1884 * and/or accessed for architectures that don't do it in hardware (most
1885 * RISC architectures). The early dirtying is also good on the i386.
1886 *
1887 * There is also a hook called "update_mmu_cache()" that architectures
1888 * with external mmu caches can use to update those (ie the Sparc or
1889 * PowerPC hashed page tables that act as extended TLBs).
1890 *
1891 * Note the "page_table_lock". It is to protect against kswapd removing
1892 * pages from under us. Note that kswapd only ever _removes_ pages, never
1893 * adds them. As such, once we have noticed that the page is not present,
1894 * we can drop the lock early.
1895 *
1896 * The adding of pages is protected by the MM semaphore (which we hold),
1897 * so we don't need to worry about a page being suddenly been added into
1898 * our VM.
1899 *
1900 * We enter with the pagetable spinlock held, we are supposed to
1901 * release it when done.
1902 */
1903static inline int handle_pte_fault(struct mm_struct *mm,
1904 struct vm_area_struct * vma, unsigned long address,
1905 int write_access, pte_t *pte, pmd_t *pmd)
1906{
1907 pte_t entry;
1908
1909 entry = *pte;
1910 if (!pte_present(entry)) {
1911 /*
1912 * If it truly wasn't present, we know that kswapd
1913 * and the PTE updates will not touch it later. So
1914 * drop the lock.
1915 */
1916 if (pte_none(entry))
1917 return do_no_page(mm, vma, address, write_access, pte, pmd);
1918 if (pte_file(entry))
1919 return do_file_page(mm, vma, address, write_access, pte, pmd);
1920 return do_swap_page(mm, vma, address, pte, pmd, entry, write_access);
1921 }
1922
1923 if (write_access) {
1924 if (!pte_write(entry))
1925 return do_wp_page(mm, vma, address, pte, pmd, entry);
1926
1927 entry = pte_mkdirty(entry);
1928 }
1929 entry = pte_mkyoung(entry);
1930 ptep_set_access_flags(vma, address, pte, entry, write_access);
1931 update_mmu_cache(vma, address, entry);
1932 lazy_mmu_prot_update(entry);
1933 pte_unmap(pte);
1934 spin_unlock(&mm->page_table_lock);
1935 return VM_FAULT_MINOR;
1936}
1937
1938/*
1939 * By the time we get here, we already hold the mm semaphore
1940 */
1941int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct * vma,
1942 unsigned long address, int write_access)
1943{
1944 pgd_t *pgd;
1945 pud_t *pud;
1946 pmd_t *pmd;
1947 pte_t *pte;
1948
1949 __set_current_state(TASK_RUNNING);
1950
1951 inc_page_state(pgfault);
1952
1953 if (is_vm_hugetlb_page(vma))
1954 return VM_FAULT_SIGBUS; /* mapping truncation does this. */
1955
1956 /*
1957 * We need the page table lock to synchronize with kswapd
1958 * and the SMP-safe atomic PTE updates.
1959 */
1960 pgd = pgd_offset(mm, address);
1961 spin_lock(&mm->page_table_lock);
1962
1963 pud = pud_alloc(mm, pgd, address);
1964 if (!pud)
1965 goto oom;
1966
1967 pmd = pmd_alloc(mm, pud, address);
1968 if (!pmd)
1969 goto oom;
1970
1971 pte = pte_alloc_map(mm, pmd, address);
1972 if (!pte)
1973 goto oom;
1974
1975 return handle_pte_fault(mm, vma, address, write_access, pte, pmd);
1976
1977 oom:
1978 spin_unlock(&mm->page_table_lock);
1979 return VM_FAULT_OOM;
1980}
1981
1982#ifndef __PAGETABLE_PUD_FOLDED
1983/*
1984 * Allocate page upper directory.
1985 *
1986 * We've already handled the fast-path in-line, and we own the
1987 * page table lock.
1988 */
1989pud_t fastcall *__pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
1990{
1991 pud_t *new;
1992
1993 spin_unlock(&mm->page_table_lock);
1994 new = pud_alloc_one(mm, address);
1995 spin_lock(&mm->page_table_lock);
1996 if (!new)
1997 return NULL;
1998
1999 /*
2000 * Because we dropped the lock, we should re-check the
2001 * entry, as somebody else could have populated it..
2002 */
2003 if (pgd_present(*pgd)) {
2004 pud_free(new);
2005 goto out;
2006 }
2007 pgd_populate(mm, pgd, new);
2008 out:
2009 return pud_offset(pgd, address);
2010}
2011#endif /* __PAGETABLE_PUD_FOLDED */
2012
2013#ifndef __PAGETABLE_PMD_FOLDED
2014/*
2015 * Allocate page middle directory.
2016 *
2017 * We've already handled the fast-path in-line, and we own the
2018 * page table lock.
2019 */
2020pmd_t fastcall *__pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
2021{
2022 pmd_t *new;
2023
2024 spin_unlock(&mm->page_table_lock);
2025 new = pmd_alloc_one(mm, address);
2026 spin_lock(&mm->page_table_lock);
2027 if (!new)
2028 return NULL;
2029
2030 /*
2031 * Because we dropped the lock, we should re-check the
2032 * entry, as somebody else could have populated it..
2033 */
2034#ifndef __ARCH_HAS_4LEVEL_HACK
2035 if (pud_present(*pud)) {
2036 pmd_free(new);
2037 goto out;
2038 }
2039 pud_populate(mm, pud, new);
2040#else
2041 if (pgd_present(*pud)) {
2042 pmd_free(new);
2043 goto out;
2044 }
2045 pgd_populate(mm, pud, new);
2046#endif /* __ARCH_HAS_4LEVEL_HACK */
2047
2048 out:
2049 return pmd_offset(pud, address);
2050}
2051#endif /* __PAGETABLE_PMD_FOLDED */
2052
2053int make_pages_present(unsigned long addr, unsigned long end)
2054{
2055 int ret, len, write;
2056 struct vm_area_struct * vma;
2057
2058 vma = find_vma(current->mm, addr);
2059 if (!vma)
2060 return -1;
2061 write = (vma->vm_flags & VM_WRITE) != 0;
2062 if (addr >= end)
2063 BUG();
2064 if (end > vma->vm_end)
2065 BUG();
2066 len = (end+PAGE_SIZE-1)/PAGE_SIZE-addr/PAGE_SIZE;
2067 ret = get_user_pages(current, current->mm, addr,
2068 len, write, 0, NULL, NULL);
2069 if (ret < 0)
2070 return ret;
2071 return ret == len ? 0 : -1;
2072}
2073
2074/*
2075 * Map a vmalloc()-space virtual address to the physical page.
2076 */
2077struct page * vmalloc_to_page(void * vmalloc_addr)
2078{
2079 unsigned long addr = (unsigned long) vmalloc_addr;
2080 struct page *page = NULL;
2081 pgd_t *pgd = pgd_offset_k(addr);
2082 pud_t *pud;
2083 pmd_t *pmd;
2084 pte_t *ptep, pte;
2085
2086 if (!pgd_none(*pgd)) {
2087 pud = pud_offset(pgd, addr);
2088 if (!pud_none(*pud)) {
2089 pmd = pmd_offset(pud, addr);
2090 if (!pmd_none(*pmd)) {
2091 ptep = pte_offset_map(pmd, addr);
2092 pte = *ptep;
2093 if (pte_present(pte))
2094 page = pte_page(pte);
2095 pte_unmap(ptep);
2096 }
2097 }
2098 }
2099 return page;
2100}
2101
2102EXPORT_SYMBOL(vmalloc_to_page);
2103
2104/*
2105 * Map a vmalloc()-space virtual address to the physical page frame number.
2106 */
2107unsigned long vmalloc_to_pfn(void * vmalloc_addr)
2108{
2109 return page_to_pfn(vmalloc_to_page(vmalloc_addr));
2110}
2111
2112EXPORT_SYMBOL(vmalloc_to_pfn);
2113
2114/*
2115 * update_mem_hiwater
2116 * - update per process rss and vm high water data
2117 */
2118void update_mem_hiwater(struct task_struct *tsk)
2119{
2120 if (tsk->mm) {
2121 unsigned long rss = get_mm_counter(tsk->mm, rss);
2122
2123 if (tsk->mm->hiwater_rss < rss)
2124 tsk->mm->hiwater_rss = rss;
2125 if (tsk->mm->hiwater_vm < tsk->mm->total_vm)
2126 tsk->mm->hiwater_vm = tsk->mm->total_vm;
2127 }
2128}
2129
2130#if !defined(__HAVE_ARCH_GATE_AREA)
2131
2132#if defined(AT_SYSINFO_EHDR)
2133struct vm_area_struct gate_vma;
2134
2135static int __init gate_vma_init(void)
2136{
2137 gate_vma.vm_mm = NULL;
2138 gate_vma.vm_start = FIXADDR_USER_START;
2139 gate_vma.vm_end = FIXADDR_USER_END;
2140 gate_vma.vm_page_prot = PAGE_READONLY;
2141 gate_vma.vm_flags = 0;
2142 return 0;
2143}
2144__initcall(gate_vma_init);
2145#endif
2146
2147struct vm_area_struct *get_gate_vma(struct task_struct *tsk)
2148{
2149#ifdef AT_SYSINFO_EHDR
2150 return &gate_vma;
2151#else
2152 return NULL;
2153#endif
2154}
2155
2156int in_gate_area_no_task(unsigned long addr)
2157{
2158#ifdef AT_SYSINFO_EHDR
2159 if ((addr >= FIXADDR_USER_START) && (addr < FIXADDR_USER_END))
2160 return 1;
2161#endif
2162 return 0;
2163}
2164
2165#endif /* __HAVE_ARCH_GATE_AREA */
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-02 13:13:05 -0500