diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2011-07-26 00:00:19 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2011-07-26 00:00:19 -0400 |
commit | 45b583b10a8b438b970e95a7d1d4db22c9e35004 (patch) | |
tree | 14fa481598289df0459580c582b48a9d95db51f6 /arch/sparc | |
parent | 154dd78d30b56ffb8b447f629bfcceb14150e5c4 (diff) | |
parent | f19da2ce8ef5e49b8b8ea199c3601dd45d71b262 (diff) |
Merge 'akpm' patch series
* Merge akpm patch series: (122 commits)
drivers/connector/cn_proc.c: remove unused local
Documentation/SubmitChecklist: add RCU debug config options
reiserfs: use hweight_long()
reiserfs: use proper little-endian bitops
pnpacpi: register disabled resources
drivers/rtc/rtc-tegra.c: properly initialize spinlock
drivers/rtc/rtc-twl.c: check return value of twl_rtc_write_u8() in twl_rtc_set_time()
drivers/rtc: add support for Qualcomm PMIC8xxx RTC
drivers/rtc/rtc-s3c.c: support clock gating
drivers/rtc/rtc-mpc5121.c: add support for RTC on MPC5200
init: skip calibration delay if previously done
misc/eeprom: add eeprom access driver for digsy_mtc board
misc/eeprom: add driver for microwire 93xx46 EEPROMs
checkpatch.pl: update $logFunctions
checkpatch: make utf-8 test --strict
checkpatch.pl: add ability to ignore various messages
checkpatch: add a "prefer __aligned" check
checkpatch: validate signature styles and To: and Cc: lines
checkpatch: add __rcu as a sparse modifier
checkpatch: suggest using min_t or max_t
...
Did this as a merge because of (trivial) conflicts in
- Documentation/feature-removal-schedule.txt
- arch/xtensa/include/asm/uaccess.h
that were just easier to fix up in the merge than in the patch series.
Diffstat (limited to 'arch/sparc')
-rw-r--r-- | arch/sparc/Kconfig | 5 | ||||
-rw-r--r-- | arch/sparc/include/asm/pgalloc_64.h | 76 | ||||
-rw-r--r-- | arch/sparc/include/asm/pgtable_64.h | 15 | ||||
-rw-r--r-- | arch/sparc/mm/Makefile | 2 | ||||
-rw-r--r-- | arch/sparc/mm/gup.c | 181 | ||||
-rw-r--r-- | arch/sparc/mm/tsb.c | 11 |
6 files changed, 264 insertions, 26 deletions
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 253986bd6bb6..1074dddcb104 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig | |||
@@ -41,6 +41,7 @@ config SPARC64 | |||
41 | select HAVE_FUNCTION_TRACE_MCOUNT_TEST | 41 | select HAVE_FUNCTION_TRACE_MCOUNT_TEST |
42 | select HAVE_KRETPROBES | 42 | select HAVE_KRETPROBES |
43 | select HAVE_KPROBES | 43 | select HAVE_KPROBES |
44 | select HAVE_RCU_TABLE_FREE if SMP | ||
44 | select HAVE_MEMBLOCK | 45 | select HAVE_MEMBLOCK |
45 | select HAVE_SYSCALL_WRAPPERS | 46 | select HAVE_SYSCALL_WRAPPERS |
46 | select HAVE_DYNAMIC_FTRACE | 47 | select HAVE_DYNAMIC_FTRACE |
@@ -81,10 +82,6 @@ config IOMMU_HELPER | |||
81 | bool | 82 | bool |
82 | default y if SPARC64 | 83 | default y if SPARC64 |
83 | 84 | ||
84 | config QUICKLIST | ||
85 | bool | ||
86 | default y if SPARC64 | ||
87 | |||
88 | config STACKTRACE_SUPPORT | 85 | config STACKTRACE_SUPPORT |
89 | bool | 86 | bool |
90 | default y if SPARC64 | 87 | default y if SPARC64 |
diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h index 4e5e0878144f..40b2d7a7023d 100644 --- a/arch/sparc/include/asm/pgalloc_64.h +++ b/arch/sparc/include/asm/pgalloc_64.h | |||
@@ -5,7 +5,6 @@ | |||
5 | #include <linux/sched.h> | 5 | #include <linux/sched.h> |
6 | #include <linux/mm.h> | 6 | #include <linux/mm.h> |
7 | #include <linux/slab.h> | 7 | #include <linux/slab.h> |
8 | #include <linux/quicklist.h> | ||
9 | 8 | ||
10 | #include <asm/spitfire.h> | 9 | #include <asm/spitfire.h> |
11 | #include <asm/cpudata.h> | 10 | #include <asm/cpudata.h> |
@@ -14,71 +13,114 @@ | |||
14 | 13 | ||
15 | /* Page table allocation/freeing. */ | 14 | /* Page table allocation/freeing. */ |
16 | 15 | ||
16 | extern struct kmem_cache *pgtable_cache; | ||
17 | |||
17 | static inline pgd_t *pgd_alloc(struct mm_struct *mm) | 18 | static inline pgd_t *pgd_alloc(struct mm_struct *mm) |
18 | { | 19 | { |
19 | return quicklist_alloc(0, GFP_KERNEL, NULL); | 20 | return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); |
20 | } | 21 | } |
21 | 22 | ||
22 | static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) | 23 | static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) |
23 | { | 24 | { |
24 | quicklist_free(0, NULL, pgd); | 25 | kmem_cache_free(pgtable_cache, pgd); |
25 | } | 26 | } |
26 | 27 | ||
27 | #define pud_populate(MM, PUD, PMD) pud_set(PUD, PMD) | 28 | #define pud_populate(MM, PUD, PMD) pud_set(PUD, PMD) |
28 | 29 | ||
29 | static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) | 30 | static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) |
30 | { | 31 | { |
31 | return quicklist_alloc(0, GFP_KERNEL, NULL); | 32 | return kmem_cache_alloc(pgtable_cache, |
33 | GFP_KERNEL|__GFP_REPEAT); | ||
32 | } | 34 | } |
33 | 35 | ||
34 | static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) | 36 | static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) |
35 | { | 37 | { |
36 | quicklist_free(0, NULL, pmd); | 38 | kmem_cache_free(pgtable_cache, pmd); |
37 | } | 39 | } |
38 | 40 | ||
39 | static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, | 41 | static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, |
40 | unsigned long address) | 42 | unsigned long address) |
41 | { | 43 | { |
42 | return quicklist_alloc(0, GFP_KERNEL, NULL); | 44 | return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO); |
43 | } | 45 | } |
44 | 46 | ||
45 | static inline pgtable_t pte_alloc_one(struct mm_struct *mm, | 47 | static inline pgtable_t pte_alloc_one(struct mm_struct *mm, |
46 | unsigned long address) | 48 | unsigned long address) |
47 | { | 49 | { |
48 | struct page *page; | 50 | struct page *page; |
49 | void *pg; | 51 | pte_t *pte; |
50 | 52 | ||
51 | pg = quicklist_alloc(0, GFP_KERNEL, NULL); | 53 | pte = pte_alloc_one_kernel(mm, address); |
52 | if (!pg) | 54 | if (!pte) |
53 | return NULL; | 55 | return NULL; |
54 | page = virt_to_page(pg); | 56 | page = virt_to_page(pte); |
55 | pgtable_page_ctor(page); | 57 | pgtable_page_ctor(page); |
56 | return page; | 58 | return page; |
57 | } | 59 | } |
58 | 60 | ||
59 | static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) | 61 | static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) |
60 | { | 62 | { |
61 | quicklist_free(0, NULL, pte); | 63 | free_page((unsigned long)pte); |
62 | } | 64 | } |
63 | 65 | ||
64 | static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) | 66 | static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) |
65 | { | 67 | { |
66 | pgtable_page_dtor(ptepage); | 68 | pgtable_page_dtor(ptepage); |
67 | quicklist_free_page(0, NULL, ptepage); | 69 | __free_page(ptepage); |
68 | } | 70 | } |
69 | 71 | ||
70 | |||
71 | #define pmd_populate_kernel(MM, PMD, PTE) pmd_set(PMD, PTE) | 72 | #define pmd_populate_kernel(MM, PMD, PTE) pmd_set(PMD, PTE) |
72 | #define pmd_populate(MM,PMD,PTE_PAGE) \ | 73 | #define pmd_populate(MM,PMD,PTE_PAGE) \ |
73 | pmd_populate_kernel(MM,PMD,page_address(PTE_PAGE)) | 74 | pmd_populate_kernel(MM,PMD,page_address(PTE_PAGE)) |
74 | #define pmd_pgtable(pmd) pmd_page(pmd) | 75 | #define pmd_pgtable(pmd) pmd_page(pmd) |
75 | 76 | ||
76 | static inline void check_pgt_cache(void) | 77 | #define check_pgt_cache() do { } while (0) |
78 | |||
79 | static inline void pgtable_free(void *table, bool is_page) | ||
80 | { | ||
81 | if (is_page) | ||
82 | free_page((unsigned long)table); | ||
83 | else | ||
84 | kmem_cache_free(pgtable_cache, table); | ||
85 | } | ||
86 | |||
87 | #ifdef CONFIG_SMP | ||
88 | |||
89 | struct mmu_gather; | ||
90 | extern void tlb_remove_table(struct mmu_gather *, void *); | ||
91 | |||
92 | static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) | ||
93 | { | ||
94 | unsigned long pgf = (unsigned long)table; | ||
95 | if (is_page) | ||
96 | pgf |= 0x1UL; | ||
97 | tlb_remove_table(tlb, (void *)pgf); | ||
98 | } | ||
99 | |||
100 | static inline void __tlb_remove_table(void *_table) | ||
101 | { | ||
102 | void *table = (void *)((unsigned long)_table & ~0x1UL); | ||
103 | bool is_page = false; | ||
104 | |||
105 | if ((unsigned long)_table & 0x1UL) | ||
106 | is_page = true; | ||
107 | pgtable_free(table, is_page); | ||
108 | } | ||
109 | #else /* CONFIG_SMP */ | ||
110 | static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) | ||
77 | { | 111 | { |
78 | quicklist_trim(0, NULL, 25, 16); | 112 | pgtable_free(table, is_page); |
113 | } | ||
114 | #endif /* !CONFIG_SMP */ | ||
115 | |||
116 | static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *ptepage, | ||
117 | unsigned long address) | ||
118 | { | ||
119 | pgtable_page_dtor(ptepage); | ||
120 | pgtable_free_tlb(tlb, page_address(ptepage), true); | ||
79 | } | 121 | } |
80 | 122 | ||
81 | #define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) | 123 | #define __pmd_free_tlb(tlb, pmd, addr) \ |
82 | #define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) | 124 | pgtable_free_tlb(tlb, pmd, false) |
83 | 125 | ||
84 | #endif /* _SPARC64_PGALLOC_H */ | 126 | #endif /* _SPARC64_PGALLOC_H */ |
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index 1e03c5a6b4f7..adf89329af59 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h | |||
@@ -95,6 +95,10 @@ | |||
95 | /* PTE bits which are the same in SUN4U and SUN4V format. */ | 95 | /* PTE bits which are the same in SUN4U and SUN4V format. */ |
96 | #define _PAGE_VALID _AC(0x8000000000000000,UL) /* Valid TTE */ | 96 | #define _PAGE_VALID _AC(0x8000000000000000,UL) /* Valid TTE */ |
97 | #define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/ | 97 | #define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/ |
98 | #define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */ | ||
99 | |||
100 | /* Advertise support for _PAGE_SPECIAL */ | ||
101 | #define __HAVE_ARCH_PTE_SPECIAL | ||
98 | 102 | ||
99 | /* SUN4U pte bits... */ | 103 | /* SUN4U pte bits... */ |
100 | #define _PAGE_SZ4MB_4U _AC(0x6000000000000000,UL) /* 4MB Page */ | 104 | #define _PAGE_SZ4MB_4U _AC(0x6000000000000000,UL) /* 4MB Page */ |
@@ -104,6 +108,7 @@ | |||
104 | #define _PAGE_NFO_4U _AC(0x1000000000000000,UL) /* No Fault Only */ | 108 | #define _PAGE_NFO_4U _AC(0x1000000000000000,UL) /* No Fault Only */ |
105 | #define _PAGE_IE_4U _AC(0x0800000000000000,UL) /* Invert Endianness */ | 109 | #define _PAGE_IE_4U _AC(0x0800000000000000,UL) /* Invert Endianness */ |
106 | #define _PAGE_SOFT2_4U _AC(0x07FC000000000000,UL) /* Software bits, set 2 */ | 110 | #define _PAGE_SOFT2_4U _AC(0x07FC000000000000,UL) /* Software bits, set 2 */ |
111 | #define _PAGE_SPECIAL_4U _AC(0x0200000000000000,UL) /* Special page */ | ||
107 | #define _PAGE_RES1_4U _AC(0x0002000000000000,UL) /* Reserved */ | 112 | #define _PAGE_RES1_4U _AC(0x0002000000000000,UL) /* Reserved */ |
108 | #define _PAGE_SZ32MB_4U _AC(0x0001000000000000,UL) /* (Panther) 32MB page */ | 113 | #define _PAGE_SZ32MB_4U _AC(0x0001000000000000,UL) /* (Panther) 32MB page */ |
109 | #define _PAGE_SZ256MB_4U _AC(0x2001000000000000,UL) /* (Panther) 256MB page */ | 114 | #define _PAGE_SZ256MB_4U _AC(0x2001000000000000,UL) /* (Panther) 256MB page */ |
@@ -133,6 +138,7 @@ | |||
133 | #define _PAGE_ACCESSED_4V _AC(0x1000000000000000,UL) /* Accessed (ref'd) */ | 138 | #define _PAGE_ACCESSED_4V _AC(0x1000000000000000,UL) /* Accessed (ref'd) */ |
134 | #define _PAGE_READ_4V _AC(0x0800000000000000,UL) /* Readable SW Bit */ | 139 | #define _PAGE_READ_4V _AC(0x0800000000000000,UL) /* Readable SW Bit */ |
135 | #define _PAGE_WRITE_4V _AC(0x0400000000000000,UL) /* Writable SW Bit */ | 140 | #define _PAGE_WRITE_4V _AC(0x0400000000000000,UL) /* Writable SW Bit */ |
141 | #define _PAGE_SPECIAL_4V _AC(0x0200000000000000,UL) /* Special page */ | ||
136 | #define _PAGE_PADDR_4V _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13] */ | 142 | #define _PAGE_PADDR_4V _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13] */ |
137 | #define _PAGE_IE_4V _AC(0x0000000000001000,UL) /* Invert Endianness */ | 143 | #define _PAGE_IE_4V _AC(0x0000000000001000,UL) /* Invert Endianness */ |
138 | #define _PAGE_E_4V _AC(0x0000000000000800,UL) /* side-Effect */ | 144 | #define _PAGE_E_4V _AC(0x0000000000000800,UL) /* side-Effect */ |
@@ -302,10 +308,10 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t prot) | |||
302 | : "=r" (mask), "=r" (tmp) | 308 | : "=r" (mask), "=r" (tmp) |
303 | : "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U | | 309 | : "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U | |
304 | _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U | _PAGE_PRESENT_4U | | 310 | _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U | _PAGE_PRESENT_4U | |
305 | _PAGE_SZBITS_4U), | 311 | _PAGE_SZBITS_4U | _PAGE_SPECIAL), |
306 | "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V | | 312 | "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V | |
307 | _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V | _PAGE_PRESENT_4V | | 313 | _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V | _PAGE_PRESENT_4V | |
308 | _PAGE_SZBITS_4V)); | 314 | _PAGE_SZBITS_4V | _PAGE_SPECIAL)); |
309 | 315 | ||
310 | return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask)); | 316 | return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask)); |
311 | } | 317 | } |
@@ -502,6 +508,7 @@ static inline pte_t pte_mkyoung(pte_t pte) | |||
502 | 508 | ||
503 | static inline pte_t pte_mkspecial(pte_t pte) | 509 | static inline pte_t pte_mkspecial(pte_t pte) |
504 | { | 510 | { |
511 | pte_val(pte) |= _PAGE_SPECIAL; | ||
505 | return pte; | 512 | return pte; |
506 | } | 513 | } |
507 | 514 | ||
@@ -607,9 +614,9 @@ static inline unsigned long pte_present(pte_t pte) | |||
607 | return val; | 614 | return val; |
608 | } | 615 | } |
609 | 616 | ||
610 | static inline int pte_special(pte_t pte) | 617 | static inline unsigned long pte_special(pte_t pte) |
611 | { | 618 | { |
612 | return 0; | 619 | return pte_val(pte) & _PAGE_SPECIAL; |
613 | } | 620 | } |
614 | 621 | ||
615 | #define pmd_set(pmdp, ptep) \ | 622 | #define pmd_set(pmdp, ptep) \ |
diff --git a/arch/sparc/mm/Makefile b/arch/sparc/mm/Makefile index 79836a7dd00c..e3cda21b5ee9 100644 --- a/arch/sparc/mm/Makefile +++ b/arch/sparc/mm/Makefile | |||
@@ -4,7 +4,7 @@ | |||
4 | asflags-y := -ansi | 4 | asflags-y := -ansi |
5 | ccflags-y := -Werror | 5 | ccflags-y := -Werror |
6 | 6 | ||
7 | obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o | 7 | obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o gup.o |
8 | obj-y += fault_$(BITS).o | 8 | obj-y += fault_$(BITS).o |
9 | obj-y += init_$(BITS).o | 9 | obj-y += init_$(BITS).o |
10 | obj-$(CONFIG_SPARC32) += loadmmu.o | 10 | obj-$(CONFIG_SPARC32) += loadmmu.o |
diff --git a/arch/sparc/mm/gup.c b/arch/sparc/mm/gup.c new file mode 100644 index 000000000000..a986b5d05712 --- /dev/null +++ b/arch/sparc/mm/gup.c | |||
@@ -0,0 +1,181 @@ | |||
1 | /* | ||
2 | * Lockless get_user_pages_fast for sparc, cribbed from powerpc | ||
3 | * | ||
4 | * Copyright (C) 2008 Nick Piggin | ||
5 | * Copyright (C) 2008 Novell Inc. | ||
6 | */ | ||
7 | |||
8 | #include <linux/sched.h> | ||
9 | #include <linux/mm.h> | ||
10 | #include <linux/vmstat.h> | ||
11 | #include <linux/pagemap.h> | ||
12 | #include <linux/rwsem.h> | ||
13 | #include <asm/pgtable.h> | ||
14 | |||
15 | /* | ||
16 | * The performance critical leaf functions are made noinline otherwise gcc | ||
17 | * inlines everything into a single function which results in too much | ||
18 | * register pressure. | ||
19 | */ | ||
20 | static noinline int gup_pte_range(pmd_t pmd, unsigned long addr, | ||
21 | unsigned long end, int write, struct page **pages, int *nr) | ||
22 | { | ||
23 | unsigned long mask, result; | ||
24 | pte_t *ptep; | ||
25 | |||
26 | if (tlb_type == hypervisor) { | ||
27 | result = _PAGE_PRESENT_4V|_PAGE_P_4V; | ||
28 | if (write) | ||
29 | result |= _PAGE_WRITE_4V; | ||
30 | } else { | ||
31 | result = _PAGE_PRESENT_4U|_PAGE_P_4U; | ||
32 | if (write) | ||
33 | result |= _PAGE_WRITE_4U; | ||
34 | } | ||
35 | mask = result | _PAGE_SPECIAL; | ||
36 | |||
37 | ptep = pte_offset_kernel(&pmd, addr); | ||
38 | do { | ||
39 | struct page *page, *head; | ||
40 | pte_t pte = *ptep; | ||
41 | |||
42 | if ((pte_val(pte) & mask) != result) | ||
43 | return 0; | ||
44 | VM_BUG_ON(!pfn_valid(pte_pfn(pte))); | ||
45 | |||
46 | /* The hugepage case is simplified on sparc64 because | ||
47 | * we encode the sub-page pfn offsets into the | ||
48 | * hugepage PTEs. We could optimize this in the future | ||
49 | * use page_cache_add_speculative() for the hugepage case. | ||
50 | */ | ||
51 | page = pte_page(pte); | ||
52 | head = compound_head(page); | ||
53 | if (!page_cache_get_speculative(head)) | ||
54 | return 0; | ||
55 | if (unlikely(pte_val(pte) != pte_val(*ptep))) { | ||
56 | put_page(head); | ||
57 | return 0; | ||
58 | } | ||
59 | |||
60 | pages[*nr] = page; | ||
61 | (*nr)++; | ||
62 | } while (ptep++, addr += PAGE_SIZE, addr != end); | ||
63 | |||
64 | return 1; | ||
65 | } | ||
66 | |||
67 | static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end, | ||
68 | int write, struct page **pages, int *nr) | ||
69 | { | ||
70 | unsigned long next; | ||
71 | pmd_t *pmdp; | ||
72 | |||
73 | pmdp = pmd_offset(&pud, addr); | ||
74 | do { | ||
75 | pmd_t pmd = *pmdp; | ||
76 | |||
77 | next = pmd_addr_end(addr, end); | ||
78 | if (pmd_none(pmd)) | ||
79 | return 0; | ||
80 | if (!gup_pte_range(pmd, addr, next, write, pages, nr)) | ||
81 | return 0; | ||
82 | } while (pmdp++, addr = next, addr != end); | ||
83 | |||
84 | return 1; | ||
85 | } | ||
86 | |||
87 | static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end, | ||
88 | int write, struct page **pages, int *nr) | ||
89 | { | ||
90 | unsigned long next; | ||
91 | pud_t *pudp; | ||
92 | |||
93 | pudp = pud_offset(&pgd, addr); | ||
94 | do { | ||
95 | pud_t pud = *pudp; | ||
96 | |||
97 | next = pud_addr_end(addr, end); | ||
98 | if (pud_none(pud)) | ||
99 | return 0; | ||
100 | if (!gup_pmd_range(pud, addr, next, write, pages, nr)) | ||
101 | return 0; | ||
102 | } while (pudp++, addr = next, addr != end); | ||
103 | |||
104 | return 1; | ||
105 | } | ||
106 | |||
107 | int get_user_pages_fast(unsigned long start, int nr_pages, int write, | ||
108 | struct page **pages) | ||
109 | { | ||
110 | struct mm_struct *mm = current->mm; | ||
111 | unsigned long addr, len, end; | ||
112 | unsigned long next; | ||
113 | pgd_t *pgdp; | ||
114 | int nr = 0; | ||
115 | |||
116 | start &= PAGE_MASK; | ||
117 | addr = start; | ||
118 | len = (unsigned long) nr_pages << PAGE_SHIFT; | ||
119 | end = start + len; | ||
120 | |||
121 | /* | ||
122 | * XXX: batch / limit 'nr', to avoid large irq off latency | ||
123 | * needs some instrumenting to determine the common sizes used by | ||
124 | * important workloads (eg. DB2), and whether limiting the batch size | ||
125 | * will decrease performance. | ||
126 | * | ||
127 | * It seems like we're in the clear for the moment. Direct-IO is | ||
128 | * the main guy that batches up lots of get_user_pages, and even | ||
129 | * they are limited to 64-at-a-time which is not so many. | ||
130 | */ | ||
131 | /* | ||
132 | * This doesn't prevent pagetable teardown, but does prevent | ||
133 | * the pagetables from being freed on sparc. | ||
134 | * | ||
135 | * So long as we atomically load page table pointers versus teardown, | ||
136 | * we can follow the address down to the the page and take a ref on it. | ||
137 | */ | ||
138 | local_irq_disable(); | ||
139 | |||
140 | pgdp = pgd_offset(mm, addr); | ||
141 | do { | ||
142 | pgd_t pgd = *pgdp; | ||
143 | |||
144 | next = pgd_addr_end(addr, end); | ||
145 | if (pgd_none(pgd)) | ||
146 | goto slow; | ||
147 | if (!gup_pud_range(pgd, addr, next, write, pages, &nr)) | ||
148 | goto slow; | ||
149 | } while (pgdp++, addr = next, addr != end); | ||
150 | |||
151 | local_irq_enable(); | ||
152 | |||
153 | VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT); | ||
154 | return nr; | ||
155 | |||
156 | { | ||
157 | int ret; | ||
158 | |||
159 | slow: | ||
160 | local_irq_enable(); | ||
161 | |||
162 | /* Try to get the remaining pages with get_user_pages */ | ||
163 | start += nr << PAGE_SHIFT; | ||
164 | pages += nr; | ||
165 | |||
166 | down_read(&mm->mmap_sem); | ||
167 | ret = get_user_pages(current, mm, start, | ||
168 | (end - start) >> PAGE_SHIFT, write, 0, pages, NULL); | ||
169 | up_read(&mm->mmap_sem); | ||
170 | |||
171 | /* Have to be a bit careful with return values */ | ||
172 | if (nr > 0) { | ||
173 | if (ret < 0) | ||
174 | ret = nr; | ||
175 | else | ||
176 | ret += nr; | ||
177 | } | ||
178 | |||
179 | return ret; | ||
180 | } | ||
181 | } | ||
diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c index a5f51b22fcbe..536412d8f416 100644 --- a/arch/sparc/mm/tsb.c +++ b/arch/sparc/mm/tsb.c | |||
@@ -236,6 +236,8 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign | |||
236 | } | 236 | } |
237 | } | 237 | } |
238 | 238 | ||
239 | struct kmem_cache *pgtable_cache __read_mostly; | ||
240 | |||
239 | static struct kmem_cache *tsb_caches[8] __read_mostly; | 241 | static struct kmem_cache *tsb_caches[8] __read_mostly; |
240 | 242 | ||
241 | static const char *tsb_cache_names[8] = { | 243 | static const char *tsb_cache_names[8] = { |
@@ -253,6 +255,15 @@ void __init pgtable_cache_init(void) | |||
253 | { | 255 | { |
254 | unsigned long i; | 256 | unsigned long i; |
255 | 257 | ||
258 | pgtable_cache = kmem_cache_create("pgtable_cache", | ||
259 | PAGE_SIZE, PAGE_SIZE, | ||
260 | 0, | ||
261 | _clear_page); | ||
262 | if (!pgtable_cache) { | ||
263 | prom_printf("pgtable_cache_init(): Could not create!\n"); | ||
264 | prom_halt(); | ||
265 | } | ||
266 | |||
256 | for (i = 0; i < 8; i++) { | 267 | for (i = 0; i < 8; i++) { |
257 | unsigned long size = 8192 << i; | 268 | unsigned long size = 8192 << i; |
258 | const char *name = tsb_cache_names[i]; | 269 | const char *name = tsb_cache_names[i]; |