diff options
-rw-r--r-- | arch/i386/kernel/paravirt.c | 84 | ||||
-rw-r--r-- | arch/i386/kernel/vmi.c | 6 | ||||
-rw-r--r-- | include/asm-i386/page.h | 79 | ||||
-rw-r--r-- | include/asm-i386/paravirt.h | 52 | ||||
-rw-r--r-- | include/asm-i386/pgtable-2level.h | 26 | ||||
-rw-r--r-- | include/asm-i386/pgtable-3level.h | 63 | ||||
-rw-r--r-- | include/asm-i386/pgtable.h | 2 |
7 files changed, 184 insertions, 128 deletions
diff --git a/arch/i386/kernel/paravirt.c b/arch/i386/kernel/paravirt.c index 3fdbd1f62379..cba7a15ce1b0 100644 --- a/arch/i386/kernel/paravirt.c +++ b/arch/i386/kernel/paravirt.c | |||
@@ -117,78 +117,6 @@ static void native_flush_tlb_single(u32 addr) | |||
117 | __native_flush_tlb_single(addr); | 117 | __native_flush_tlb_single(addr); |
118 | } | 118 | } |
119 | 119 | ||
120 | #ifndef CONFIG_X86_PAE | ||
121 | static void native_set_pte(pte_t *ptep, pte_t pteval) | ||
122 | { | ||
123 | *ptep = pteval; | ||
124 | } | ||
125 | |||
126 | static void native_set_pte_at(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pteval) | ||
127 | { | ||
128 | *ptep = pteval; | ||
129 | } | ||
130 | |||
131 | static void native_set_pmd(pmd_t *pmdp, pmd_t pmdval) | ||
132 | { | ||
133 | *pmdp = pmdval; | ||
134 | } | ||
135 | |||
136 | #else /* CONFIG_X86_PAE */ | ||
137 | |||
138 | static void native_set_pte(pte_t *ptep, pte_t pte) | ||
139 | { | ||
140 | ptep->pte_high = pte.pte_high; | ||
141 | smp_wmb(); | ||
142 | ptep->pte_low = pte.pte_low; | ||
143 | } | ||
144 | |||
145 | static void native_set_pte_at(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pte) | ||
146 | { | ||
147 | ptep->pte_high = pte.pte_high; | ||
148 | smp_wmb(); | ||
149 | ptep->pte_low = pte.pte_low; | ||
150 | } | ||
151 | |||
152 | static void native_set_pte_present(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) | ||
153 | { | ||
154 | ptep->pte_low = 0; | ||
155 | smp_wmb(); | ||
156 | ptep->pte_high = pte.pte_high; | ||
157 | smp_wmb(); | ||
158 | ptep->pte_low = pte.pte_low; | ||
159 | } | ||
160 | |||
161 | static void native_set_pte_atomic(pte_t *ptep, pte_t pteval) | ||
162 | { | ||
163 | set_64bit((unsigned long long *)ptep,pte_val(pteval)); | ||
164 | } | ||
165 | |||
166 | static void native_set_pmd(pmd_t *pmdp, pmd_t pmdval) | ||
167 | { | ||
168 | set_64bit((unsigned long long *)pmdp,pmd_val(pmdval)); | ||
169 | } | ||
170 | |||
171 | static void native_set_pud(pud_t *pudp, pud_t pudval) | ||
172 | { | ||
173 | *pudp = pudval; | ||
174 | } | ||
175 | |||
176 | static void native_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | ||
177 | { | ||
178 | ptep->pte_low = 0; | ||
179 | smp_wmb(); | ||
180 | ptep->pte_high = 0; | ||
181 | } | ||
182 | |||
183 | static void native_pmd_clear(pmd_t *pmd) | ||
184 | { | ||
185 | u32 *tmp = (u32 *)pmd; | ||
186 | *tmp = 0; | ||
187 | smp_wmb(); | ||
188 | *(tmp + 1) = 0; | ||
189 | } | ||
190 | #endif /* CONFIG_X86_PAE */ | ||
191 | |||
192 | /* These are in entry.S */ | 120 | /* These are in entry.S */ |
193 | extern void native_iret(void); | 121 | extern void native_iret(void); |
194 | extern void native_irq_enable_sysexit(void); | 122 | extern void native_irq_enable_sysexit(void); |
@@ -282,14 +210,26 @@ struct paravirt_ops paravirt_ops = { | |||
282 | .set_pmd = native_set_pmd, | 210 | .set_pmd = native_set_pmd, |
283 | .pte_update = paravirt_nop, | 211 | .pte_update = paravirt_nop, |
284 | .pte_update_defer = paravirt_nop, | 212 | .pte_update_defer = paravirt_nop, |
213 | |||
214 | .ptep_get_and_clear = native_ptep_get_and_clear, | ||
215 | |||
285 | #ifdef CONFIG_X86_PAE | 216 | #ifdef CONFIG_X86_PAE |
286 | .set_pte_atomic = native_set_pte_atomic, | 217 | .set_pte_atomic = native_set_pte_atomic, |
287 | .set_pte_present = native_set_pte_present, | 218 | .set_pte_present = native_set_pte_present, |
288 | .set_pud = native_set_pud, | 219 | .set_pud = native_set_pud, |
289 | .pte_clear = native_pte_clear, | 220 | .pte_clear = native_pte_clear, |
290 | .pmd_clear = native_pmd_clear, | 221 | .pmd_clear = native_pmd_clear, |
222 | |||
223 | .pmd_val = native_pmd_val, | ||
224 | .make_pmd = native_make_pmd, | ||
291 | #endif | 225 | #endif |
292 | 226 | ||
227 | .pte_val = native_pte_val, | ||
228 | .pgd_val = native_pgd_val, | ||
229 | |||
230 | .make_pte = native_make_pte, | ||
231 | .make_pgd = native_make_pgd, | ||
232 | |||
293 | .irq_enable_sysexit = native_irq_enable_sysexit, | 233 | .irq_enable_sysexit = native_irq_enable_sysexit, |
294 | .iret = native_iret, | 234 | .iret = native_iret, |
295 | 235 | ||
diff --git a/arch/i386/kernel/vmi.c b/arch/i386/kernel/vmi.c index 8f3bac473450..ea77d93f59dd 100644 --- a/arch/i386/kernel/vmi.c +++ b/arch/i386/kernel/vmi.c | |||
@@ -443,13 +443,13 @@ static void vmi_release_pd(u32 pfn) | |||
443 | ((level) | (is_current_as(mm, user) ? \ | 443 | ((level) | (is_current_as(mm, user) ? \ |
444 | (VMI_PAGE_DEFER | VMI_PAGE_CURRENT_AS | ((addr) & VMI_PAGE_VA_MASK)) : 0)) | 444 | (VMI_PAGE_DEFER | VMI_PAGE_CURRENT_AS | ((addr) & VMI_PAGE_VA_MASK)) : 0)) |
445 | 445 | ||
446 | static void vmi_update_pte(struct mm_struct *mm, u32 addr, pte_t *ptep) | 446 | static void vmi_update_pte(struct mm_struct *mm, unsigned long addr, pte_t *ptep) |
447 | { | 447 | { |
448 | vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE); | 448 | vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE); |
449 | vmi_ops.update_pte(ptep, vmi_flags_addr(mm, addr, VMI_PAGE_PT, 0)); | 449 | vmi_ops.update_pte(ptep, vmi_flags_addr(mm, addr, VMI_PAGE_PT, 0)); |
450 | } | 450 | } |
451 | 451 | ||
452 | static void vmi_update_pte_defer(struct mm_struct *mm, u32 addr, pte_t *ptep) | 452 | static void vmi_update_pte_defer(struct mm_struct *mm, unsigned long addr, pte_t *ptep) |
453 | { | 453 | { |
454 | vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE); | 454 | vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE); |
455 | vmi_ops.update_pte(ptep, vmi_flags_addr_defer(mm, addr, VMI_PAGE_PT, 0)); | 455 | vmi_ops.update_pte(ptep, vmi_flags_addr_defer(mm, addr, VMI_PAGE_PT, 0)); |
@@ -462,7 +462,7 @@ static void vmi_set_pte(pte_t *ptep, pte_t pte) | |||
462 | vmi_ops.set_pte(pte, ptep, VMI_PAGE_PT); | 462 | vmi_ops.set_pte(pte, ptep, VMI_PAGE_PT); |
463 | } | 463 | } |
464 | 464 | ||
465 | static void vmi_set_pte_at(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pte) | 465 | static void vmi_set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) |
466 | { | 466 | { |
467 | vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE); | 467 | vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE); |
468 | vmi_ops.set_pte(pte, ptep, vmi_flags_addr(mm, addr, VMI_PAGE_PT, 0)); | 468 | vmi_ops.set_pte(pte, ptep, vmi_flags_addr(mm, addr, VMI_PAGE_PT, 0)); |
diff --git a/include/asm-i386/page.h b/include/asm-i386/page.h index fd3f64ace248..818ac8bf01e2 100644 --- a/include/asm-i386/page.h +++ b/include/asm-i386/page.h | |||
@@ -12,7 +12,6 @@ | |||
12 | #ifdef __KERNEL__ | 12 | #ifdef __KERNEL__ |
13 | #ifndef __ASSEMBLY__ | 13 | #ifndef __ASSEMBLY__ |
14 | 14 | ||
15 | |||
16 | #ifdef CONFIG_X86_USE_3DNOW | 15 | #ifdef CONFIG_X86_USE_3DNOW |
17 | 16 | ||
18 | #include <asm/mmx.h> | 17 | #include <asm/mmx.h> |
@@ -42,26 +41,81 @@ | |||
42 | * These are used to make use of C type-checking.. | 41 | * These are used to make use of C type-checking.. |
43 | */ | 42 | */ |
44 | extern int nx_enabled; | 43 | extern int nx_enabled; |
44 | |||
45 | #ifdef CONFIG_X86_PAE | 45 | #ifdef CONFIG_X86_PAE |
46 | extern unsigned long long __supported_pte_mask; | 46 | extern unsigned long long __supported_pte_mask; |
47 | typedef struct { unsigned long pte_low, pte_high; } pte_t; | 47 | typedef struct { unsigned long pte_low, pte_high; } pte_t; |
48 | typedef struct { unsigned long long pmd; } pmd_t; | 48 | typedef struct { unsigned long long pmd; } pmd_t; |
49 | typedef struct { unsigned long long pgd; } pgd_t; | 49 | typedef struct { unsigned long long pgd; } pgd_t; |
50 | typedef struct { unsigned long long pgprot; } pgprot_t; | 50 | typedef struct { unsigned long long pgprot; } pgprot_t; |
51 | #define pmd_val(x) ((x).pmd) | 51 | |
52 | #define pte_val(x) ((x).pte_low | ((unsigned long long)(x).pte_high << 32)) | 52 | static inline unsigned long long native_pgd_val(pgd_t pgd) |
53 | #define __pmd(x) ((pmd_t) { (x) } ) | 53 | { |
54 | return pgd.pgd; | ||
55 | } | ||
56 | |||
57 | static inline unsigned long long native_pmd_val(pmd_t pmd) | ||
58 | { | ||
59 | return pmd.pmd; | ||
60 | } | ||
61 | |||
62 | static inline unsigned long long native_pte_val(pte_t pte) | ||
63 | { | ||
64 | return pte.pte_low | ((unsigned long long)pte.pte_high << 32); | ||
65 | } | ||
66 | |||
67 | static inline pgd_t native_make_pgd(unsigned long long val) | ||
68 | { | ||
69 | return (pgd_t) { val }; | ||
70 | } | ||
71 | |||
72 | static inline pmd_t native_make_pmd(unsigned long long val) | ||
73 | { | ||
74 | return (pmd_t) { val }; | ||
75 | } | ||
76 | |||
77 | static inline pte_t native_make_pte(unsigned long long val) | ||
78 | { | ||
79 | return (pte_t) { .pte_low = val, .pte_high = (val >> 32) } ; | ||
80 | } | ||
81 | |||
82 | #ifndef CONFIG_PARAVIRT | ||
83 | #define pmd_val(x) native_pmd_val(x) | ||
84 | #define __pmd(x) native_make_pmd(x) | ||
85 | #endif | ||
86 | |||
54 | #define HPAGE_SHIFT 21 | 87 | #define HPAGE_SHIFT 21 |
55 | #include <asm-generic/pgtable-nopud.h> | 88 | #include <asm-generic/pgtable-nopud.h> |
56 | #else | 89 | #else /* !CONFIG_X86_PAE */ |
57 | typedef struct { unsigned long pte_low; } pte_t; | 90 | typedef struct { unsigned long pte_low; } pte_t; |
58 | typedef struct { unsigned long pgd; } pgd_t; | 91 | typedef struct { unsigned long pgd; } pgd_t; |
59 | typedef struct { unsigned long pgprot; } pgprot_t; | 92 | typedef struct { unsigned long pgprot; } pgprot_t; |
60 | #define boot_pte_t pte_t /* or would you rather have a typedef */ | 93 | #define boot_pte_t pte_t /* or would you rather have a typedef */ |
61 | #define pte_val(x) ((x).pte_low) | 94 | |
95 | static inline unsigned long native_pgd_val(pgd_t pgd) | ||
96 | { | ||
97 | return pgd.pgd; | ||
98 | } | ||
99 | |||
100 | static inline unsigned long native_pte_val(pte_t pte) | ||
101 | { | ||
102 | return pte.pte_low; | ||
103 | } | ||
104 | |||
105 | static inline pgd_t native_make_pgd(unsigned long val) | ||
106 | { | ||
107 | return (pgd_t) { val }; | ||
108 | } | ||
109 | |||
110 | static inline pte_t native_make_pte(unsigned long val) | ||
111 | { | ||
112 | return (pte_t) { .pte_low = val }; | ||
113 | } | ||
114 | |||
62 | #define HPAGE_SHIFT 22 | 115 | #define HPAGE_SHIFT 22 |
63 | #include <asm-generic/pgtable-nopmd.h> | 116 | #include <asm-generic/pgtable-nopmd.h> |
64 | #endif | 117 | #endif /* CONFIG_X86_PAE */ |
118 | |||
65 | #define PTE_MASK PAGE_MASK | 119 | #define PTE_MASK PAGE_MASK |
66 | 120 | ||
67 | #ifdef CONFIG_HUGETLB_PAGE | 121 | #ifdef CONFIG_HUGETLB_PAGE |
@@ -71,13 +125,16 @@ typedef struct { unsigned long pgprot; } pgprot_t; | |||
71 | #define HAVE_ARCH_HUGETLB_UNMAPPED_AREA | 125 | #define HAVE_ARCH_HUGETLB_UNMAPPED_AREA |
72 | #endif | 126 | #endif |
73 | 127 | ||
74 | #define pgd_val(x) ((x).pgd) | ||
75 | #define pgprot_val(x) ((x).pgprot) | 128 | #define pgprot_val(x) ((x).pgprot) |
76 | |||
77 | #define __pte(x) ((pte_t) { (x) } ) | ||
78 | #define __pgd(x) ((pgd_t) { (x) } ) | ||
79 | #define __pgprot(x) ((pgprot_t) { (x) } ) | 129 | #define __pgprot(x) ((pgprot_t) { (x) } ) |
80 | 130 | ||
131 | #ifndef CONFIG_PARAVIRT | ||
132 | #define pgd_val(x) native_pgd_val(x) | ||
133 | #define __pgd(x) native_make_pgd(x) | ||
134 | #define pte_val(x) native_pte_val(x) | ||
135 | #define __pte(x) native_make_pte(x) | ||
136 | #endif | ||
137 | |||
81 | #endif /* !__ASSEMBLY__ */ | 138 | #endif /* !__ASSEMBLY__ */ |
82 | 139 | ||
83 | /* to align the pointer to the (next) page boundary */ | 140 | /* to align the pointer to the (next) page boundary */ |
diff --git a/include/asm-i386/paravirt.h b/include/asm-i386/paravirt.h index f0bdaea6235d..0aacb13bb929 100644 --- a/include/asm-i386/paravirt.h +++ b/include/asm-i386/paravirt.h | |||
@@ -2,7 +2,6 @@ | |||
2 | #define __ASM_PARAVIRT_H | 2 | #define __ASM_PARAVIRT_H |
3 | /* Various instructions on x86 need to be replaced for | 3 | /* Various instructions on x86 need to be replaced for |
4 | * para-virtualization: those hooks are defined here. */ | 4 | * para-virtualization: those hooks are defined here. */ |
5 | #include <linux/linkage.h> | ||
6 | #include <linux/stringify.h> | 5 | #include <linux/stringify.h> |
7 | #include <asm/page.h> | 6 | #include <asm/page.h> |
8 | 7 | ||
@@ -25,6 +24,8 @@ | |||
25 | #define CLBR_ANY 0x7 | 24 | #define CLBR_ANY 0x7 |
26 | 25 | ||
27 | #ifndef __ASSEMBLY__ | 26 | #ifndef __ASSEMBLY__ |
27 | #include <linux/types.h> | ||
28 | |||
28 | struct thread_struct; | 29 | struct thread_struct; |
29 | struct Xgt_desc_struct; | 30 | struct Xgt_desc_struct; |
30 | struct tss_struct; | 31 | struct tss_struct; |
@@ -55,11 +56,6 @@ struct paravirt_ops | |||
55 | int (*set_wallclock)(unsigned long); | 56 | int (*set_wallclock)(unsigned long); |
56 | void (*time_init)(void); | 57 | void (*time_init)(void); |
57 | 58 | ||
58 | /* All the function pointers here are declared as "fastcall" | ||
59 | so that we get a specific register-based calling | ||
60 | convention. This makes it easier to implement inline | ||
61 | assembler replacements. */ | ||
62 | |||
63 | void (*cpuid)(unsigned int *eax, unsigned int *ebx, | 59 | void (*cpuid)(unsigned int *eax, unsigned int *ebx, |
64 | unsigned int *ecx, unsigned int *edx); | 60 | unsigned int *ecx, unsigned int *edx); |
65 | 61 | ||
@@ -139,16 +135,33 @@ struct paravirt_ops | |||
139 | void (*release_pd)(u32 pfn); | 135 | void (*release_pd)(u32 pfn); |
140 | 136 | ||
141 | void (*set_pte)(pte_t *ptep, pte_t pteval); | 137 | void (*set_pte)(pte_t *ptep, pte_t pteval); |
142 | void (*set_pte_at)(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pteval); | 138 | void (*set_pte_at)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pteval); |
143 | void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval); | 139 | void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval); |
144 | void (*pte_update)(struct mm_struct *mm, u32 addr, pte_t *ptep); | 140 | void (*pte_update)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); |
145 | void (*pte_update_defer)(struct mm_struct *mm, u32 addr, pte_t *ptep); | 141 | void (*pte_update_defer)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); |
142 | |||
143 | pte_t (*ptep_get_and_clear)(pte_t *ptep); | ||
144 | |||
146 | #ifdef CONFIG_X86_PAE | 145 | #ifdef CONFIG_X86_PAE |
147 | void (*set_pte_atomic)(pte_t *ptep, pte_t pteval); | 146 | void (*set_pte_atomic)(pte_t *ptep, pte_t pteval); |
148 | void (*set_pte_present)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte); | 147 | void (*set_pte_present)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte); |
149 | void (*set_pud)(pud_t *pudp, pud_t pudval); | 148 | void (*set_pud)(pud_t *pudp, pud_t pudval); |
150 | void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); | 149 | void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); |
151 | void (*pmd_clear)(pmd_t *pmdp); | 150 | void (*pmd_clear)(pmd_t *pmdp); |
151 | |||
152 | unsigned long long (*pte_val)(pte_t); | ||
153 | unsigned long long (*pmd_val)(pmd_t); | ||
154 | unsigned long long (*pgd_val)(pgd_t); | ||
155 | |||
156 | pte_t (*make_pte)(unsigned long long pte); | ||
157 | pmd_t (*make_pmd)(unsigned long long pmd); | ||
158 | pgd_t (*make_pgd)(unsigned long long pgd); | ||
159 | #else | ||
160 | unsigned long (*pte_val)(pte_t); | ||
161 | unsigned long (*pgd_val)(pgd_t); | ||
162 | |||
163 | pte_t (*make_pte)(unsigned long pte); | ||
164 | pgd_t (*make_pgd)(unsigned long pgd); | ||
152 | #endif | 165 | #endif |
153 | 166 | ||
154 | void (*set_lazy_mode)(int mode); | 167 | void (*set_lazy_mode)(int mode); |
@@ -219,6 +232,8 @@ static inline void __cpuid(unsigned int *eax, unsigned int *ebx, | |||
219 | #define read_cr4_safe(x) paravirt_ops.read_cr4_safe() | 232 | #define read_cr4_safe(x) paravirt_ops.read_cr4_safe() |
220 | #define write_cr4(x) paravirt_ops.write_cr4(x) | 233 | #define write_cr4(x) paravirt_ops.write_cr4(x) |
221 | 234 | ||
235 | #define raw_ptep_get_and_clear(xp) (paravirt_ops.ptep_get_and_clear(xp)) | ||
236 | |||
222 | static inline void raw_safe_halt(void) | 237 | static inline void raw_safe_halt(void) |
223 | { | 238 | { |
224 | paravirt_ops.safe_halt(); | 239 | paravirt_ops.safe_halt(); |
@@ -304,6 +319,17 @@ static inline void halt(void) | |||
304 | (paravirt_ops.write_idt_entry((dt), (entry), (low), (high))) | 319 | (paravirt_ops.write_idt_entry((dt), (entry), (low), (high))) |
305 | #define set_iopl_mask(mask) (paravirt_ops.set_iopl_mask(mask)) | 320 | #define set_iopl_mask(mask) (paravirt_ops.set_iopl_mask(mask)) |
306 | 321 | ||
322 | #define __pte(x) paravirt_ops.make_pte(x) | ||
323 | #define __pgd(x) paravirt_ops.make_pgd(x) | ||
324 | |||
325 | #define pte_val(x) paravirt_ops.pte_val(x) | ||
326 | #define pgd_val(x) paravirt_ops.pgd_val(x) | ||
327 | |||
328 | #ifdef CONFIG_X86_PAE | ||
329 | #define __pmd(x) paravirt_ops.make_pmd(x) | ||
330 | #define pmd_val(x) paravirt_ops.pmd_val(x) | ||
331 | #endif | ||
332 | |||
307 | /* The paravirtualized I/O functions */ | 333 | /* The paravirtualized I/O functions */ |
308 | static inline void slow_down_io(void) { | 334 | static inline void slow_down_io(void) { |
309 | paravirt_ops.io_delay(); | 335 | paravirt_ops.io_delay(); |
@@ -344,6 +370,7 @@ static inline void setup_secondary_clock(void) | |||
344 | } | 370 | } |
345 | #endif | 371 | #endif |
346 | 372 | ||
373 | |||
347 | #ifdef CONFIG_SMP | 374 | #ifdef CONFIG_SMP |
348 | static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip, | 375 | static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip, |
349 | unsigned long start_esp) | 376 | unsigned long start_esp) |
@@ -371,7 +398,8 @@ static inline void set_pte(pte_t *ptep, pte_t pteval) | |||
371 | paravirt_ops.set_pte(ptep, pteval); | 398 | paravirt_ops.set_pte(ptep, pteval); |
372 | } | 399 | } |
373 | 400 | ||
374 | static inline void set_pte_at(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pteval) | 401 | static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, |
402 | pte_t *ptep, pte_t pteval) | ||
375 | { | 403 | { |
376 | paravirt_ops.set_pte_at(mm, addr, ptep, pteval); | 404 | paravirt_ops.set_pte_at(mm, addr, ptep, pteval); |
377 | } | 405 | } |
diff --git a/include/asm-i386/pgtable-2level.h b/include/asm-i386/pgtable-2level.h index 38c3fcc0676d..043a2bcfa86a 100644 --- a/include/asm-i386/pgtable-2level.h +++ b/include/asm-i386/pgtable-2level.h | |||
@@ -11,10 +11,23 @@ | |||
11 | * within a page table are directly modified. Thus, the following | 11 | * within a page table are directly modified. Thus, the following |
12 | * hook is made available. | 12 | * hook is made available. |
13 | */ | 13 | */ |
14 | static inline void native_set_pte(pte_t *ptep , pte_t pte) | ||
15 | { | ||
16 | *ptep = pte; | ||
17 | } | ||
18 | static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr, | ||
19 | pte_t *ptep , pte_t pte) | ||
20 | { | ||
21 | native_set_pte(ptep, pte); | ||
22 | } | ||
23 | static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) | ||
24 | { | ||
25 | *pmdp = pmd; | ||
26 | } | ||
14 | #ifndef CONFIG_PARAVIRT | 27 | #ifndef CONFIG_PARAVIRT |
15 | #define set_pte(pteptr, pteval) (*(pteptr) = pteval) | 28 | #define set_pte(pteptr, pteval) native_set_pte(pteptr, pteval) |
16 | #define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) | 29 | #define set_pte_at(mm,addr,ptep,pteval) native_set_pte_at(mm, addr, ptep, pteval) |
17 | #define set_pmd(pmdptr, pmdval) (*(pmdptr) = (pmdval)) | 30 | #define set_pmd(pmdptr, pmdval) native_set_pmd(pmdptr, pmdval) |
18 | #endif | 31 | #endif |
19 | 32 | ||
20 | #define set_pte_atomic(pteptr, pteval) set_pte(pteptr,pteval) | 33 | #define set_pte_atomic(pteptr, pteval) set_pte(pteptr,pteval) |
@@ -23,11 +36,14 @@ | |||
23 | #define pte_clear(mm,addr,xp) do { set_pte_at(mm, addr, xp, __pte(0)); } while (0) | 36 | #define pte_clear(mm,addr,xp) do { set_pte_at(mm, addr, xp, __pte(0)); } while (0) |
24 | #define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0) | 37 | #define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0) |
25 | 38 | ||
26 | #define raw_ptep_get_and_clear(xp) __pte(xchg(&(xp)->pte_low, 0)) | 39 | static inline pte_t native_ptep_get_and_clear(pte_t *xp) |
40 | { | ||
41 | return __pte(xchg(&xp->pte_low, 0)); | ||
42 | } | ||
27 | 43 | ||
28 | #define pte_page(x) pfn_to_page(pte_pfn(x)) | 44 | #define pte_page(x) pfn_to_page(pte_pfn(x)) |
29 | #define pte_none(x) (!(x).pte_low) | 45 | #define pte_none(x) (!(x).pte_low) |
30 | #define pte_pfn(x) ((unsigned long)(((x).pte_low >> PAGE_SHIFT))) | 46 | #define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT) |
31 | #define pfn_pte(pfn, prot) __pte(((pfn) << PAGE_SHIFT) | pgprot_val(prot)) | 47 | #define pfn_pte(pfn, prot) __pte(((pfn) << PAGE_SHIFT) | pgprot_val(prot)) |
32 | #define pfn_pmd(pfn, prot) __pmd(((pfn) << PAGE_SHIFT) | pgprot_val(prot)) | 48 | #define pfn_pmd(pfn, prot) __pmd(((pfn) << PAGE_SHIFT) | pgprot_val(prot)) |
33 | 49 | ||
diff --git a/include/asm-i386/pgtable-3level.h b/include/asm-i386/pgtable-3level.h index 7a2318f38303..be6017f37a91 100644 --- a/include/asm-i386/pgtable-3level.h +++ b/include/asm-i386/pgtable-3level.h | |||
@@ -42,20 +42,23 @@ static inline int pte_exec_kernel(pte_t pte) | |||
42 | return pte_x(pte); | 42 | return pte_x(pte); |
43 | } | 43 | } |
44 | 44 | ||
45 | #ifndef CONFIG_PARAVIRT | ||
46 | /* Rules for using set_pte: the pte being assigned *must* be | 45 | /* Rules for using set_pte: the pte being assigned *must* be |
47 | * either not present or in a state where the hardware will | 46 | * either not present or in a state where the hardware will |
48 | * not attempt to update the pte. In places where this is | 47 | * not attempt to update the pte. In places where this is |
49 | * not possible, use pte_get_and_clear to obtain the old pte | 48 | * not possible, use pte_get_and_clear to obtain the old pte |
50 | * value and then use set_pte to update it. -ben | 49 | * value and then use set_pte to update it. -ben |
51 | */ | 50 | */ |
52 | static inline void set_pte(pte_t *ptep, pte_t pte) | 51 | static inline void native_set_pte(pte_t *ptep, pte_t pte) |
53 | { | 52 | { |
54 | ptep->pte_high = pte.pte_high; | 53 | ptep->pte_high = pte.pte_high; |
55 | smp_wmb(); | 54 | smp_wmb(); |
56 | ptep->pte_low = pte.pte_low; | 55 | ptep->pte_low = pte.pte_low; |
57 | } | 56 | } |
58 | #define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) | 57 | static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr, |
58 | pte_t *ptep , pte_t pte) | ||
59 | { | ||
60 | native_set_pte(ptep, pte); | ||
61 | } | ||
59 | 62 | ||
60 | /* | 63 | /* |
61 | * Since this is only called on user PTEs, and the page fault handler | 64 | * Since this is only called on user PTEs, and the page fault handler |
@@ -63,7 +66,8 @@ static inline void set_pte(pte_t *ptep, pte_t pte) | |||
63 | * we are justified in merely clearing the PTE present bit, followed | 66 | * we are justified in merely clearing the PTE present bit, followed |
64 | * by a set. The ordering here is important. | 67 | * by a set. The ordering here is important. |
65 | */ | 68 | */ |
66 | static inline void set_pte_present(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) | 69 | static inline void native_set_pte_present(struct mm_struct *mm, unsigned long addr, |
70 | pte_t *ptep, pte_t pte) | ||
67 | { | 71 | { |
68 | ptep->pte_low = 0; | 72 | ptep->pte_low = 0; |
69 | smp_wmb(); | 73 | smp_wmb(); |
@@ -72,32 +76,48 @@ static inline void set_pte_present(struct mm_struct *mm, unsigned long addr, pte | |||
72 | ptep->pte_low = pte.pte_low; | 76 | ptep->pte_low = pte.pte_low; |
73 | } | 77 | } |
74 | 78 | ||
75 | #define set_pte_atomic(pteptr,pteval) \ | 79 | static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) |
76 | set_64bit((unsigned long long *)(pteptr),pte_val(pteval)) | 80 | { |
77 | #define set_pmd(pmdptr,pmdval) \ | 81 | set_64bit((unsigned long long *)(ptep),native_pte_val(pte)); |
78 | set_64bit((unsigned long long *)(pmdptr),pmd_val(pmdval)) | 82 | } |
79 | #define set_pud(pudptr,pudval) \ | 83 | static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) |
80 | (*(pudptr) = (pudval)) | 84 | { |
85 | set_64bit((unsigned long long *)(pmdp),native_pmd_val(pmd)); | ||
86 | } | ||
87 | static inline void native_set_pud(pud_t *pudp, pud_t pud) | ||
88 | { | ||
89 | *pudp = pud; | ||
90 | } | ||
81 | 91 | ||
82 | /* | 92 | /* |
83 | * For PTEs and PDEs, we must clear the P-bit first when clearing a page table | 93 | * For PTEs and PDEs, we must clear the P-bit first when clearing a page table |
84 | * entry, so clear the bottom half first and enforce ordering with a compiler | 94 | * entry, so clear the bottom half first and enforce ordering with a compiler |
85 | * barrier. | 95 | * barrier. |
86 | */ | 96 | */ |
87 | static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | 97 | static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) |
88 | { | 98 | { |
89 | ptep->pte_low = 0; | 99 | ptep->pte_low = 0; |
90 | smp_wmb(); | 100 | smp_wmb(); |
91 | ptep->pte_high = 0; | 101 | ptep->pte_high = 0; |
92 | } | 102 | } |
93 | 103 | ||
94 | static inline void pmd_clear(pmd_t *pmd) | 104 | static inline void native_pmd_clear(pmd_t *pmd) |
95 | { | 105 | { |
96 | u32 *tmp = (u32 *)pmd; | 106 | u32 *tmp = (u32 *)pmd; |
97 | *tmp = 0; | 107 | *tmp = 0; |
98 | smp_wmb(); | 108 | smp_wmb(); |
99 | *(tmp + 1) = 0; | 109 | *(tmp + 1) = 0; |
100 | } | 110 | } |
111 | |||
112 | #ifndef CONFIG_PARAVIRT | ||
113 | #define set_pte(ptep, pte) native_set_pte(ptep, pte) | ||
114 | #define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte) | ||
115 | #define set_pte_present(mm, addr, ptep, pte) native_set_pte_present(mm, addr, ptep, pte) | ||
116 | #define set_pte_atomic(ptep, pte) native_set_pte_atomic(ptep, pte) | ||
117 | #define set_pmd(pmdp, pmd) native_set_pmd(pmdp, pmd) | ||
118 | #define set_pud(pudp, pud) native_set_pud(pudp, pud) | ||
119 | #define pte_clear(mm, addr, ptep) native_pte_clear(mm, addr, ptep) | ||
120 | #define pmd_clear(pmd) native_pmd_clear(pmd) | ||
101 | #endif | 121 | #endif |
102 | 122 | ||
103 | /* | 123 | /* |
@@ -119,7 +139,7 @@ static inline void pud_clear (pud_t * pud) { } | |||
119 | #define pmd_offset(pud, address) ((pmd_t *) pud_page(*(pud)) + \ | 139 | #define pmd_offset(pud, address) ((pmd_t *) pud_page(*(pud)) + \ |
120 | pmd_index(address)) | 140 | pmd_index(address)) |
121 | 141 | ||
122 | static inline pte_t raw_ptep_get_and_clear(pte_t *ptep) | 142 | static inline pte_t native_ptep_get_and_clear(pte_t *ptep) |
123 | { | 143 | { |
124 | pte_t res; | 144 | pte_t res; |
125 | 145 | ||
@@ -146,28 +166,21 @@ static inline int pte_none(pte_t pte) | |||
146 | 166 | ||
147 | static inline unsigned long pte_pfn(pte_t pte) | 167 | static inline unsigned long pte_pfn(pte_t pte) |
148 | { | 168 | { |
149 | return (pte.pte_low >> PAGE_SHIFT) | | 169 | return pte_val(pte) >> PAGE_SHIFT; |
150 | (pte.pte_high << (32 - PAGE_SHIFT)); | ||
151 | } | 170 | } |
152 | 171 | ||
153 | extern unsigned long long __supported_pte_mask; | 172 | extern unsigned long long __supported_pte_mask; |
154 | 173 | ||
155 | static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot) | 174 | static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot) |
156 | { | 175 | { |
157 | pte_t pte; | 176 | return __pte((((unsigned long long)page_nr << PAGE_SHIFT) | |
158 | 177 | pgprot_val(pgprot)) & __supported_pte_mask); | |
159 | pte.pte_high = (page_nr >> (32 - PAGE_SHIFT)) | \ | ||
160 | (pgprot_val(pgprot) >> 32); | ||
161 | pte.pte_high &= (__supported_pte_mask >> 32); | ||
162 | pte.pte_low = ((page_nr << PAGE_SHIFT) | pgprot_val(pgprot)) & \ | ||
163 | __supported_pte_mask; | ||
164 | return pte; | ||
165 | } | 178 | } |
166 | 179 | ||
167 | static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot) | 180 | static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot) |
168 | { | 181 | { |
169 | return __pmd((((unsigned long long)page_nr << PAGE_SHIFT) | \ | 182 | return __pmd((((unsigned long long)page_nr << PAGE_SHIFT) | |
170 | pgprot_val(pgprot)) & __supported_pte_mask); | 183 | pgprot_val(pgprot)) & __supported_pte_mask); |
171 | } | 184 | } |
172 | 185 | ||
173 | /* | 186 | /* |
diff --git a/include/asm-i386/pgtable.h b/include/asm-i386/pgtable.h index 143ddc42b86f..147f2553784d 100644 --- a/include/asm-i386/pgtable.h +++ b/include/asm-i386/pgtable.h | |||
@@ -266,6 +266,8 @@ static inline pte_t pte_mkhuge(pte_t pte) { (pte).pte_low |= _PAGE_PSE; return p | |||
266 | #define pte_update(mm, addr, ptep) do { } while (0) | 266 | #define pte_update(mm, addr, ptep) do { } while (0) |
267 | #define pte_update_defer(mm, addr, ptep) do { } while (0) | 267 | #define pte_update_defer(mm, addr, ptep) do { } while (0) |
268 | #define paravirt_map_pt_hook(slot, va, pfn) do { } while (0) | 268 | #define paravirt_map_pt_hook(slot, va, pfn) do { } while (0) |
269 | |||
270 | #define raw_ptep_get_and_clear(xp) native_ptep_get_and_clear(xp) | ||
269 | #endif | 271 | #endif |
270 | 272 | ||
271 | /* | 273 | /* |