aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--arch/i386/kernel/paravirt.c84
-rw-r--r--arch/i386/kernel/vmi.c6
-rw-r--r--include/asm-i386/page.h79
-rw-r--r--include/asm-i386/paravirt.h52
-rw-r--r--include/asm-i386/pgtable-2level.h26
-rw-r--r--include/asm-i386/pgtable-3level.h63
-rw-r--r--include/asm-i386/pgtable.h2
7 files changed, 184 insertions, 128 deletions
diff --git a/arch/i386/kernel/paravirt.c b/arch/i386/kernel/paravirt.c
index 3fdbd1f62379..cba7a15ce1b0 100644
--- a/arch/i386/kernel/paravirt.c
+++ b/arch/i386/kernel/paravirt.c
@@ -117,78 +117,6 @@ static void native_flush_tlb_single(u32 addr)
117 __native_flush_tlb_single(addr); 117 __native_flush_tlb_single(addr);
118} 118}
119 119
120#ifndef CONFIG_X86_PAE
121static void native_set_pte(pte_t *ptep, pte_t pteval)
122{
123 *ptep = pteval;
124}
125
126static void native_set_pte_at(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pteval)
127{
128 *ptep = pteval;
129}
130
131static void native_set_pmd(pmd_t *pmdp, pmd_t pmdval)
132{
133 *pmdp = pmdval;
134}
135
136#else /* CONFIG_X86_PAE */
137
138static void native_set_pte(pte_t *ptep, pte_t pte)
139{
140 ptep->pte_high = pte.pte_high;
141 smp_wmb();
142 ptep->pte_low = pte.pte_low;
143}
144
145static void native_set_pte_at(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pte)
146{
147 ptep->pte_high = pte.pte_high;
148 smp_wmb();
149 ptep->pte_low = pte.pte_low;
150}
151
152static void native_set_pte_present(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte)
153{
154 ptep->pte_low = 0;
155 smp_wmb();
156 ptep->pte_high = pte.pte_high;
157 smp_wmb();
158 ptep->pte_low = pte.pte_low;
159}
160
161static void native_set_pte_atomic(pte_t *ptep, pte_t pteval)
162{
163 set_64bit((unsigned long long *)ptep,pte_val(pteval));
164}
165
166static void native_set_pmd(pmd_t *pmdp, pmd_t pmdval)
167{
168 set_64bit((unsigned long long *)pmdp,pmd_val(pmdval));
169}
170
171static void native_set_pud(pud_t *pudp, pud_t pudval)
172{
173 *pudp = pudval;
174}
175
176static void native_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
177{
178 ptep->pte_low = 0;
179 smp_wmb();
180 ptep->pte_high = 0;
181}
182
183static void native_pmd_clear(pmd_t *pmd)
184{
185 u32 *tmp = (u32 *)pmd;
186 *tmp = 0;
187 smp_wmb();
188 *(tmp + 1) = 0;
189}
190#endif /* CONFIG_X86_PAE */
191
192/* These are in entry.S */ 120/* These are in entry.S */
193extern void native_iret(void); 121extern void native_iret(void);
194extern void native_irq_enable_sysexit(void); 122extern void native_irq_enable_sysexit(void);
@@ -282,14 +210,26 @@ struct paravirt_ops paravirt_ops = {
282 .set_pmd = native_set_pmd, 210 .set_pmd = native_set_pmd,
283 .pte_update = paravirt_nop, 211 .pte_update = paravirt_nop,
284 .pte_update_defer = paravirt_nop, 212 .pte_update_defer = paravirt_nop,
213
214 .ptep_get_and_clear = native_ptep_get_and_clear,
215
285#ifdef CONFIG_X86_PAE 216#ifdef CONFIG_X86_PAE
286 .set_pte_atomic = native_set_pte_atomic, 217 .set_pte_atomic = native_set_pte_atomic,
287 .set_pte_present = native_set_pte_present, 218 .set_pte_present = native_set_pte_present,
288 .set_pud = native_set_pud, 219 .set_pud = native_set_pud,
289 .pte_clear = native_pte_clear, 220 .pte_clear = native_pte_clear,
290 .pmd_clear = native_pmd_clear, 221 .pmd_clear = native_pmd_clear,
222
223 .pmd_val = native_pmd_val,
224 .make_pmd = native_make_pmd,
291#endif 225#endif
292 226
227 .pte_val = native_pte_val,
228 .pgd_val = native_pgd_val,
229
230 .make_pte = native_make_pte,
231 .make_pgd = native_make_pgd,
232
293 .irq_enable_sysexit = native_irq_enable_sysexit, 233 .irq_enable_sysexit = native_irq_enable_sysexit,
294 .iret = native_iret, 234 .iret = native_iret,
295 235
diff --git a/arch/i386/kernel/vmi.c b/arch/i386/kernel/vmi.c
index 8f3bac473450..ea77d93f59dd 100644
--- a/arch/i386/kernel/vmi.c
+++ b/arch/i386/kernel/vmi.c
@@ -443,13 +443,13 @@ static void vmi_release_pd(u32 pfn)
443 ((level) | (is_current_as(mm, user) ? \ 443 ((level) | (is_current_as(mm, user) ? \
444 (VMI_PAGE_DEFER | VMI_PAGE_CURRENT_AS | ((addr) & VMI_PAGE_VA_MASK)) : 0)) 444 (VMI_PAGE_DEFER | VMI_PAGE_CURRENT_AS | ((addr) & VMI_PAGE_VA_MASK)) : 0))
445 445
446static void vmi_update_pte(struct mm_struct *mm, u32 addr, pte_t *ptep) 446static void vmi_update_pte(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
447{ 447{
448 vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE); 448 vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE);
449 vmi_ops.update_pte(ptep, vmi_flags_addr(mm, addr, VMI_PAGE_PT, 0)); 449 vmi_ops.update_pte(ptep, vmi_flags_addr(mm, addr, VMI_PAGE_PT, 0));
450} 450}
451 451
452static void vmi_update_pte_defer(struct mm_struct *mm, u32 addr, pte_t *ptep) 452static void vmi_update_pte_defer(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
453{ 453{
454 vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE); 454 vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE);
455 vmi_ops.update_pte(ptep, vmi_flags_addr_defer(mm, addr, VMI_PAGE_PT, 0)); 455 vmi_ops.update_pte(ptep, vmi_flags_addr_defer(mm, addr, VMI_PAGE_PT, 0));
@@ -462,7 +462,7 @@ static void vmi_set_pte(pte_t *ptep, pte_t pte)
462 vmi_ops.set_pte(pte, ptep, VMI_PAGE_PT); 462 vmi_ops.set_pte(pte, ptep, VMI_PAGE_PT);
463} 463}
464 464
465static void vmi_set_pte_at(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pte) 465static void vmi_set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte)
466{ 466{
467 vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE); 467 vmi_check_page_type(__pa(ptep) >> PAGE_SHIFT, VMI_PAGE_PTE);
468 vmi_ops.set_pte(pte, ptep, vmi_flags_addr(mm, addr, VMI_PAGE_PT, 0)); 468 vmi_ops.set_pte(pte, ptep, vmi_flags_addr(mm, addr, VMI_PAGE_PT, 0));
diff --git a/include/asm-i386/page.h b/include/asm-i386/page.h
index fd3f64ace248..818ac8bf01e2 100644
--- a/include/asm-i386/page.h
+++ b/include/asm-i386/page.h
@@ -12,7 +12,6 @@
12#ifdef __KERNEL__ 12#ifdef __KERNEL__
13#ifndef __ASSEMBLY__ 13#ifndef __ASSEMBLY__
14 14
15
16#ifdef CONFIG_X86_USE_3DNOW 15#ifdef CONFIG_X86_USE_3DNOW
17 16
18#include <asm/mmx.h> 17#include <asm/mmx.h>
@@ -42,26 +41,81 @@
42 * These are used to make use of C type-checking.. 41 * These are used to make use of C type-checking..
43 */ 42 */
44extern int nx_enabled; 43extern int nx_enabled;
44
45#ifdef CONFIG_X86_PAE 45#ifdef CONFIG_X86_PAE
46extern unsigned long long __supported_pte_mask; 46extern unsigned long long __supported_pte_mask;
47typedef struct { unsigned long pte_low, pte_high; } pte_t; 47typedef struct { unsigned long pte_low, pte_high; } pte_t;
48typedef struct { unsigned long long pmd; } pmd_t; 48typedef struct { unsigned long long pmd; } pmd_t;
49typedef struct { unsigned long long pgd; } pgd_t; 49typedef struct { unsigned long long pgd; } pgd_t;
50typedef struct { unsigned long long pgprot; } pgprot_t; 50typedef struct { unsigned long long pgprot; } pgprot_t;
51#define pmd_val(x) ((x).pmd) 51
52#define pte_val(x) ((x).pte_low | ((unsigned long long)(x).pte_high << 32)) 52static inline unsigned long long native_pgd_val(pgd_t pgd)
53#define __pmd(x) ((pmd_t) { (x) } ) 53{
54 return pgd.pgd;
55}
56
57static inline unsigned long long native_pmd_val(pmd_t pmd)
58{
59 return pmd.pmd;
60}
61
62static inline unsigned long long native_pte_val(pte_t pte)
63{
64 return pte.pte_low | ((unsigned long long)pte.pte_high << 32);
65}
66
67static inline pgd_t native_make_pgd(unsigned long long val)
68{
69 return (pgd_t) { val };
70}
71
72static inline pmd_t native_make_pmd(unsigned long long val)
73{
74 return (pmd_t) { val };
75}
76
77static inline pte_t native_make_pte(unsigned long long val)
78{
79 return (pte_t) { .pte_low = val, .pte_high = (val >> 32) } ;
80}
81
82#ifndef CONFIG_PARAVIRT
83#define pmd_val(x) native_pmd_val(x)
84#define __pmd(x) native_make_pmd(x)
85#endif
86
54#define HPAGE_SHIFT 21 87#define HPAGE_SHIFT 21
55#include <asm-generic/pgtable-nopud.h> 88#include <asm-generic/pgtable-nopud.h>
56#else 89#else /* !CONFIG_X86_PAE */
57typedef struct { unsigned long pte_low; } pte_t; 90typedef struct { unsigned long pte_low; } pte_t;
58typedef struct { unsigned long pgd; } pgd_t; 91typedef struct { unsigned long pgd; } pgd_t;
59typedef struct { unsigned long pgprot; } pgprot_t; 92typedef struct { unsigned long pgprot; } pgprot_t;
60#define boot_pte_t pte_t /* or would you rather have a typedef */ 93#define boot_pte_t pte_t /* or would you rather have a typedef */
61#define pte_val(x) ((x).pte_low) 94
95static inline unsigned long native_pgd_val(pgd_t pgd)
96{
97 return pgd.pgd;
98}
99
100static inline unsigned long native_pte_val(pte_t pte)
101{
102 return pte.pte_low;
103}
104
105static inline pgd_t native_make_pgd(unsigned long val)
106{
107 return (pgd_t) { val };
108}
109
110static inline pte_t native_make_pte(unsigned long val)
111{
112 return (pte_t) { .pte_low = val };
113}
114
62#define HPAGE_SHIFT 22 115#define HPAGE_SHIFT 22
63#include <asm-generic/pgtable-nopmd.h> 116#include <asm-generic/pgtable-nopmd.h>
64#endif 117#endif /* CONFIG_X86_PAE */
118
65#define PTE_MASK PAGE_MASK 119#define PTE_MASK PAGE_MASK
66 120
67#ifdef CONFIG_HUGETLB_PAGE 121#ifdef CONFIG_HUGETLB_PAGE
@@ -71,13 +125,16 @@ typedef struct { unsigned long pgprot; } pgprot_t;
71#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA 125#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
72#endif 126#endif
73 127
74#define pgd_val(x) ((x).pgd)
75#define pgprot_val(x) ((x).pgprot) 128#define pgprot_val(x) ((x).pgprot)
76
77#define __pte(x) ((pte_t) { (x) } )
78#define __pgd(x) ((pgd_t) { (x) } )
79#define __pgprot(x) ((pgprot_t) { (x) } ) 129#define __pgprot(x) ((pgprot_t) { (x) } )
80 130
131#ifndef CONFIG_PARAVIRT
132#define pgd_val(x) native_pgd_val(x)
133#define __pgd(x) native_make_pgd(x)
134#define pte_val(x) native_pte_val(x)
135#define __pte(x) native_make_pte(x)
136#endif
137
81#endif /* !__ASSEMBLY__ */ 138#endif /* !__ASSEMBLY__ */
82 139
83/* to align the pointer to the (next) page boundary */ 140/* to align the pointer to the (next) page boundary */
diff --git a/include/asm-i386/paravirt.h b/include/asm-i386/paravirt.h
index f0bdaea6235d..0aacb13bb929 100644
--- a/include/asm-i386/paravirt.h
+++ b/include/asm-i386/paravirt.h
@@ -2,7 +2,6 @@
2#define __ASM_PARAVIRT_H 2#define __ASM_PARAVIRT_H
3/* Various instructions on x86 need to be replaced for 3/* Various instructions on x86 need to be replaced for
4 * para-virtualization: those hooks are defined here. */ 4 * para-virtualization: those hooks are defined here. */
5#include <linux/linkage.h>
6#include <linux/stringify.h> 5#include <linux/stringify.h>
7#include <asm/page.h> 6#include <asm/page.h>
8 7
@@ -25,6 +24,8 @@
25#define CLBR_ANY 0x7 24#define CLBR_ANY 0x7
26 25
27#ifndef __ASSEMBLY__ 26#ifndef __ASSEMBLY__
27#include <linux/types.h>
28
28struct thread_struct; 29struct thread_struct;
29struct Xgt_desc_struct; 30struct Xgt_desc_struct;
30struct tss_struct; 31struct tss_struct;
@@ -55,11 +56,6 @@ struct paravirt_ops
55 int (*set_wallclock)(unsigned long); 56 int (*set_wallclock)(unsigned long);
56 void (*time_init)(void); 57 void (*time_init)(void);
57 58
58 /* All the function pointers here are declared as "fastcall"
59 so that we get a specific register-based calling
60 convention. This makes it easier to implement inline
61 assembler replacements. */
62
63 void (*cpuid)(unsigned int *eax, unsigned int *ebx, 59 void (*cpuid)(unsigned int *eax, unsigned int *ebx,
64 unsigned int *ecx, unsigned int *edx); 60 unsigned int *ecx, unsigned int *edx);
65 61
@@ -139,16 +135,33 @@ struct paravirt_ops
139 void (*release_pd)(u32 pfn); 135 void (*release_pd)(u32 pfn);
140 136
141 void (*set_pte)(pte_t *ptep, pte_t pteval); 137 void (*set_pte)(pte_t *ptep, pte_t pteval);
142 void (*set_pte_at)(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pteval); 138 void (*set_pte_at)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pteval);
143 void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval); 139 void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
144 void (*pte_update)(struct mm_struct *mm, u32 addr, pte_t *ptep); 140 void (*pte_update)(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
145 void (*pte_update_defer)(struct mm_struct *mm, u32 addr, pte_t *ptep); 141 void (*pte_update_defer)(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
142
143 pte_t (*ptep_get_and_clear)(pte_t *ptep);
144
146#ifdef CONFIG_X86_PAE 145#ifdef CONFIG_X86_PAE
147 void (*set_pte_atomic)(pte_t *ptep, pte_t pteval); 146 void (*set_pte_atomic)(pte_t *ptep, pte_t pteval);
148 void (*set_pte_present)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte); 147 void (*set_pte_present)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte);
149 void (*set_pud)(pud_t *pudp, pud_t pudval); 148 void (*set_pud)(pud_t *pudp, pud_t pudval);
150 void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); 149 void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
151 void (*pmd_clear)(pmd_t *pmdp); 150 void (*pmd_clear)(pmd_t *pmdp);
151
152 unsigned long long (*pte_val)(pte_t);
153 unsigned long long (*pmd_val)(pmd_t);
154 unsigned long long (*pgd_val)(pgd_t);
155
156 pte_t (*make_pte)(unsigned long long pte);
157 pmd_t (*make_pmd)(unsigned long long pmd);
158 pgd_t (*make_pgd)(unsigned long long pgd);
159#else
160 unsigned long (*pte_val)(pte_t);
161 unsigned long (*pgd_val)(pgd_t);
162
163 pte_t (*make_pte)(unsigned long pte);
164 pgd_t (*make_pgd)(unsigned long pgd);
152#endif 165#endif
153 166
154 void (*set_lazy_mode)(int mode); 167 void (*set_lazy_mode)(int mode);
@@ -219,6 +232,8 @@ static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
219#define read_cr4_safe(x) paravirt_ops.read_cr4_safe() 232#define read_cr4_safe(x) paravirt_ops.read_cr4_safe()
220#define write_cr4(x) paravirt_ops.write_cr4(x) 233#define write_cr4(x) paravirt_ops.write_cr4(x)
221 234
235#define raw_ptep_get_and_clear(xp) (paravirt_ops.ptep_get_and_clear(xp))
236
222static inline void raw_safe_halt(void) 237static inline void raw_safe_halt(void)
223{ 238{
224 paravirt_ops.safe_halt(); 239 paravirt_ops.safe_halt();
@@ -304,6 +319,17 @@ static inline void halt(void)
304 (paravirt_ops.write_idt_entry((dt), (entry), (low), (high))) 319 (paravirt_ops.write_idt_entry((dt), (entry), (low), (high)))
305#define set_iopl_mask(mask) (paravirt_ops.set_iopl_mask(mask)) 320#define set_iopl_mask(mask) (paravirt_ops.set_iopl_mask(mask))
306 321
322#define __pte(x) paravirt_ops.make_pte(x)
323#define __pgd(x) paravirt_ops.make_pgd(x)
324
325#define pte_val(x) paravirt_ops.pte_val(x)
326#define pgd_val(x) paravirt_ops.pgd_val(x)
327
328#ifdef CONFIG_X86_PAE
329#define __pmd(x) paravirt_ops.make_pmd(x)
330#define pmd_val(x) paravirt_ops.pmd_val(x)
331#endif
332
307/* The paravirtualized I/O functions */ 333/* The paravirtualized I/O functions */
308static inline void slow_down_io(void) { 334static inline void slow_down_io(void) {
309 paravirt_ops.io_delay(); 335 paravirt_ops.io_delay();
@@ -344,6 +370,7 @@ static inline void setup_secondary_clock(void)
344} 370}
345#endif 371#endif
346 372
373
347#ifdef CONFIG_SMP 374#ifdef CONFIG_SMP
348static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip, 375static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
349 unsigned long start_esp) 376 unsigned long start_esp)
@@ -371,7 +398,8 @@ static inline void set_pte(pte_t *ptep, pte_t pteval)
371 paravirt_ops.set_pte(ptep, pteval); 398 paravirt_ops.set_pte(ptep, pteval);
372} 399}
373 400
374static inline void set_pte_at(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pteval) 401static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
402 pte_t *ptep, pte_t pteval)
375{ 403{
376 paravirt_ops.set_pte_at(mm, addr, ptep, pteval); 404 paravirt_ops.set_pte_at(mm, addr, ptep, pteval);
377} 405}
diff --git a/include/asm-i386/pgtable-2level.h b/include/asm-i386/pgtable-2level.h
index 38c3fcc0676d..043a2bcfa86a 100644
--- a/include/asm-i386/pgtable-2level.h
+++ b/include/asm-i386/pgtable-2level.h
@@ -11,10 +11,23 @@
11 * within a page table are directly modified. Thus, the following 11 * within a page table are directly modified. Thus, the following
12 * hook is made available. 12 * hook is made available.
13 */ 13 */
14static inline void native_set_pte(pte_t *ptep , pte_t pte)
15{
16 *ptep = pte;
17}
18static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
19 pte_t *ptep , pte_t pte)
20{
21 native_set_pte(ptep, pte);
22}
23static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
24{
25 *pmdp = pmd;
26}
14#ifndef CONFIG_PARAVIRT 27#ifndef CONFIG_PARAVIRT
15#define set_pte(pteptr, pteval) (*(pteptr) = pteval) 28#define set_pte(pteptr, pteval) native_set_pte(pteptr, pteval)
16#define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) 29#define set_pte_at(mm,addr,ptep,pteval) native_set_pte_at(mm, addr, ptep, pteval)
17#define set_pmd(pmdptr, pmdval) (*(pmdptr) = (pmdval)) 30#define set_pmd(pmdptr, pmdval) native_set_pmd(pmdptr, pmdval)
18#endif 31#endif
19 32
20#define set_pte_atomic(pteptr, pteval) set_pte(pteptr,pteval) 33#define set_pte_atomic(pteptr, pteval) set_pte(pteptr,pteval)
@@ -23,11 +36,14 @@
23#define pte_clear(mm,addr,xp) do { set_pte_at(mm, addr, xp, __pte(0)); } while (0) 36#define pte_clear(mm,addr,xp) do { set_pte_at(mm, addr, xp, __pte(0)); } while (0)
24#define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0) 37#define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0)
25 38
26#define raw_ptep_get_and_clear(xp) __pte(xchg(&(xp)->pte_low, 0)) 39static inline pte_t native_ptep_get_and_clear(pte_t *xp)
40{
41 return __pte(xchg(&xp->pte_low, 0));
42}
27 43
28#define pte_page(x) pfn_to_page(pte_pfn(x)) 44#define pte_page(x) pfn_to_page(pte_pfn(x))
29#define pte_none(x) (!(x).pte_low) 45#define pte_none(x) (!(x).pte_low)
30#define pte_pfn(x) ((unsigned long)(((x).pte_low >> PAGE_SHIFT))) 46#define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT)
31#define pfn_pte(pfn, prot) __pte(((pfn) << PAGE_SHIFT) | pgprot_val(prot)) 47#define pfn_pte(pfn, prot) __pte(((pfn) << PAGE_SHIFT) | pgprot_val(prot))
32#define pfn_pmd(pfn, prot) __pmd(((pfn) << PAGE_SHIFT) | pgprot_val(prot)) 48#define pfn_pmd(pfn, prot) __pmd(((pfn) << PAGE_SHIFT) | pgprot_val(prot))
33 49
diff --git a/include/asm-i386/pgtable-3level.h b/include/asm-i386/pgtable-3level.h
index 7a2318f38303..be6017f37a91 100644
--- a/include/asm-i386/pgtable-3level.h
+++ b/include/asm-i386/pgtable-3level.h
@@ -42,20 +42,23 @@ static inline int pte_exec_kernel(pte_t pte)
42 return pte_x(pte); 42 return pte_x(pte);
43} 43}
44 44
45#ifndef CONFIG_PARAVIRT
46/* Rules for using set_pte: the pte being assigned *must* be 45/* Rules for using set_pte: the pte being assigned *must* be
47 * either not present or in a state where the hardware will 46 * either not present or in a state where the hardware will
48 * not attempt to update the pte. In places where this is 47 * not attempt to update the pte. In places where this is
49 * not possible, use pte_get_and_clear to obtain the old pte 48 * not possible, use pte_get_and_clear to obtain the old pte
50 * value and then use set_pte to update it. -ben 49 * value and then use set_pte to update it. -ben
51 */ 50 */
52static inline void set_pte(pte_t *ptep, pte_t pte) 51static inline void native_set_pte(pte_t *ptep, pte_t pte)
53{ 52{
54 ptep->pte_high = pte.pte_high; 53 ptep->pte_high = pte.pte_high;
55 smp_wmb(); 54 smp_wmb();
56 ptep->pte_low = pte.pte_low; 55 ptep->pte_low = pte.pte_low;
57} 56}
58#define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) 57static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
58 pte_t *ptep , pte_t pte)
59{
60 native_set_pte(ptep, pte);
61}
59 62
60/* 63/*
61 * Since this is only called on user PTEs, and the page fault handler 64 * Since this is only called on user PTEs, and the page fault handler
@@ -63,7 +66,8 @@ static inline void set_pte(pte_t *ptep, pte_t pte)
63 * we are justified in merely clearing the PTE present bit, followed 66 * we are justified in merely clearing the PTE present bit, followed
64 * by a set. The ordering here is important. 67 * by a set. The ordering here is important.
65 */ 68 */
66static inline void set_pte_present(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) 69static inline void native_set_pte_present(struct mm_struct *mm, unsigned long addr,
70 pte_t *ptep, pte_t pte)
67{ 71{
68 ptep->pte_low = 0; 72 ptep->pte_low = 0;
69 smp_wmb(); 73 smp_wmb();
@@ -72,32 +76,48 @@ static inline void set_pte_present(struct mm_struct *mm, unsigned long addr, pte
72 ptep->pte_low = pte.pte_low; 76 ptep->pte_low = pte.pte_low;
73} 77}
74 78
75#define set_pte_atomic(pteptr,pteval) \ 79static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
76 set_64bit((unsigned long long *)(pteptr),pte_val(pteval)) 80{
77#define set_pmd(pmdptr,pmdval) \ 81 set_64bit((unsigned long long *)(ptep),native_pte_val(pte));
78 set_64bit((unsigned long long *)(pmdptr),pmd_val(pmdval)) 82}
79#define set_pud(pudptr,pudval) \ 83static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
80 (*(pudptr) = (pudval)) 84{
85 set_64bit((unsigned long long *)(pmdp),native_pmd_val(pmd));
86}
87static inline void native_set_pud(pud_t *pudp, pud_t pud)
88{
89 *pudp = pud;
90}
81 91
82/* 92/*
83 * For PTEs and PDEs, we must clear the P-bit first when clearing a page table 93 * For PTEs and PDEs, we must clear the P-bit first when clearing a page table
84 * entry, so clear the bottom half first and enforce ordering with a compiler 94 * entry, so clear the bottom half first and enforce ordering with a compiler
85 * barrier. 95 * barrier.
86 */ 96 */
87static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) 97static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
88{ 98{
89 ptep->pte_low = 0; 99 ptep->pte_low = 0;
90 smp_wmb(); 100 smp_wmb();
91 ptep->pte_high = 0; 101 ptep->pte_high = 0;
92} 102}
93 103
94static inline void pmd_clear(pmd_t *pmd) 104static inline void native_pmd_clear(pmd_t *pmd)
95{ 105{
96 u32 *tmp = (u32 *)pmd; 106 u32 *tmp = (u32 *)pmd;
97 *tmp = 0; 107 *tmp = 0;
98 smp_wmb(); 108 smp_wmb();
99 *(tmp + 1) = 0; 109 *(tmp + 1) = 0;
100} 110}
111
112#ifndef CONFIG_PARAVIRT
113#define set_pte(ptep, pte) native_set_pte(ptep, pte)
114#define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte)
115#define set_pte_present(mm, addr, ptep, pte) native_set_pte_present(mm, addr, ptep, pte)
116#define set_pte_atomic(ptep, pte) native_set_pte_atomic(ptep, pte)
117#define set_pmd(pmdp, pmd) native_set_pmd(pmdp, pmd)
118#define set_pud(pudp, pud) native_set_pud(pudp, pud)
119#define pte_clear(mm, addr, ptep) native_pte_clear(mm, addr, ptep)
120#define pmd_clear(pmd) native_pmd_clear(pmd)
101#endif 121#endif
102 122
103/* 123/*
@@ -119,7 +139,7 @@ static inline void pud_clear (pud_t * pud) { }
119#define pmd_offset(pud, address) ((pmd_t *) pud_page(*(pud)) + \ 139#define pmd_offset(pud, address) ((pmd_t *) pud_page(*(pud)) + \
120 pmd_index(address)) 140 pmd_index(address))
121 141
122static inline pte_t raw_ptep_get_and_clear(pte_t *ptep) 142static inline pte_t native_ptep_get_and_clear(pte_t *ptep)
123{ 143{
124 pte_t res; 144 pte_t res;
125 145
@@ -146,28 +166,21 @@ static inline int pte_none(pte_t pte)
146 166
147static inline unsigned long pte_pfn(pte_t pte) 167static inline unsigned long pte_pfn(pte_t pte)
148{ 168{
149 return (pte.pte_low >> PAGE_SHIFT) | 169 return pte_val(pte) >> PAGE_SHIFT;
150 (pte.pte_high << (32 - PAGE_SHIFT));
151} 170}
152 171
153extern unsigned long long __supported_pte_mask; 172extern unsigned long long __supported_pte_mask;
154 173
155static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot) 174static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
156{ 175{
157 pte_t pte; 176 return __pte((((unsigned long long)page_nr << PAGE_SHIFT) |
158 177 pgprot_val(pgprot)) & __supported_pte_mask);
159 pte.pte_high = (page_nr >> (32 - PAGE_SHIFT)) | \
160 (pgprot_val(pgprot) >> 32);
161 pte.pte_high &= (__supported_pte_mask >> 32);
162 pte.pte_low = ((page_nr << PAGE_SHIFT) | pgprot_val(pgprot)) & \
163 __supported_pte_mask;
164 return pte;
165} 178}
166 179
167static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot) 180static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
168{ 181{
169 return __pmd((((unsigned long long)page_nr << PAGE_SHIFT) | \ 182 return __pmd((((unsigned long long)page_nr << PAGE_SHIFT) |
170 pgprot_val(pgprot)) & __supported_pte_mask); 183 pgprot_val(pgprot)) & __supported_pte_mask);
171} 184}
172 185
173/* 186/*
diff --git a/include/asm-i386/pgtable.h b/include/asm-i386/pgtable.h
index 143ddc42b86f..147f2553784d 100644
--- a/include/asm-i386/pgtable.h
+++ b/include/asm-i386/pgtable.h
@@ -266,6 +266,8 @@ static inline pte_t pte_mkhuge(pte_t pte) { (pte).pte_low |= _PAGE_PSE; return p
266#define pte_update(mm, addr, ptep) do { } while (0) 266#define pte_update(mm, addr, ptep) do { } while (0)
267#define pte_update_defer(mm, addr, ptep) do { } while (0) 267#define pte_update_defer(mm, addr, ptep) do { } while (0)
268#define paravirt_map_pt_hook(slot, va, pfn) do { } while (0) 268#define paravirt_map_pt_hook(slot, va, pfn) do { } while (0)
269
270#define raw_ptep_get_and_clear(xp) native_ptep_get_and_clear(xp)
269#endif 271#endif
270 272
271/* 273/*