aboutsummaryrefslogtreecommitdiffstats
path: root/include
diff options
context:
space:
mode:
authorZachary Amsden <zach@vmware.com>2006-04-27 14:32:29 -0400
committerLinus Torvalds <torvalds@g5.osdl.org>2006-04-27 15:00:59 -0400
commit6e5882cfa24e1456702e463f6920fc0ca3c3d2b8 (patch)
tree636800c0619ec3551f9a8a7da707d913937cf43d /include
parent2be4d50295e2b6f62c07b614e1b103e280dddb84 (diff)
[PATCH] x86/PAE: Fix pte_clear for the >4GB RAM case
Proposed fix for ptep_get_and_clear_full PAE bug. Pte_clear had the same bug, so use the same fix for both. Turns out pmd_clear had it as well, but pgds are not affected. The problem is rather intricate. Page table entries in PAE mode are 64-bits wide, but the only atomic 8-byte write operation available in 32-bit mode is cmpxchg8b, which is expensive (at least on P4), and thus avoided. But it can happen that the processor may prefetch entries into the TLB in the middle of an operation which clears a page table entry. So one must always clear the P-bit in the low word of the page table entry first when clearing it. Since the sequence *ptep = __pte(0) leaves the order of the write dependent on the compiler, it must be coded explicitly as a clear of the low word followed by a clear of the high word. Further, there must be a write memory barrier here to enforce proper ordering by the compiler (and, in the future, by the processor as well). On > 4GB memory machines, the implementation of pte_clear for PAE was clearly deficient, as it could leave virtual mappings of physical memory above 4GB aliased to memory below 4GB in the TLB. The implementation of ptep_get_and_clear_full has a similar bug, although not nearly as likely to occur, since the mappings being cleared are in the process of being destroyed, and should never be dereferenced again. But, as luck would have it, it is possible to trigger bugs even without ever dereferencing these bogus TLB mappings, even if the clear is followed fairly soon after with a TLB flush or invalidation. The problem is that memory above 4GB may now be aliased into the first 4GB of memory, and in fact, may hit a region of memory with non-memory semantics. These regions include AGP and PCI space. As such, these memory regions are not cached by the processor. This introduces the bug. The processor can speculate memory operations, including memory writes, as long as they are committed with the proper ordering. Speculating a memory write to a linear address that has a bogus TLB mapping is possible. Normally, the speculation is harmless. But for cached memory, it does leave the falsely speculated cacheline unmodified, but in a dirty state. This cache line will be eventually written back. If this cacheline happens to intersect a region of memory that is not protected by the cache coherency protocol, it can corrupt data in I/O memory, which is generally a very bad thing to do, and can cause total system failure or just plain undefined behavior. These bugs are extremely unlikely, but the severity is of such magnitude, and the fix so simple that I think fixing them immediately is justified. Also, they are nearly impossible to debug. Signed-off-by: Zachary Amsden <zach@vmware.com> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Diffstat (limited to 'include')
-rw-r--r--include/asm-i386/pgtable-2level.h3
-rw-r--r--include/asm-i386/pgtable-3level.h20
-rw-r--r--include/asm-i386/pgtable.h4
3 files changed, 24 insertions, 3 deletions
diff --git a/include/asm-i386/pgtable-2level.h b/include/asm-i386/pgtable-2level.h
index 27bde973abc7..2756d4b04c27 100644
--- a/include/asm-i386/pgtable-2level.h
+++ b/include/asm-i386/pgtable-2level.h
@@ -18,6 +18,9 @@
18#define set_pte_atomic(pteptr, pteval) set_pte(pteptr,pteval) 18#define set_pte_atomic(pteptr, pteval) set_pte(pteptr,pteval)
19#define set_pmd(pmdptr, pmdval) (*(pmdptr) = (pmdval)) 19#define set_pmd(pmdptr, pmdval) (*(pmdptr) = (pmdval))
20 20
21#define pte_clear(mm,addr,xp) do { set_pte_at(mm, addr, xp, __pte(0)); } while (0)
22#define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0)
23
21#define ptep_get_and_clear(mm,addr,xp) __pte(xchg(&(xp)->pte_low, 0)) 24#define ptep_get_and_clear(mm,addr,xp) __pte(xchg(&(xp)->pte_low, 0))
22#define pte_same(a, b) ((a).pte_low == (b).pte_low) 25#define pte_same(a, b) ((a).pte_low == (b).pte_low)
23#define pte_page(x) pfn_to_page(pte_pfn(x)) 26#define pte_page(x) pfn_to_page(pte_pfn(x))
diff --git a/include/asm-i386/pgtable-3level.h b/include/asm-i386/pgtable-3level.h
index 36a5aa63cbbf..dccb1b3337ad 100644
--- a/include/asm-i386/pgtable-3level.h
+++ b/include/asm-i386/pgtable-3level.h
@@ -85,6 +85,26 @@ static inline void pud_clear (pud_t * pud) { }
85#define pmd_offset(pud, address) ((pmd_t *) pud_page(*(pud)) + \ 85#define pmd_offset(pud, address) ((pmd_t *) pud_page(*(pud)) + \
86 pmd_index(address)) 86 pmd_index(address))
87 87
88/*
89 * For PTEs and PDEs, we must clear the P-bit first when clearing a page table
90 * entry, so clear the bottom half first and enforce ordering with a compiler
91 * barrier.
92 */
93static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
94{
95 ptep->pte_low = 0;
96 smp_wmb();
97 ptep->pte_high = 0;
98}
99
100static inline void pmd_clear(pmd_t *pmd)
101{
102 u32 *tmp = (u32 *)pmd;
103 *tmp = 0;
104 smp_wmb();
105 *(tmp + 1) = 0;
106}
107
88static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) 108static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
89{ 109{
90 pte_t res; 110 pte_t res;
diff --git a/include/asm-i386/pgtable.h b/include/asm-i386/pgtable.h
index ee056c41a9fb..672c3f76b9df 100644
--- a/include/asm-i386/pgtable.h
+++ b/include/asm-i386/pgtable.h
@@ -204,12 +204,10 @@ extern unsigned long long __PAGE_KERNEL, __PAGE_KERNEL_EXEC;
204extern unsigned long pg0[]; 204extern unsigned long pg0[];
205 205
206#define pte_present(x) ((x).pte_low & (_PAGE_PRESENT | _PAGE_PROTNONE)) 206#define pte_present(x) ((x).pte_low & (_PAGE_PRESENT | _PAGE_PROTNONE))
207#define pte_clear(mm,addr,xp) do { set_pte_at(mm, addr, xp, __pte(0)); } while (0)
208 207
209/* To avoid harmful races, pmd_none(x) should check only the lower when PAE */ 208/* To avoid harmful races, pmd_none(x) should check only the lower when PAE */
210#define pmd_none(x) (!(unsigned long)pmd_val(x)) 209#define pmd_none(x) (!(unsigned long)pmd_val(x))
211#define pmd_present(x) (pmd_val(x) & _PAGE_PRESENT) 210#define pmd_present(x) (pmd_val(x) & _PAGE_PRESENT)
212#define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0)
213#define pmd_bad(x) ((pmd_val(x) & (~PAGE_MASK & ~_PAGE_USER)) != _KERNPG_TABLE) 211#define pmd_bad(x) ((pmd_val(x) & (~PAGE_MASK & ~_PAGE_USER)) != _KERNPG_TABLE)
214 212
215 213
@@ -268,7 +266,7 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, unsigned long
268 pte_t pte; 266 pte_t pte;
269 if (full) { 267 if (full) {
270 pte = *ptep; 268 pte = *ptep;
271 *ptep = __pte(0); 269 pte_clear(mm, addr, ptep);
272 } else { 270 } else {
273 pte = ptep_get_and_clear(mm, addr, ptep); 271 pte = ptep_get_and_clear(mm, addr, ptep);
274 } 272 }