7 files changed, 56 insertions, 10 deletions

diff --git a/include/asm-i386/atomic.h b/include/asm-i386/atomic.h
index 22d80ece95cb..4ddce5296a78 100644
--- a/include/asm-i386/atomic.h
+++ b/include/asm-i386/atomic.h
@@ -183,6 +183,7 @@ static __inline__ int atomic_add_return(int i, atomic_t *v)
 {
         int __i;
 #ifdef CONFIG_M386
+        unsigned long flags;
         if(unlikely(boot_cpu_data.x86==3))
                 goto no_xadd;
 #endif
@@ -196,10 +197,10 @@ static __inline__ int atomic_add_return(int i, atomic_t *v)
 
 #ifdef CONFIG_M386
 no_xadd: /* Legacy 386 processor */
-        local_irq_disable();
+        local_irq_save(flags);
         __i = atomic_read(v);
         atomic_set(v, i + __i);
-        local_irq_enable();
+        local_irq_restore(flags);
         return i + __i;
 #endif
 }
diff --git a/include/asm-i386/cpufeature.h b/include/asm-i386/cpufeature.h
index 5c0b5876b931..b44bfc6239cb 100644
--- a/include/asm-i386/cpufeature.h
+++ b/include/asm-i386/cpufeature.h
@@ -71,6 +71,7 @@
 #define X86_FEATURE_P4          (3*32+ 7) /* P4 */
 #define X86_FEATURE_CONSTANT_TSC (3*32+ 8) /* TSC ticks at a constant rate */
 #define X86_FEATURE_UP          (3*32+ 9) /* smp kernel running on up */
+#define X86_FEATURE_FXSAVE_LEAK (3*32+10) /* FXSAVE leaks FOP/FIP/FOP */
 
 /* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
 #define X86_FEATURE_XMM3        (4*32+ 0) /* Streaming SIMD Extensions-3 */
diff --git a/include/asm-i386/i387.h b/include/asm-i386/i387.h
index 152d0baa576a..bc1d6edae1ed 100644
--- a/include/asm-i386/i387.h
+++ b/include/asm-i386/i387.h
@@ -13,6 +13,7 @@
 
 #include <linux/sched.h>
 #include <linux/init.h>
+#include <linux/kernel_stat.h>
 #include <asm/processor.h>
 #include <asm/sigcontext.h>
 #include <asm/user.h>
@@ -38,17 +39,38 @@ extern void init_fpu(struct task_struct *);
 extern void kernel_fpu_begin(void);
 #define kernel_fpu_end() do { stts(); preempt_enable(); } while(0)
 
+/* We need a safe address that is cheap to find and that is already
+   in L1 during context switch. The best choices are unfortunately
+   different for UP and SMP */
+#ifdef CONFIG_SMP
+#define safe_address (__per_cpu_offset[0])
+#else
+#define safe_address (kstat_cpu(0).cpustat.user)
+#endif
+
 /*
  * These must be called with preempt disabled
  */
 static inline void __save_init_fpu( struct task_struct *tsk )
 {
+        /* Use more nops than strictly needed in case the compiler
+           varies code */
         alternative_input(
-                "fnsave %1 ; fwait ;" GENERIC_NOP2,
-                "fxsave %1 ; fnclex",
+                "fnsave %[fx] ;fwait;" GENERIC_NOP8 GENERIC_NOP4,
+                "fxsave %[fx]\n"
+                "bt $7,%[fsw] ; jnc 1f ; fnclex\n1:",
                 X86_FEATURE_FXSR,
-                "m" (tsk->thread.i387.fxsave)
-                :"memory");
+                [fx] "m" (tsk->thread.i387.fxsave),
+                [fsw] "m" (tsk->thread.i387.fxsave.swd) : "memory");
+        /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception
+           is pending.  Clear the x87 state here by setting it to fixed
+           values. safe_address is a random variable that should be in L1 */
+        alternative_input(
+                GENERIC_NOP8 GENERIC_NOP2,
+                "emms\n\t"              /* clear stack tags */
+                "fildl %[addr]",        /* set F?P to defined value */
+                X86_FEATURE_FXSAVE_LEAK,
+                [addr] "m" (safe_address));
         task_thread_info(tsk)->status &= ~TS_USEDFPU;
 }
 
diff --git a/include/asm-i386/pgtable-2level.h b/include/asm-i386/pgtable-2level.h
index 27bde973abc7..2756d4b04c27 100644
--- a/include/asm-i386/pgtable-2level.h
+++ b/include/asm-i386/pgtable-2level.h
@@ -18,6 +18,9 @@
 #define set_pte_atomic(pteptr, pteval) set_pte(pteptr,pteval)
 #define set_pmd(pmdptr, pmdval) (*(pmdptr) = (pmdval))
 
+#define pte_clear(mm,addr,xp)   do { set_pte_at(mm, addr, xp, __pte(0)); } while (0)
+#define pmd_clear(xp)   do { set_pmd(xp, __pmd(0)); } while (0)
+
 #define ptep_get_and_clear(mm,addr,xp)  __pte(xchg(&(xp)->pte_low, 0))
 #define pte_same(a, b)          ((a).pte_low == (b).pte_low)
 #define pte_page(x)             pfn_to_page(pte_pfn(x))
diff --git a/include/asm-i386/pgtable-3level.h b/include/asm-i386/pgtable-3level.h
index 36a5aa63cbbf..dccb1b3337ad 100644
--- a/include/asm-i386/pgtable-3level.h
+++ b/include/asm-i386/pgtable-3level.h
@@ -85,6 +85,26 @@ static inline void pud_clear (pud_t * pud) { }
 #define pmd_offset(pud, address) ((pmd_t *) pud_page(*(pud)) + \
                         pmd_index(address))
 
+/*
+ * For PTEs and PDEs, we must clear the P-bit first when clearing a page table
+ * entry, so clear the bottom half first and enforce ordering with a compiler
+ * barrier.
+ */
+static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
+{
+        ptep->pte_low = 0;
+        smp_wmb();
+        ptep->pte_high = 0;
+}
+
+static inline void pmd_clear(pmd_t *pmd)
+{
+        u32 *tmp = (u32 *)pmd;
+        *tmp = 0;
+        smp_wmb();
+        *(tmp + 1) = 0;
+}
+
 static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 {
         pte_t res;
diff --git a/include/asm-i386/pgtable.h b/include/asm-i386/pgtable.h
index ee056c41a9fb..672c3f76b9df 100644
--- a/include/asm-i386/pgtable.h
+++ b/include/asm-i386/pgtable.h
@@ -204,12 +204,10 @@ extern unsigned long long __PAGE_KERNEL, __PAGE_KERNEL_EXEC;
 extern unsigned long pg0[];
 
 #define pte_present(x)  ((x).pte_low & (_PAGE_PRESENT | _PAGE_PROTNONE))
-#define pte_clear(mm,addr,xp)   do { set_pte_at(mm, addr, xp, __pte(0)); } while (0)
 
 /* To avoid harmful races, pmd_none(x) should check only the lower when PAE */
 #define pmd_none(x)     (!(unsigned long)pmd_val(x))
 #define pmd_present(x)  (pmd_val(x) & _PAGE_PRESENT)
-#define pmd_clear(xp)   do { set_pmd(xp, __pmd(0)); } while (0)
 #define pmd_bad(x)      ((pmd_val(x) & (~PAGE_MASK & ~_PAGE_USER)) != _KERNPG_TABLE)
 
 
@@ -268,7 +266,7 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, unsigned long
         pte_t pte;
         if (full) {
                 pte = *ptep;
-                *ptep = __pte(0);
+                pte_clear(mm, addr, ptep);
         } else {
                 pte = ptep_get_and_clear(mm, addr, ptep);
         }
diff --git a/include/asm-i386/unistd.h b/include/asm-i386/unistd.h
index d81d6cfc1bb4..eb4b152c82fc 100644
--- a/include/asm-i386/unistd.h
+++ b/include/asm-i386/unistd.h
@@ -321,8 +321,9 @@
 #define __NR_splice             313
 #define __NR_sync_file_range    314
 #define __NR_tee                315
+#define __NR_vmsplice           316
 
-#define NR_syscalls 316
+#define NR_syscalls 317
 
 /*
  * user-visible error numbers are in the range -1 - -128: see