11 files changed, 168 insertions, 42 deletions

diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index bacf5deeec2d..aa89387006fe 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -18,6 +18,8 @@
 #include <asm/ia32.h>
 #include <asm/bootparam.h>
 
+#include <xen/interface/xen.h>
+
 #define __NO_STUBS 1
 #undef __SYSCALL
 #undef _ASM_X86_64_UNISTD_H_
@@ -131,5 +133,14 @@ int main(void)
         OFFSET(BP_loadflags, boot_params, hdr.loadflags);
         OFFSET(BP_hardware_subarch, boot_params, hdr.hardware_subarch);
         OFFSET(BP_version, boot_params, hdr.version);
+
+        BLANK();
+        DEFINE(PAGE_SIZE_asm, PAGE_SIZE);
+#ifdef CONFIG_XEN
+        BLANK();
+        OFFSET(XEN_vcpu_info_mask, vcpu_info, evtchn_upcall_mask);
+        OFFSET(XEN_vcpu_info_pending, vcpu_info, evtchn_upcall_pending);
+#undef ENTRY
+#endif
         return 0;
 }
diff --git a/arch/x86/kernel/cpu/amd_64.c b/arch/x86/kernel/cpu/amd_64.c
index 7c36fb8a28d4..d1692b2a41ff 100644
--- a/arch/x86/kernel/cpu/amd_64.c
+++ b/arch/x86/kernel/cpu/amd_64.c
@@ -115,6 +115,8 @@ static void __cpuinit early_init_amd(struct cpuinfo_x86 *c)
         /* c->x86_power is 8000_0007 edx. Bit 8 is constant TSC */
         if (c->x86_power & (1<<8))
                 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
+
+        set_cpu_cap(c, X86_FEATURE_SYSCALL32);
 }
 
 static void __cpuinit init_amd(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/common_64.c b/arch/x86/kernel/cpu/common_64.c
index 7b8cc72feb40..736f50fa433d 100644
--- a/arch/x86/kernel/cpu/common_64.c
+++ b/arch/x86/kernel/cpu/common_64.c
@@ -16,6 +16,7 @@
 #include <asm/i387.h>
 #include <asm/msr.h>
 #include <asm/io.h>
+#include <asm/linkage.h>
 #include <asm/mmu_context.h>
 #include <asm/mtrr.h>
 #include <asm/mce.h>
@@ -316,9 +317,6 @@ static void __cpuinit early_identify_cpu(struct cpuinfo_x86 *c)
                 c->x86_phys_bits = eax & 0xff;
         }
 
-        /* Assume all 64-bit CPUs support 32-bit syscall */
-        set_cpu_cap(c, X86_FEATURE_SYSCALL32);
-
         if (c->x86_vendor != X86_VENDOR_UNKNOWN &&
             cpu_devs[c->x86_vendor]->c_early_init)
                 cpu_devs[c->x86_vendor]->c_early_init(c);
@@ -517,8 +515,7 @@ void pda_init(int cpu)
 }
 
 char boot_exception_stacks[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ +
-                           DEBUG_STKSZ]
-__attribute__((section(".bss.page_aligned")));
+                           DEBUG_STKSZ] __page_aligned_bss;
 
 extern asmlinkage void ignore_sysret(void);
 
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index ae63e584c340..80d5663db3bc 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -1189,6 +1189,7 @@ END(device_not_available)
         /* runs on exception stack */
 KPROBE_ENTRY(debug)
         INTR_FRAME
+        PARAVIRT_ADJUST_EXCEPTION_FRAME
         pushq $0
         CFI_ADJUST_CFA_OFFSET 8         
         paranoidentry do_debug, DEBUG_STACK
@@ -1198,6 +1199,7 @@ KPROBE_END(debug)
         /* runs on exception stack */   
 KPROBE_ENTRY(nmi)
         INTR_FRAME
+        PARAVIRT_ADJUST_EXCEPTION_FRAME
         pushq $-1
         CFI_ADJUST_CFA_OFFSET 8
         paranoidentry do_nmi, 0, 0
@@ -1211,6 +1213,7 @@ KPROBE_END(nmi)
 
 KPROBE_ENTRY(int3)
         INTR_FRAME
+        PARAVIRT_ADJUST_EXCEPTION_FRAME
         pushq $0
         CFI_ADJUST_CFA_OFFSET 8
         paranoidentry do_int3, DEBUG_STACK
@@ -1237,6 +1240,7 @@ END(coprocessor_segment_overrun)
         /* runs on exception stack */
 ENTRY(double_fault)
         XCPT_FRAME
+        PARAVIRT_ADJUST_EXCEPTION_FRAME
         paranoidentry do_double_fault
         jmp paranoid_exit1
         CFI_ENDPROC
@@ -1253,6 +1257,7 @@ END(segment_not_present)
         /* runs on exception stack */
 ENTRY(stack_segment)
         XCPT_FRAME
+        PARAVIRT_ADJUST_EXCEPTION_FRAME
         paranoidentry do_stack_segment
         jmp paranoid_exit1
         CFI_ENDPROC
@@ -1278,6 +1283,7 @@ END(spurious_interrupt_bug)
         /* runs on exception stack */
 ENTRY(machine_check)
         INTR_FRAME
+        PARAVIRT_ADJUST_EXCEPTION_FRAME
         pushq $0
         CFI_ADJUST_CFA_OFFSET 8 
         paranoidentry do_machine_check
@@ -1312,3 +1318,103 @@ KPROBE_ENTRY(ignore_sysret)
         sysret
         CFI_ENDPROC
 ENDPROC(ignore_sysret)
+
+#ifdef CONFIG_XEN
+ENTRY(xen_hypervisor_callback)
+        zeroentry xen_do_hypervisor_callback
+END(xen_hypervisor_callback)
+
+/*
+# A note on the "critical region" in our callback handler.
+# We want to avoid stacking callback handlers due to events occurring
+# during handling of the last event. To do this, we keep events disabled
+# until we've done all processing. HOWEVER, we must enable events before
+# popping the stack frame (can't be done atomically) and so it would still
+# be possible to get enough handler activations to overflow the stack.
+# Although unlikely, bugs of that kind are hard to track down, so we'd
+# like to avoid the possibility.
+# So, on entry to the handler we detect whether we interrupted an
+# existing activation in its critical region -- if so, we pop the current
+# activation and restart the handler using the previous one.
+*/
+ENTRY(xen_do_hypervisor_callback)   # do_hypervisor_callback(struct *pt_regs)
+        CFI_STARTPROC
+/* Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
+   see the correct pointer to the pt_regs */
+        movq %rdi, %rsp            # we don't return, adjust the stack frame
+        CFI_ENDPROC
+        CFI_DEFAULT_STACK
+11:     incl %gs:pda_irqcount
+        movq %rsp,%rbp
+        CFI_DEF_CFA_REGISTER rbp
+        cmovzq %gs:pda_irqstackptr,%rsp
+        pushq %rbp                      # backlink for old unwinder
+        call xen_evtchn_do_upcall
+        popq %rsp
+        CFI_DEF_CFA_REGISTER rsp
+        decl %gs:pda_irqcount
+        jmp  error_exit
+        CFI_ENDPROC
+END(do_hypervisor_callback)
+
+/*
+# Hypervisor uses this for application faults while it executes.
+# We get here for two reasons:
+#  1. Fault while reloading DS, ES, FS or GS
+#  2. Fault while executing IRET
+# Category 1 we do not need to fix up as Xen has already reloaded all segment
+# registers that could be reloaded and zeroed the others.
+# Category 2 we fix up by killing the current process. We cannot use the
+# normal Linux return path in this case because if we use the IRET hypercall
+# to pop the stack frame we end up in an infinite loop of failsafe callbacks.
+# We distinguish between categories by comparing each saved segment register
+# with its current contents: any discrepancy means we in category 1.
+*/
+ENTRY(xen_failsafe_callback)
+        framesz = (RIP-0x30)    /* workaround buggy gas */
+        _frame framesz
+        CFI_REL_OFFSET rcx, 0
+        CFI_REL_OFFSET r11, 8
+        movw %ds,%cx
+        cmpw %cx,0x10(%rsp)
+        CFI_REMEMBER_STATE
+        jne 1f
+        movw %es,%cx
+        cmpw %cx,0x18(%rsp)
+        jne 1f
+        movw %fs,%cx
+        cmpw %cx,0x20(%rsp)
+        jne 1f
+        movw %gs,%cx
+        cmpw %cx,0x28(%rsp)
+        jne 1f
+        /* All segments match their saved values => Category 2 (Bad IRET). */
+        movq (%rsp),%rcx
+        CFI_RESTORE rcx
+        movq 8(%rsp),%r11
+        CFI_RESTORE r11
+        addq $0x30,%rsp
+        CFI_ADJUST_CFA_OFFSET -0x30
+        pushq $0
+        CFI_ADJUST_CFA_OFFSET 8
+        pushq %r11
+        CFI_ADJUST_CFA_OFFSET 8
+        pushq %rcx
+        CFI_ADJUST_CFA_OFFSET 8
+        jmp general_protection
+        CFI_RESTORE_STATE
+1:      /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
+        movq (%rsp),%rcx
+        CFI_RESTORE rcx
+        movq 8(%rsp),%r11
+        CFI_RESTORE r11
+        addq $0x30,%rsp
+        CFI_ADJUST_CFA_OFFSET -0x30
+        pushq $0
+        CFI_ADJUST_CFA_OFFSET 8
+        SAVE_ALL
+        jmp error_exit
+        CFI_ENDPROC
+END(xen_failsafe_callback)
+
+#endif /* CONFIG_XEN */
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index c97819829146..1b318e903bf6 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -39,6 +39,13 @@ static struct x8664_pda *__cpu_pda[NR_CPUS] __initdata;
 static struct x8664_pda *__cpu_pda[NR_CPUS] __read_mostly;
 #endif
 
+void __init x86_64_init_pda(void)
+{
+        _cpu_pda = __cpu_pda;
+        cpu_pda(0) = &_boot_cpu_pda;
+        pda_init(0);
+}
+
 static void __init zap_identity_mappings(void)
 {
         pgd_t *pgd = pgd_offset_k(0UL);
@@ -102,9 +109,7 @@ void __init x86_64_start_kernel(char * real_mode_data)
 
         early_printk("Kernel alive\n");
 
-        _cpu_pda = __cpu_pda;
-        cpu_pda(0) = &_boot_cpu_pda;
-        pda_init(0);
+        x86_64_init_pda();
 
         early_printk("Kernel really alive\n");
 
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index b07ac7b217cb..db3280afe886 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -407,6 +407,7 @@ ENTRY(phys_base)
         /* This must match the first entry in level2_kernel_pgt */
         .quad   0x0000000000000000
 
+#include "../../x86/xen/xen-head.S"
         
         .section .bss, "aw", @nobits
         .align L1_CACHE_BYTES
diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index 47a6f6f12478..1cf8c1fcc088 100644
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -83,11 +83,8 @@ union irq_ctx {
 static union irq_ctx *hardirq_ctx[NR_CPUS] __read_mostly;
 static union irq_ctx *softirq_ctx[NR_CPUS] __read_mostly;
 
-static char softirq_stack[NR_CPUS * THREAD_SIZE]
-                __attribute__((__section__(".bss.page_aligned")));
-
-static char hardirq_stack[NR_CPUS * THREAD_SIZE]
-                __attribute__((__section__(".bss.page_aligned")));
+static char softirq_stack[NR_CPUS * THREAD_SIZE] __page_aligned_bss;
+static char hardirq_stack[NR_CPUS * THREAD_SIZE] __page_aligned_bss;
 
 static void call_on_stack(void *func, void *stack)
 {
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index e0f571d58c19..2963ab5d91ee 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -29,6 +29,7 @@
 #include <asm/desc.h>
 #include <asm/setup.h>
 #include <asm/arch_hooks.h>
+#include <asm/pgtable.h>
 #include <asm/time.h>
 #include <asm/pgalloc.h>
 #include <asm/irq.h>
@@ -373,6 +374,9 @@ struct pv_mmu_ops pv_mmu_ops = {
 #ifndef CONFIG_X86_64
         .pagetable_setup_start = native_pagetable_setup_start,
         .pagetable_setup_done = native_pagetable_setup_done,
+#else
+        .pagetable_setup_start = paravirt_nop,
+        .pagetable_setup_done = paravirt_nop,
 #endif
 
         .read_cr2 = native_read_cr2,
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index a8e53626ac9a..e8a8e1b99817 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -537,8 +537,8 @@ static inline void __switch_to_xtra(struct task_struct *prev_p,
 struct task_struct *
 __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 {
-        struct thread_struct *prev = &prev_p->thread,
-                                 *next = &next_p->thread;
+        struct thread_struct *prev = &prev_p->thread;
+        struct thread_struct *next = &next_p->thread;
         int cpu = smp_processor_id();
         struct tss_struct *tss = &per_cpu(init_tss, cpu);
         unsigned fsindex, gsindex;
@@ -586,35 +586,34 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 
         /* 
          * Switch FS and GS.
+         *
+         * Segment register != 0 always requires a reload.  Also
+         * reload when it has changed.  When prev process used 64bit
+         * base always reload to avoid an information leak.
          */
-        { 
-                /* segment register != 0 always requires a reload. 
-                   also reload when it has changed. 
-                   when prev process used 64bit base always reload
-                   to avoid an information leak. */
-                if (unlikely(fsindex | next->fsindex | prev->fs)) {
-                        loadsegment(fs, next->fsindex);
-                        /* check if the user used a selector != 0
-                         * if yes clear 64bit base, since overloaded base
-                         * is always mapped to the Null selector
-                         */
-                        if (fsindex)
+        if (unlikely(fsindex | next->fsindex | prev->fs)) {
+                loadsegment(fs, next->fsindex);
+                /* 
+                 * Check if the user used a selector != 0; if yes
+                 *  clear 64bit base, since overloaded base is always
+                 *  mapped to the Null selector
+                 */
+                if (fsindex)
                         prev->fs = 0;                           
-                }
-                /* when next process has a 64bit base use it */
-                if (next->fs) 
-                        wrmsrl(MSR_FS_BASE, next->fs); 
-                prev->fsindex = fsindex;
-
-                if (unlikely(gsindex | next->gsindex | prev->gs)) {
-                        load_gs_index(next->gsindex);
-                        if (gsindex)
+        }
+        /* when next process has a 64bit base use it */
+        if (next->fs)
+                wrmsrl(MSR_FS_BASE, next->fs);
+        prev->fsindex = fsindex;
+
+        if (unlikely(gsindex | next->gsindex | prev->gs)) {
+                load_gs_index(next->gsindex);
+                if (gsindex)
                         prev->gs = 0;                           
-                }
-                if (next->gs)
-                        wrmsrl(MSR_KERNEL_GS_BASE, next->gs); 
-                prev->gsindex = gsindex;
         }
+        if (next->gs)
+                wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
+        prev->gsindex = gsindex;
 
         /* Must be after DS reload */
         unlazy_fpu(prev_p);
@@ -627,7 +626,8 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
         write_pda(pcurrent, next_p); 
 
         write_pda(kernelstack,
-        (unsigned long)task_stack_page(next_p) + THREAD_SIZE - PDA_STACKOFFSET);
+                  (unsigned long)task_stack_page(next_p) +
+                  THREAD_SIZE - PDA_STACKOFFSET);
 #ifdef CONFIG_CC_STACKPROTECTOR
         write_pda(stack_canary, next_p->stack_canary);
         /*
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 531b55b8e81a..c9010f82141d 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -824,7 +824,10 @@ void __init setup_arch(char **cmdline_p)
         vmi_init();
 #endif
 
+        paravirt_pagetable_setup_start(swapper_pg_dir);
         paging_init();
+        paravirt_pagetable_setup_done(swapper_pg_dir);
+        paravirt_post_allocator_init();
 
 #ifdef CONFIG_X86_64
         map_vsyscall();
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 687376ab07e8..1deb3b624a79 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -768,7 +768,7 @@ static void __cpuinit do_fork_idle(struct work_struct *work)
  *
  * Must be called after the _cpu_pda pointer table is initialized.
  */
-static int __cpuinit get_local_pda(int cpu)
+int __cpuinit get_local_pda(int cpu)
 {
         struct x8664_pda *oldpda, *newpda;
         unsigned long size = sizeof(struct x8664_pda);