8 files changed, 164 insertions, 53 deletions

diff --git a/arch/x86/xen/Makefile b/arch/x86/xen/Makefile
index 7410640db173..3bb4fc21f4f2 100644
--- a/arch/x86/xen/Makefile
+++ b/arch/x86/xen/Makefile
@@ -8,6 +8,7 @@ endif
 # Make sure early boot has no stackprotector
 nostackp := $(call cc-option, -fno-stack-protector)
 CFLAGS_enlighten.o              := $(nostackp)
+CFLAGS_mmu.o                    := $(nostackp)
 
 obj-y           := enlighten.o setup.o multicalls.o mmu.o irq.o \
                         time.o xen-asm.o xen-asm_$(BITS).o \
@@ -16,3 +17,4 @@ obj-y		:= enlighten.o setup.o multicalls.o mmu.o irq.o \
 obj-$(CONFIG_SMP)               += smp.o
 obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= spinlock.o
 obj-$(CONFIG_XEN_DEBUG_FS)      += debugfs.o
+
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index e90540a46a0b..544eb7496531 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -51,6 +51,7 @@
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
 #include <asm/reboot.h>
+#include <asm/stackprotector.h>
 
 #include "xen-ops.h"
 #include "mmu.h"
@@ -215,6 +216,7 @@ static __init void xen_init_cpuid_mask(void)
                           (1 << X86_FEATURE_ACPI));  /* disable ACPI */
 
         ax = 1;
+        cx = 0;
         xen_cpuid(&ax, &bx, &cx, &dx);
 
         /* cpuid claims we support xsave; try enabling it to see what happens */
@@ -329,18 +331,28 @@ static void xen_load_gdt(const struct desc_ptr *dtr)
         unsigned long frames[pages];
         int f;
 
-        /* A GDT can be up to 64k in size, which corresponds to 8192
-           8-byte entries, or 16 4k pages.. */
+        /*
+         * A GDT can be up to 64k in size, which corresponds to 8192
+         * 8-byte entries, or 16 4k pages..
+         */
 
         BUG_ON(size > 65536);
         BUG_ON(va & ~PAGE_MASK);
 
         for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
                 int level;
-                pte_t *ptep = lookup_address(va, &level);
+                pte_t *ptep;
                 unsigned long pfn, mfn;
                 void *virt;
 
+                /*
+                 * The GDT is per-cpu and is in the percpu data area.
+                 * That can be virtually mapped, so we need to do a
+                 * page-walk to get the underlying MFN for the
+                 * hypercall.  The page can also be in the kernel's
+                 * linear range, so we need to RO that mapping too.
+                 */
+                ptep = lookup_address(va, &level);
                 BUG_ON(ptep == NULL);
 
                 pfn = pte_pfn(*ptep);
@@ -357,6 +369,44 @@ static void xen_load_gdt(const struct desc_ptr *dtr)
                 BUG();
 }
 
+/*
+ * load_gdt for early boot, when the gdt is only mapped once
+ */
+static __init void xen_load_gdt_boot(const struct desc_ptr *dtr)
+{
+        unsigned long va = dtr->address;
+        unsigned int size = dtr->size + 1;
+        unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
+        unsigned long frames[pages];
+        int f;
+
+        /*
+         * A GDT can be up to 64k in size, which corresponds to 8192
+         * 8-byte entries, or 16 4k pages..
+         */
+
+        BUG_ON(size > 65536);
+        BUG_ON(va & ~PAGE_MASK);
+
+        for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
+                pte_t pte;
+                unsigned long pfn, mfn;
+
+                pfn = virt_to_pfn(va);
+                mfn = pfn_to_mfn(pfn);
+
+                pte = pfn_pte(pfn, PAGE_KERNEL_RO);
+
+                if (HYPERVISOR_update_va_mapping((unsigned long)va, pte, 0))
+                        BUG();
+
+                frames[f] = mfn;
+        }
+
+        if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
+                BUG();
+}
+
 static void load_TLS_descriptor(struct thread_struct *t,
                                 unsigned int cpu, unsigned int i)
 {
@@ -580,6 +630,29 @@ static void xen_write_gdt_entry(struct desc_struct *dt, int entry,
         preempt_enable();
 }
 
+/*
+ * Version of write_gdt_entry for use at early boot-time needed to
+ * update an entry as simply as possible.
+ */
+static __init void xen_write_gdt_entry_boot(struct desc_struct *dt, int entry,
+                                            const void *desc, int type)
+{
+        switch (type) {
+        case DESC_LDT:
+        case DESC_TSS:
+                /* ignore */
+                break;
+
+        default: {
+                xmaddr_t maddr = virt_to_machine(&dt[entry]);
+
+                if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
+                        dt[entry] = *(struct desc_struct *)desc;
+        }
+
+        }
+}
+
 static void xen_load_sp0(struct tss_struct *tss,
                          struct thread_struct *thread)
 {
@@ -713,7 +786,7 @@ static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
         set:
                 base = ((u64)high << 32) | low;
                 if (HYPERVISOR_set_segment_base(which, base) != 0)
-                        ret = -EFAULT;
+                        ret = -EIO;
                 break;
 #endif
 
@@ -839,19 +912,9 @@ static const struct pv_info xen_info __initdata = {
 
 static const struct pv_init_ops xen_init_ops __initdata = {
         .patch = xen_patch,
-
-        .banner = xen_banner,
-        .memory_setup = xen_memory_setup,
-        .arch_setup = xen_arch_setup,
-        .post_allocator_init = xen_post_allocator_init,
 };
 
 static const struct pv_time_ops xen_time_ops __initdata = {
-        .time_init = xen_time_init,
-
-        .set_wallclock = xen_set_wallclock,
-        .get_wallclock = xen_get_wallclock,
-        .get_tsc_khz = xen_tsc_khz,
         .sched_clock = xen_sched_clock,
 };
 
@@ -917,8 +980,6 @@ static const struct pv_cpu_ops xen_cpu_ops __initdata = {
 
 static const struct pv_apic_ops xen_apic_ops __initdata = {
 #ifdef CONFIG_X86_LOCAL_APIC
-        .setup_boot_clock = paravirt_nop,
-        .setup_secondary_clock = paravirt_nop,
         .startup_ipi_hook = paravirt_nop,
 #endif
 };
@@ -964,6 +1025,23 @@ static const struct machine_ops __initdata xen_machine_ops = {
         .emergency_restart = xen_emergency_restart,
 };
 
+/*
+ * Set up the GDT and segment registers for -fstack-protector.  Until
+ * we do this, we have to be careful not to call any stack-protected
+ * function, which is most of the kernel.
+ */
+static void __init xen_setup_stackprotector(void)
+{
+        pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
+        pv_cpu_ops.load_gdt = xen_load_gdt_boot;
+
+        setup_stack_canary_segment(0);
+        switch_to_new_gdt(0);
+
+        pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry;
+        pv_cpu_ops.load_gdt = xen_load_gdt;
+}
+
 /* First C function to be called on Xen boot */
 asmlinkage void __init xen_start_kernel(void)
 {
@@ -980,16 +1058,43 @@ asmlinkage void __init xen_start_kernel(void)
         pv_time_ops = xen_time_ops;
         pv_cpu_ops = xen_cpu_ops;
         pv_apic_ops = xen_apic_ops;
-        pv_mmu_ops = xen_mmu_ops;
 
-#ifdef CONFIG_X86_64
+        x86_init.resources.memory_setup = xen_memory_setup;
+        x86_init.oem.arch_setup = xen_arch_setup;
+        x86_init.oem.banner = xen_banner;
+
+        x86_init.timers.timer_init = xen_time_init;
+        x86_init.timers.setup_percpu_clockev = x86_init_noop;
+        x86_cpuinit.setup_percpu_clockev = x86_init_noop;
+
+        x86_platform.calibrate_tsc = xen_tsc_khz;
+        x86_platform.get_wallclock = xen_get_wallclock;
+        x86_platform.set_wallclock = xen_set_wallclock;
+
         /*
-         * Setup percpu state.  We only need to do this for 64-bit
-         * because 32-bit already has %fs set properly.
+         * Set up some pagetable state before starting to set any ptes.
          */
-        load_percpu_segment(0);
-#endif
 
+        /* Prevent unwanted bits from being set in PTEs. */
+        __supported_pte_mask &= ~_PAGE_GLOBAL;
+        if (!xen_initial_domain())
+                __supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);
+
+        __supported_pte_mask |= _PAGE_IOMAP;
+
+        xen_setup_features();
+
+        /* Get mfn list */
+        if (!xen_feature(XENFEAT_auto_translated_physmap))
+                xen_build_dynamic_phys_to_machine();
+
+        /*
+         * Set up kernel GDT and segment registers, mainly so that
+         * -fstack-protector code can be executed.
+         */
+        xen_setup_stackprotector();
+
+        xen_init_mmu_ops();
         xen_init_irq_ops();
         xen_init_cpuid_mask();
 
@@ -1000,8 +1105,6 @@ asmlinkage void __init xen_start_kernel(void)
         set_xen_basic_apic_ops();
 #endif
 
-        xen_setup_features();
-
         if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
                 pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start;
                 pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit;
@@ -1018,17 +1121,8 @@ asmlinkage void __init xen_start_kernel(void)
 
         xen_smp_init();
 
-        /* Get mfn list */
-        if (!xen_feature(XENFEAT_auto_translated_physmap))
-                xen_build_dynamic_phys_to_machine();
-
         pgd = (pgd_t *)xen_start_info->pt_base;
 
-        /* Prevent unwanted bits from being set in PTEs. */
-        __supported_pte_mask &= ~_PAGE_GLOBAL;
-        if (!xen_initial_domain())
-                __supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);
-
 #ifdef CONFIG_X86_64
         /* Work out if we support NX */
         check_efer();
@@ -1059,6 +1153,7 @@ asmlinkage void __init xen_start_kernel(void)
         /* set up basic CPUID stuff */
         cpu_detect(&new_cpu_data);
         new_cpu_data.hard_math = 1;
+        new_cpu_data.wp_works_ok = 1;
         new_cpu_data.x86_capability[0] = cpuid_edx(1);
 #endif
 
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index cfd17799bd6d..9d30105a0c4a 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -1,5 +1,7 @@
 #include <linux/hardirq.h>
 
+#include <asm/x86_init.h>
+
 #include <xen/interface/xen.h>
 #include <xen/interface/sched.h>
 #include <xen/interface/vcpu.h>
@@ -112,8 +114,6 @@ static void xen_halt(void)
 }
 
 static const struct pv_irq_ops xen_irq_ops __initdata = {
-        .init_IRQ = xen_init_IRQ,
-
         .save_fl = PV_CALLEE_SAVE(xen_save_fl),
         .restore_fl = PV_CALLEE_SAVE(xen_restore_fl),
         .irq_disable = PV_CALLEE_SAVE(xen_irq_disable),
@@ -129,4 +129,5 @@ static const struct pv_irq_ops xen_irq_ops __initdata = {
 void __init xen_init_irq_ops()
 {
         pv_irq_ops = xen_irq_ops;
+        x86_init.irqs.intr_init = xen_init_IRQ;
 }
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 4ceb28581652..093dd59b5385 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1229,9 +1229,12 @@ static __init void xen_pagetable_setup_start(pgd_t *base)
 {
 }
 
+static void xen_post_allocator_init(void);
+
 static __init void xen_pagetable_setup_done(pgd_t *base)
 {
         xen_setup_shared_info();
+        xen_post_allocator_init();
 }
 
 static void xen_write_cr2(unsigned long cr2)
@@ -1841,7 +1844,7 @@ static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot)
 #endif
 }
 
-__init void xen_post_allocator_init(void)
+static __init void xen_post_allocator_init(void)
 {
         pv_mmu_ops.set_pte = xen_set_pte;
         pv_mmu_ops.set_pmd = xen_set_pmd;
@@ -1875,10 +1878,7 @@ static void xen_leave_lazy_mmu(void)
         preempt_enable();
 }
 
-const struct pv_mmu_ops xen_mmu_ops __initdata = {
-        .pagetable_setup_start = xen_pagetable_setup_start,
-        .pagetable_setup_done = xen_pagetable_setup_done,
-
+static const struct pv_mmu_ops xen_mmu_ops __initdata = {
         .read_cr2 = xen_read_cr2,
         .write_cr2 = xen_write_cr2,
 
@@ -1954,6 +1954,12 @@ const struct pv_mmu_ops xen_mmu_ops __initdata = {
         .set_fixmap = xen_set_fixmap,
 };
 
+void __init xen_init_mmu_ops(void)
+{
+        x86_init.paging.pagetable_setup_start = xen_pagetable_setup_start;
+        x86_init.paging.pagetable_setup_done = xen_pagetable_setup_done;
+        pv_mmu_ops = xen_mmu_ops;
+}
 
 #ifdef CONFIG_XEN_DEBUG_FS
 
diff --git a/arch/x86/xen/mmu.h b/arch/x86/xen/mmu.h
index da7302624897..5fe6bc7f5ecf 100644
--- a/arch/x86/xen/mmu.h
+++ b/arch/x86/xen/mmu.h
@@ -59,5 +59,5 @@ void  xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
 
 unsigned long xen_read_cr2_direct(void);
 
-extern const struct pv_mmu_ops xen_mmu_ops;
+extern void xen_init_mmu_ops(void);
 #endif  /* _XEN_MMU_H */
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 429834ec1687..fe03eeed7b48 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -236,6 +236,7 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
         ctxt->user_regs.ss = __KERNEL_DS;
 #ifdef CONFIG_X86_32
         ctxt->user_regs.fs = __KERNEL_PERCPU;
+        ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
 #else
         ctxt->gs_base_kernel = per_cpu_offset(cpu);
 #endif
diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c
index 5601506f2dd9..36a5141108df 100644
--- a/arch/x86/xen/spinlock.c
+++ b/arch/x86/xen/spinlock.c
@@ -187,7 +187,6 @@ static noinline int xen_spin_lock_slow(struct raw_spinlock *lock, bool irq_enabl
         struct xen_spinlock *prev;
         int irq = __get_cpu_var(lock_kicker_irq);
         int ret;
-        unsigned long flags;
         u64 start;
 
         /* If kicker interrupts not initialized yet, just spin */
@@ -199,16 +198,12 @@ static noinline int xen_spin_lock_slow(struct raw_spinlock *lock, bool irq_enabl
         /* announce we're spinning */
         prev = spinning_lock(xl);
 
-        flags = __raw_local_save_flags();
-        if (irq_enable) {
-                ADD_STATS(taken_slow_irqenable, 1);
-                raw_local_irq_enable();
-        }
-
         ADD_STATS(taken_slow, 1);
         ADD_STATS(taken_slow_nested, prev != NULL);
 
         do {
+                unsigned long flags;
+
                 /* clear pending */
                 xen_clear_irq_pending(irq);
 
@@ -228,6 +223,12 @@ static noinline int xen_spin_lock_slow(struct raw_spinlock *lock, bool irq_enabl
                         goto out;
                 }
 
+                flags = __raw_local_save_flags();
+                if (irq_enable) {
+                        ADD_STATS(taken_slow_irqenable, 1);
+                        raw_local_irq_enable();
+                }
+
                 /*
                  * Block until irq becomes pending.  If we're
                  * interrupted at this point (after the trylock but
@@ -238,13 +239,15 @@ static noinline int xen_spin_lock_slow(struct raw_spinlock *lock, bool irq_enabl
                  * pending.
                  */
                 xen_poll_irq(irq);
+
+                raw_local_irq_restore(flags);
+
                 ADD_STATS(taken_slow_spurious, !xen_test_irq_pending(irq));
         } while (!xen_test_irq_pending(irq)); /* check for spurious wakeups */
 
         kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
 
 out:
-        raw_local_irq_restore(flags);
         unspinning_lock(xl, prev);
         spin_time_accum_blocked(start);
 
@@ -323,8 +326,13 @@ static void xen_spin_unlock(struct raw_spinlock *lock)
         smp_wmb();              /* make sure no writes get moved after unlock */
         xl->lock = 0;           /* release lock */
 
-        /* make sure unlock happens before kick */
-        barrier();
+        /*
+         * Make sure unlock happens before checking for waiting
+         * spinners.  We need a strong barrier to enforce the
+         * write-read ordering to different memory locations, as the
+         * CPU makes no implied guarantees about their ordering.
+         */
+        mb();
 
         if (unlikely(xl->spinners))
                 xen_spin_unlock_slow(xl);
diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h
index 22494fd4c9b5..355fa6b99c9c 100644
--- a/arch/x86/xen/xen-ops.h
+++ b/arch/x86/xen/xen-ops.h
@@ -30,8 +30,6 @@ pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn);
 void xen_ident_map_ISA(void);
 void xen_reserve_top(void);
 
-void xen_post_allocator_init(void);
-
 char * __init xen_memory_setup(void);
 void __init xen_arch_setup(void);
 void __init xen_init_IRQ(void);