Merge branch 'x86-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 mm updates from Ingo Molnar:
 "The main changes in this cycle were:

   - Continued work to add support for 5-level paging provided by future
     Intel CPUs. In particular we switch the x86 GUP code to the generic
     implementation. (Kirill A. Shutemov)

   - Continued work to add PCID CPU support to native kernels as well.
     In this round most of the focus is on reworking/refreshing the TLB
     flush infrastructure for the upcoming PCID changes. (Andy
     Lutomirski)"

* 'x86-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (34 commits)
  x86/mm: Delete a big outdated comment about TLB flushing
  x86/mm: Don't reenter flush_tlb_func_common()
  x86/KASLR: Fix detection 32/64 bit bootloaders for 5-level paging
  x86/ftrace: Exclude functions in head64.c from function-tracing
  x86/mmap, ASLR: Do not treat unlimited-stack tasks as legacy mmap
  x86/mm: Remove reset_lazy_tlbstate()
  x86/ldt: Simplify the LDT switching logic
  x86/boot/64: Put __startup_64() into .head.text
  x86/mm: Add support for 5-level paging for KASLR
  x86/mm: Make kernel_physical_mapping_init() support 5-level paging
  x86/mm: Add sync_global_pgds() for configuration with 5-level paging
  x86/boot/64: Add support of additional page table level during early boot
  x86/boot/64: Rename init_level4_pgt and early_level4_pgt
  x86/boot/64: Rewrite startup_64() in C
  x86/boot/compressed: Enable 5-level paging during decompression stage
  x86/boot/efi: Define __KERNEL32_CS GDT on 64-bit configurations
  x86/boot/efi: Fix __KERNEL_CS definition of GDT entry on 64-bit configurations
  x86/boot/efi: Cleanup initialization of GDT entries
  x86/asm: Fix comment in return_from_SYSCALL_64()
  x86/mm/gup: Switch GUP to the generic get_user_page_fast() implementation
  ...

11 files changed, 198 insertions, 150 deletions

diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 3c7c419c4e3e..a01892bdd61a 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -18,6 +18,7 @@ CFLAGS_REMOVE_pvclock.o = -pg
 CFLAGS_REMOVE_kvmclock.o = -pg
 CFLAGS_REMOVE_ftrace.o = -pg
 CFLAGS_REMOVE_early_printk.o = -pg
+CFLAGS_REMOVE_head64.o = -pg
 endif
 
 KASAN_SANITIZE_head$(BITS).o                            := n
diff --git a/arch/x86/kernel/espfix_64.c b/arch/x86/kernel/espfix_64.c
index 8e598a1ad986..6b91e2eb8d3f 100644
--- a/arch/x86/kernel/espfix_64.c
+++ b/arch/x86/kernel/espfix_64.c
@@ -125,7 +125,7 @@ void __init init_espfix_bsp(void)
         p4d_t *p4d;
 
         /* Install the espfix pud into the kernel page directory */
-        pgd = &init_level4_pgt[pgd_index(ESPFIX_BASE_ADDR)];
+        pgd = &init_top_pgt[pgd_index(ESPFIX_BASE_ADDR)];
         p4d = p4d_alloc(&init_mm, pgd, ESPFIX_BASE_ADDR);
         p4d_populate(&init_mm, p4d, espfix_pud_page);
 
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 43b7002f44fb..46c3c73e7f43 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -33,17 +33,120 @@
 /*
  * Manage page tables very early on.
  */
-extern pgd_t early_level4_pgt[PTRS_PER_PGD];
+extern pgd_t early_top_pgt[PTRS_PER_PGD];
 extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
-static unsigned int __initdata next_early_pgt = 2;
+static unsigned int __initdata next_early_pgt;
 pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
 
+#define __head  __section(.head.text)
+
+static void __head *fixup_pointer(void *ptr, unsigned long physaddr)
+{
+        return ptr - (void *)_text + (void *)physaddr;
+}
+
+void __head __startup_64(unsigned long physaddr)
+{
+        unsigned long load_delta, *p;
+        pgdval_t *pgd;
+        p4dval_t *p4d;
+        pudval_t *pud;
+        pmdval_t *pmd, pmd_entry;
+        int i;
+
+        /* Is the address too large? */
+        if (physaddr >> MAX_PHYSMEM_BITS)
+                for (;;);
+
+        /*
+         * Compute the delta between the address I am compiled to run at
+         * and the address I am actually running at.
+         */
+        load_delta = physaddr - (unsigned long)(_text - __START_KERNEL_map);
+
+        /* Is the address not 2M aligned? */
+        if (load_delta & ~PMD_PAGE_MASK)
+                for (;;);
+
+        /* Fixup the physical addresses in the page table */
+
+        pgd = fixup_pointer(&early_top_pgt, physaddr);
+        pgd[pgd_index(__START_KERNEL_map)] += load_delta;
+
+        if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
+                p4d = fixup_pointer(&level4_kernel_pgt, physaddr);
+                p4d[511] += load_delta;
+        }
+
+        pud = fixup_pointer(&level3_kernel_pgt, physaddr);
+        pud[510] += load_delta;
+        pud[511] += load_delta;
+
+        pmd = fixup_pointer(level2_fixmap_pgt, physaddr);
+        pmd[506] += load_delta;
+
+        /*
+         * Set up the identity mapping for the switchover.  These
+         * entries should *NOT* have the global bit set!  This also
+         * creates a bunch of nonsense entries but that is fine --
+         * it avoids problems around wraparound.
+         */
+
+        pud = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
+        pmd = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
+
+        if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
+                p4d = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
+
+                i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
+                pgd[i + 0] = (pgdval_t)p4d + _KERNPG_TABLE;
+                pgd[i + 1] = (pgdval_t)p4d + _KERNPG_TABLE;
+
+                i = (physaddr >> P4D_SHIFT) % PTRS_PER_P4D;
+                p4d[i + 0] = (pgdval_t)pud + _KERNPG_TABLE;
+                p4d[i + 1] = (pgdval_t)pud + _KERNPG_TABLE;
+        } else {
+                i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
+                pgd[i + 0] = (pgdval_t)pud + _KERNPG_TABLE;
+                pgd[i + 1] = (pgdval_t)pud + _KERNPG_TABLE;
+        }
+
+        i = (physaddr >> PUD_SHIFT) % PTRS_PER_PUD;
+        pud[i + 0] = (pudval_t)pmd + _KERNPG_TABLE;
+        pud[i + 1] = (pudval_t)pmd + _KERNPG_TABLE;
+
+        pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
+        pmd_entry +=  physaddr;
+
+        for (i = 0; i < DIV_ROUND_UP(_end - _text, PMD_SIZE); i++) {
+                int idx = i + (physaddr >> PMD_SHIFT) % PTRS_PER_PMD;
+                pmd[idx] = pmd_entry + i * PMD_SIZE;
+        }
+
+        /*
+         * Fixup the kernel text+data virtual addresses. Note that
+         * we might write invalid pmds, when the kernel is relocated
+         * cleanup_highmap() fixes this up along with the mappings
+         * beyond _end.
+         */
+
+        pmd = fixup_pointer(level2_kernel_pgt, physaddr);
+        for (i = 0; i < PTRS_PER_PMD; i++) {
+                if (pmd[i] & _PAGE_PRESENT)
+                        pmd[i] += load_delta;
+        }
+
+        /* Fixup phys_base */
+        p = fixup_pointer(&phys_base, physaddr);
+        *p += load_delta;
+}
+
 /* Wipe all early page tables except for the kernel symbol map */
 static void __init reset_early_page_tables(void)
 {
-        memset(early_level4_pgt, 0, sizeof(pgd_t)*(PTRS_PER_PGD-1));
+        memset(early_top_pgt, 0, sizeof(pgd_t)*(PTRS_PER_PGD-1));
         next_early_pgt = 0;
-        write_cr3(__pa_nodebug(early_level4_pgt));
+        write_cr3(__pa_nodebug(early_top_pgt));
 }
 
 /* Create a new PMD entry */
@@ -51,15 +154,16 @@ int __init early_make_pgtable(unsigned long address)
 {
         unsigned long physaddr = address - __PAGE_OFFSET;
         pgdval_t pgd, *pgd_p;
+        p4dval_t p4d, *p4d_p;
         pudval_t pud, *pud_p;
         pmdval_t pmd, *pmd_p;
 
         /* Invalid address or early pgt is done ?  */
-        if (physaddr >= MAXMEM || read_cr3() != __pa_nodebug(early_level4_pgt))
+        if (physaddr >= MAXMEM || read_cr3_pa() != __pa_nodebug(early_top_pgt))
                 return -1;
 
 again:
-        pgd_p = &early_level4_pgt[pgd_index(address)].pgd;
+        pgd_p = &early_top_pgt[pgd_index(address)].pgd;
         pgd = *pgd_p;
 
         /*
@@ -67,8 +171,25 @@ again:
          * critical -- __PAGE_OFFSET would point us back into the dynamic
          * range and we might end up looping forever...
          */
-        if (pgd)
-                pud_p = (pudval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
+        if (!IS_ENABLED(CONFIG_X86_5LEVEL))
+                p4d_p = pgd_p;
+        else if (pgd)
+                p4d_p = (p4dval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
+        else {
+                if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
+                        reset_early_page_tables();
+                        goto again;
+                }
+
+                p4d_p = (p4dval_t *)early_dynamic_pgts[next_early_pgt++];
+                memset(p4d_p, 0, sizeof(*p4d_p) * PTRS_PER_P4D);
+                *pgd_p = (pgdval_t)p4d_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
+        }
+        p4d_p += p4d_index(address);
+        p4d = *p4d_p;
+
+        if (p4d)
+                pud_p = (pudval_t *)((p4d & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
         else {
                 if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
                         reset_early_page_tables();
@@ -77,7 +198,7 @@ again:
 
                 pud_p = (pudval_t *)early_dynamic_pgts[next_early_pgt++];
                 memset(pud_p, 0, sizeof(*pud_p) * PTRS_PER_PUD);
-                *pgd_p = (pgdval_t)pud_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
+                *p4d_p = (p4dval_t)pud_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
         }
         pud_p += pud_index(address);
         pud = *pud_p;
@@ -156,7 +277,7 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
 
         clear_bss();
 
-        clear_page(init_level4_pgt);
+        clear_page(init_top_pgt);
 
         kasan_early_init();
 
@@ -171,8 +292,8 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
          */
         load_ucode_bsp();
 
-        /* set init_level4_pgt kernel high mapping*/
-        init_level4_pgt[511] = early_level4_pgt[511];
+        /* set init_top_pgt kernel high mapping*/
+        init_top_pgt[511] = early_top_pgt[511];
 
         x86_64_start_reservations(real_mode_data);
 }
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index ac9d327d2e42..6225550883df 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -37,10 +37,11 @@
  *
  */
 
+#define p4d_index(x)    (((x) >> P4D_SHIFT) & (PTRS_PER_P4D-1))
 #define pud_index(x)    (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
 
-L4_PAGE_OFFSET = pgd_index(__PAGE_OFFSET_BASE)
-L4_START_KERNEL = pgd_index(__START_KERNEL_map)
+PGD_PAGE_OFFSET = pgd_index(__PAGE_OFFSET_BASE)
+PGD_START_KERNEL = pgd_index(__START_KERNEL_map)
 L3_START_KERNEL = pud_index(__START_KERNEL_map)
 
         .text
@@ -72,101 +73,12 @@ startup_64:
         /* Sanitize CPU configuration */
         call verify_cpu
 
-        /*
-         * Compute the delta between the address I am compiled to run at and the
-         * address I am actually running at.
-         */
-        leaq    _text(%rip), %rbp
-        subq    $_text - __START_KERNEL_map, %rbp
-
-        /* Is the address not 2M aligned? */
-        testl   $~PMD_PAGE_MASK, %ebp
-        jnz     bad_address
-
-        /*
-         * Is the address too large?
-         */
-        leaq    _text(%rip), %rax
-        shrq    $MAX_PHYSMEM_BITS, %rax
-        jnz     bad_address
-
-        /*
-         * Fixup the physical addresses in the page table
-         */
-        addq    %rbp, early_level4_pgt + (L4_START_KERNEL*8)(%rip)
-
-        addq    %rbp, level3_kernel_pgt + (510*8)(%rip)
-        addq    %rbp, level3_kernel_pgt + (511*8)(%rip)
-
-        addq    %rbp, level2_fixmap_pgt + (506*8)(%rip)
-
-        /*
-         * Set up the identity mapping for the switchover.  These
-         * entries should *NOT* have the global bit set!  This also
-         * creates a bunch of nonsense entries but that is fine --
-         * it avoids problems around wraparound.
-         */
         leaq    _text(%rip), %rdi
-        leaq    early_level4_pgt(%rip), %rbx
-
-        movq    %rdi, %rax
-        shrq    $PGDIR_SHIFT, %rax
-
-        leaq    (PAGE_SIZE + _KERNPG_TABLE)(%rbx), %rdx
-        movq    %rdx, 0(%rbx,%rax,8)
-        movq    %rdx, 8(%rbx,%rax,8)
-
-        addq    $PAGE_SIZE, %rdx
-        movq    %rdi, %rax
-        shrq    $PUD_SHIFT, %rax
-        andl    $(PTRS_PER_PUD-1), %eax
-        movq    %rdx, PAGE_SIZE(%rbx,%rax,8)
-        incl    %eax
-        andl    $(PTRS_PER_PUD-1), %eax
-        movq    %rdx, PAGE_SIZE(%rbx,%rax,8)
-
-        addq    $PAGE_SIZE * 2, %rbx
-        movq    %rdi, %rax
-        shrq    $PMD_SHIFT, %rdi
-        addq    $(__PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL), %rax
-        leaq    (_end - 1)(%rip), %rcx
-        shrq    $PMD_SHIFT, %rcx
-        subq    %rdi, %rcx
-        incl    %ecx
-
-1:
-        andq    $(PTRS_PER_PMD - 1), %rdi
-        movq    %rax, (%rbx,%rdi,8)
-        incq    %rdi
-        addq    $PMD_SIZE, %rax
-        decl    %ecx
-        jnz     1b
-
-        test %rbp, %rbp
-        jz .Lskip_fixup
+        pushq   %rsi
+        call    __startup_64
+        popq    %rsi
 
-        /*
-         * Fixup the kernel text+data virtual addresses. Note that
-         * we might write invalid pmds, when the kernel is relocated
-         * cleanup_highmap() fixes this up along with the mappings
-         * beyond _end.
-         */
-        leaq    level2_kernel_pgt(%rip), %rdi
-        leaq    PAGE_SIZE(%rdi), %r8
-        /* See if it is a valid page table entry */
-1:      testb   $_PAGE_PRESENT, 0(%rdi)
-        jz      2f
-        addq    %rbp, 0(%rdi)
-        /* Go to the next page */
-2:      addq    $8, %rdi
-        cmp     %r8, %rdi
-        jne     1b
-
-        /* Fixup phys_base */
-        addq    %rbp, phys_base(%rip)
-
-.Lskip_fixup:
-        movq    $(early_level4_pgt - __START_KERNEL_map), %rax
+        movq    $(early_top_pgt - __START_KERNEL_map), %rax
         jmp 1f
 ENTRY(secondary_startup_64)
         /*
@@ -186,14 +98,17 @@ ENTRY(secondary_startup_64)
         /* Sanitize CPU configuration */
         call verify_cpu
 
-        movq    $(init_level4_pgt - __START_KERNEL_map), %rax
+        movq    $(init_top_pgt - __START_KERNEL_map), %rax
 1:
 
-        /* Enable PAE mode and PGE */
+        /* Enable PAE mode, PGE and LA57 */
         movl    $(X86_CR4_PAE | X86_CR4_PGE), %ecx
+#ifdef CONFIG_X86_5LEVEL
+        orl     $X86_CR4_LA57, %ecx
+#endif
         movq    %rcx, %cr4
 
-        /* Setup early boot stage 4 level pagetables. */
+        /* Setup early boot stage 4-/5-level pagetables. */
         addq    phys_base(%rip), %rax
         movq    %rax, %cr3
 
@@ -417,9 +332,13 @@ GLOBAL(name)
         .endr
 
         __INITDATA
-NEXT_PAGE(early_level4_pgt)
+NEXT_PAGE(early_top_pgt)
         .fill   511,8,0
+#ifdef CONFIG_X86_5LEVEL
+        .quad   level4_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
+#else
         .quad   level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
+#endif
 
 NEXT_PAGE(early_dynamic_pgts)
         .fill   512*EARLY_DYNAMIC_PAGE_TABLES,8,0
@@ -427,14 +346,14 @@ NEXT_PAGE(early_dynamic_pgts)
         .data
 
 #ifndef CONFIG_XEN
-NEXT_PAGE(init_level4_pgt)
+NEXT_PAGE(init_top_pgt)
         .fill   512,8,0
 #else
-NEXT_PAGE(init_level4_pgt)
+NEXT_PAGE(init_top_pgt)
         .quad   level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
-        .org    init_level4_pgt + L4_PAGE_OFFSET*8, 0
+        .org    init_top_pgt + PGD_PAGE_OFFSET*8, 0
         .quad   level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
-        .org    init_level4_pgt + L4_START_KERNEL*8, 0
+        .org    init_top_pgt + PGD_START_KERNEL*8, 0
         /* (2^48-(2*1024*1024*1024))/(2^39) = 511 */
         .quad   level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
 
@@ -448,6 +367,12 @@ NEXT_PAGE(level2_ident_pgt)
         PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD)
 #endif
 
+#ifdef CONFIG_X86_5LEVEL
+NEXT_PAGE(level4_kernel_pgt)
+        .fill   511,8,0
+        .quad   level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
+#endif
+
 NEXT_PAGE(level3_kernel_pgt)
         .fill   L3_START_KERNEL,8,0
         /* (2^48-(2*1024*1024*1024)-((2^39)*511))/(2^30) = 510 */
diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c
index d4a15831ac58..a870910c8565 100644
--- a/arch/x86/kernel/ldt.c
+++ b/arch/x86/kernel/ldt.c
@@ -22,24 +22,25 @@
 #include <asm/syscalls.h>
 
 /* context.lock is held for us, so we don't need any locking. */
-static void flush_ldt(void *current_mm)
+static void flush_ldt(void *__mm)
 {
+        struct mm_struct *mm = __mm;
         mm_context_t *pc;
 
-        if (current->active_mm != current_mm)
+        if (this_cpu_read(cpu_tlbstate.loaded_mm) != mm)
                 return;
 
-        pc = &current->active_mm->context;
-        set_ldt(pc->ldt->entries, pc->ldt->size);
+        pc = &mm->context;
+        set_ldt(pc->ldt->entries, pc->ldt->nr_entries);
 }
 
 /* The caller must call finalize_ldt_struct on the result. LDT starts zeroed. */
-static struct ldt_struct *alloc_ldt_struct(unsigned int size)
+static struct ldt_struct *alloc_ldt_struct(unsigned int num_entries)
 {
         struct ldt_struct *new_ldt;
         unsigned int alloc_size;
 
-        if (size > LDT_ENTRIES)
+        if (num_entries > LDT_ENTRIES)
                 return NULL;
 
         new_ldt = kmalloc(sizeof(struct ldt_struct), GFP_KERNEL);
@@ -47,7 +48,7 @@ static struct ldt_struct *alloc_ldt_struct(unsigned int size)
                 return NULL;
 
         BUILD_BUG_ON(LDT_ENTRY_SIZE != sizeof(struct desc_struct));
-        alloc_size = size * LDT_ENTRY_SIZE;
+        alloc_size = num_entries * LDT_ENTRY_SIZE;
 
         /*
          * Xen is very picky: it requires a page-aligned LDT that has no
@@ -65,14 +66,14 @@ static struct ldt_struct *alloc_ldt_struct(unsigned int size)
                 return NULL;
         }
 
-        new_ldt->size = size;
+        new_ldt->nr_entries = num_entries;
         return new_ldt;
 }
 
 /* After calling this, the LDT is immutable. */
 static void finalize_ldt_struct(struct ldt_struct *ldt)
 {
-        paravirt_alloc_ldt(ldt->entries, ldt->size);
+        paravirt_alloc_ldt(ldt->entries, ldt->nr_entries);
 }
 
 /* context.lock is held */
@@ -91,8 +92,8 @@ static void free_ldt_struct(struct ldt_struct *ldt)
         if (likely(!ldt))
                 return;
 
-        paravirt_free_ldt(ldt->entries, ldt->size);
-        if (ldt->size * LDT_ENTRY_SIZE > PAGE_SIZE)
+        paravirt_free_ldt(ldt->entries, ldt->nr_entries);
+        if (ldt->nr_entries * LDT_ENTRY_SIZE > PAGE_SIZE)
                 vfree_atomic(ldt->entries);
         else
                 free_page((unsigned long)ldt->entries);
@@ -122,14 +123,14 @@ int init_new_context_ldt(struct task_struct *tsk, struct mm_struct *mm)
                 goto out_unlock;
         }
 
-        new_ldt = alloc_ldt_struct(old_mm->context.ldt->size);
+        new_ldt = alloc_ldt_struct(old_mm->context.ldt->nr_entries);
         if (!new_ldt) {
                 retval = -ENOMEM;
                 goto out_unlock;
         }
 
         memcpy(new_ldt->entries, old_mm->context.ldt->entries,
-               new_ldt->size * LDT_ENTRY_SIZE);
+               new_ldt->nr_entries * LDT_ENTRY_SIZE);
         finalize_ldt_struct(new_ldt);
 
         mm->context.ldt = new_ldt;
@@ -152,9 +153,9 @@ void destroy_context_ldt(struct mm_struct *mm)
 
 static int read_ldt(void __user *ptr, unsigned long bytecount)
 {
-        int retval;
-        unsigned long size;
         struct mm_struct *mm = current->mm;
+        unsigned long entries_size;
+        int retval;
 
         mutex_lock(&mm->context.lock);
 
@@ -166,18 +167,18 @@ static int read_ldt(void __user *ptr, unsigned long bytecount)
         if (bytecount > LDT_ENTRY_SIZE * LDT_ENTRIES)
                 bytecount = LDT_ENTRY_SIZE * LDT_ENTRIES;
 
-        size = mm->context.ldt->size * LDT_ENTRY_SIZE;
-        if (size > bytecount)
-                size = bytecount;
+        entries_size = mm->context.ldt->nr_entries * LDT_ENTRY_SIZE;
+        if (entries_size > bytecount)
+                entries_size = bytecount;
 
-        if (copy_to_user(ptr, mm->context.ldt->entries, size)) {
+        if (copy_to_user(ptr, mm->context.ldt->entries, entries_size)) {
                 retval = -EFAULT;
                 goto out_unlock;
         }
 
-        if (size != bytecount) {
+        if (entries_size != bytecount) {
                 /* Zero-fill the rest and pretend we read bytecount bytes. */
-                if (clear_user(ptr + size, bytecount - size)) {
+                if (clear_user(ptr + entries_size, bytecount - entries_size)) {
                         retval = -EFAULT;
                         goto out_unlock;
                 }
@@ -208,7 +209,7 @@ static int write_ldt(void __user *ptr, unsigned long bytecount, int oldmode)
 {
         struct mm_struct *mm = current->mm;
         struct ldt_struct *new_ldt, *old_ldt;
-        unsigned int oldsize, newsize;
+        unsigned int old_nr_entries, new_nr_entries;
         struct user_desc ldt_info;
         struct desc_struct ldt;
         int error;
@@ -247,17 +248,18 @@ static int write_ldt(void __user *ptr, unsigned long bytecount, int oldmode)
 
         mutex_lock(&mm->context.lock);
 
-        old_ldt = mm->context.ldt;
-        oldsize = old_ldt ? old_ldt->size : 0;
-        newsize = max(ldt_info.entry_number + 1, oldsize);
+        old_ldt       = mm->context.ldt;
+        old_nr_entries = old_ldt ? old_ldt->nr_entries : 0;
+        new_nr_entries = max(ldt_info.entry_number + 1, old_nr_entries);
 
         error = -ENOMEM;
-        new_ldt = alloc_ldt_struct(newsize);
+        new_ldt = alloc_ldt_struct(new_nr_entries);
         if (!new_ldt)
                 goto out_unlock;
 
         if (old_ldt)
-                memcpy(new_ldt->entries, old_ldt->entries, oldsize * LDT_ENTRY_SIZE);
+                memcpy(new_ldt->entries, old_ldt->entries, old_nr_entries * LDT_ENTRY_SIZE);
+
         new_ldt->entries[ldt_info.entry_number] = ldt;
         finalize_ldt_struct(new_ldt);
 
diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c
index 6f5ca4ebe6e5..cb0a30473c23 100644
--- a/arch/x86/kernel/machine_kexec_64.c
+++ b/arch/x86/kernel/machine_kexec_64.c
@@ -347,7 +347,7 @@ void machine_kexec(struct kimage *image)
 void arch_crash_save_vmcoreinfo(void)
 {
         VMCOREINFO_NUMBER(phys_base);
-        VMCOREINFO_SYMBOL(init_level4_pgt);
+        VMCOREINFO_SYMBOL(init_top_pgt);
 
 #ifdef CONFIG_NUMA
         VMCOREINFO_SYMBOL(node_data);
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 3586996fc50d..bc0a849589bb 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -391,7 +391,7 @@ struct pv_mmu_ops pv_mmu_ops __ro_after_init = {
 
         .read_cr2 = native_read_cr2,
         .write_cr2 = native_write_cr2,
-        .read_cr3 = native_read_cr3,
+        .read_cr3 = __native_read_cr3,
         .write_cr3 = native_write_cr3,
 
         .flush_tlb_user = native_flush_tlb,
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index ffeae818aa7a..c6d6dc5f8bb2 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -92,7 +92,7 @@ void __show_regs(struct pt_regs *regs, int all)
 
         cr0 = read_cr0();
         cr2 = read_cr2();
-        cr3 = read_cr3();
+        cr3 = __read_cr3();
         cr4 = __read_cr4();
         printk(KERN_DEFAULT "CR0: %08lx CR2: %08lx CR3: %08lx CR4: %08lx\n",
                         cr0, cr2, cr3, cr4);
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index b6840bf3940b..c3169be4c596 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -104,7 +104,7 @@ void __show_regs(struct pt_regs *regs, int all)
 
         cr0 = read_cr0();
         cr2 = read_cr2();
-        cr3 = read_cr3();
+        cr3 = __read_cr3();
         cr4 = __read_cr4();
 
         printk(KERN_DEFAULT "FS:  %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
@@ -142,7 +142,7 @@ void release_thread(struct task_struct *dead_task)
                         pr_warn("WARNING: dead process %s still has LDT? <%p/%d>\n",
                                 dead_task->comm,
                                 dead_task->mm->context.ldt->entries,
-                                dead_task->mm->context.ldt->size);
+                                dead_task->mm->context.ldt->nr_entries);
                         BUG();
                 }
 #endif
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 045e4f993bd2..b474c8de7fba 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -1589,7 +1589,6 @@ void native_cpu_die(unsigned int cpu)
 void play_dead_common(void)
 {
         idle_task_exit();
-        reset_lazy_tlbstate();
 
         /* Ack it */
         (void)cpu_report_death();
diff --git a/arch/x86/kernel/step.c b/arch/x86/kernel/step.c
index f07f83b3611b..5f25cfbd952e 100644
--- a/arch/x86/kernel/step.c
+++ b/arch/x86/kernel/step.c
@@ -34,7 +34,7 @@ unsigned long convert_ip_to_linear(struct task_struct *child, struct pt_regs *re
 
                 mutex_lock(&child->mm->context.lock);
                 if (unlikely(!child->mm->context.ldt ||
-                             seg >= child->mm->context.ldt->size))
+                             seg >= child->mm->context.ldt->nr_entries))
                         addr = -1L; /* bogus selector, access would fault */
                 else {
                         desc = &child->mm->context.ldt->entries[seg];