Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Will Deacon:

 - virt_to_page/page_address optimisations

 - support for NUMA systems described using device-tree

 - support for hibernate/suspend-to-disk

 - proper support for maxcpus= command line parameter

 - detection and graceful handling of AArch64-only CPUs

 - miscellaneous cleanups and non-critical fixes

* tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (92 commits)
  arm64: do not enforce strict 16 byte alignment to stack pointer
  arm64: kernel: Fix incorrect brk randomization
  arm64: cpuinfo: Missing NULL terminator in compat_hwcap_str
  arm64: secondary_start_kernel: Remove unnecessary barrier
  arm64: Ensure pmd_present() returns false after pmd_mknotpresent()
  arm64: Replace hard-coded values in the pmd/pud_bad() macros
  arm64: Implement pmdp_set_access_flags() for hardware AF/DBM
  arm64: Fix typo in the pmdp_huge_get_and_clear() definition
  arm64: mm: remove unnecessary EXPORT_SYMBOL_GPL
  arm64: always use STRICT_MM_TYPECHECKS
  arm64: kvm: Fix kvm teardown for systems using the extended idmap
  arm64: kaslr: increase randomization granularity
  arm64: kconfig: drop CONFIG_RTC_LIB dependency
  arm64: make ARCH_SUPPORTS_DEBUG_PAGEALLOC depend on !HIBERNATION
  arm64: hibernate: Refuse to hibernate if the boot cpu is offline
  arm64: kernel: Add support for hibernate/suspend-to-disk
  PM / Hibernate: Call flush_icache_range() on pages restored in-place
  arm64: Add new asm macro copy_page
  arm64: Promote KERNEL_START/KERNEL_END definitions to a header file
  arm64: kernel: Include _AC definition in page.h
  ...

6 files changed, 114 insertions, 11 deletions

diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index eba89e42f0ed..3246c4aba5b1 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -186,6 +186,13 @@ int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
                 exit_handler = kvm_get_exit_handler(vcpu);
 
                 return exit_handler(vcpu, run);
+        case ARM_EXCEPTION_HYP_GONE:
+                /*
+                 * EL2 has been reset to the hyp-stub. This happens when a guest
+                 * is pre-empted by kvm_reboot()'s shutdown call.
+                 */
+                run->exit_reason = KVM_EXIT_FAIL_ENTRY;
+                return 0;
         default:
                 kvm_pr_unimpl("Unsupported exception type: %d",
                               exception_index);
diff --git a/arch/arm64/kvm/hyp-init.S b/arch/arm64/kvm/hyp-init.S
index 7d8747c6427c..a873a6d8be90 100644
--- a/arch/arm64/kvm/hyp-init.S
+++ b/arch/arm64/kvm/hyp-init.S
@@ -21,6 +21,7 @@
 #include <asm/kvm_arm.h>
 #include <asm/kvm_mmu.h>
 #include <asm/pgtable-hwdef.h>
+#include <asm/sysreg.h>
 
         .text
         .pushsection    .hyp.idmap.text, "ax"
@@ -103,8 +104,8 @@ __do_hyp_init:
         dsb     sy
 
         mrs     x4, sctlr_el2
-        and     x4, x4, #SCTLR_EL2_EE   // preserve endianness of EL2
-        ldr     x5, =SCTLR_EL2_FLAGS
+        and     x4, x4, #SCTLR_ELx_EE   // preserve endianness of EL2
+        ldr     x5, =SCTLR_ELx_FLAGS
         orr     x4, x4, x5
         msr     sctlr_el2, x4
         isb
@@ -138,6 +139,49 @@ merged:
         eret
 ENDPROC(__kvm_hyp_init)
 
+        /*
+         * Reset kvm back to the hyp stub. This is the trampoline dance in
+         * reverse. If kvm used an extended idmap, __extended_idmap_trampoline
+         * calls this code directly in the idmap. In this case switching to the
+         * boot tables is a no-op.
+         *
+         * x0: HYP boot pgd
+         * x1: HYP phys_idmap_start
+         */
+ENTRY(__kvm_hyp_reset)
+        /* We're in trampoline code in VA, switch back to boot page tables */
+        msr     ttbr0_el2, x0
+        isb
+
+        /* Ensure the PA branch doesn't find a stale tlb entry or stale code. */
+        ic      iallu
+        tlbi    alle2
+        dsb     sy
+        isb
+
+        /* Branch into PA space */
+        adr     x0, 1f
+        bfi     x1, x0, #0, #PAGE_SHIFT
+        br      x1
+
+        /* We're now in idmap, disable MMU */
+1:      mrs     x0, sctlr_el2
+        ldr     x1, =SCTLR_ELx_FLAGS
+        bic     x0, x0, x1              // Clear SCTL_M and etc
+        msr     sctlr_el2, x0
+        isb
+
+        /* Invalidate the old TLBs */
+        tlbi    alle2
+        dsb     sy
+
+        /* Install stub vectors */
+        adr_l   x0, __hyp_stub_vectors
+        msr     vbar_el2, x0
+
+        eret
+ENDPROC(__kvm_hyp_reset)
+
         .ltorg
 
         .popsection
diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S
index 48f19a37b3df..7ce931565151 100644
--- a/arch/arm64/kvm/hyp.S
+++ b/arch/arm64/kvm/hyp.S
@@ -35,16 +35,21 @@
  * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c).  Return values are
  * passed in x0.
  *
- * A function pointer with a value of 0 has a special meaning, and is
- * used to implement __hyp_get_vectors in the same way as in
+ * A function pointer with a value less than 0xfff has a special meaning,
+ * and is used to implement __hyp_get_vectors in the same way as in
  * arch/arm64/kernel/hyp_stub.S.
+ * HVC behaves as a 'bl' call and will clobber lr.
  */
 ENTRY(__kvm_call_hyp)
-alternative_if_not ARM64_HAS_VIRT_HOST_EXTN     
+alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
+        str     lr, [sp, #-16]!
         hvc     #0
+        ldr     lr, [sp], #16
         ret
 alternative_else
         b       __vhe_hyp_call
         nop
+        nop
+        nop
 alternative_endif
 ENDPROC(__kvm_call_hyp)
diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S
index ce9e5e5f28cf..70254a65bd5b 100644
--- a/arch/arm64/kvm/hyp/entry.S
+++ b/arch/arm64/kvm/hyp/entry.S
@@ -164,3 +164,22 @@ alternative_endif
 
         eret
 ENDPROC(__fpsimd_guest_restore)
+
+/*
+ * When using the extended idmap, we don't have a trampoline page we can use
+ * while we switch pages tables during __kvm_hyp_reset. Accessing the idmap
+ * directly would be ideal, but if we're using the extended idmap then the
+ * idmap is located above HYP_PAGE_OFFSET, and the address will be masked by
+ * kvm_call_hyp using kern_hyp_va.
+ *
+ * x0: HYP boot pgd
+ * x1: HYP phys_idmap_start
+ */
+ENTRY(__extended_idmap_trampoline)
+        mov     x4, x1
+        adr_l   x3, __kvm_hyp_reset
+
+        /* insert __kvm_hyp_reset()s offset into phys_idmap_start */
+        bfi     x4, x3, #0, #PAGE_SHIFT
+        br      x4
+ENDPROC(__extended_idmap_trampoline)
diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index 3488894397ff..2d87f36d5cb4 100644
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -42,19 +42,17 @@
          * Shuffle the parameters before calling the function
          * pointed to in x0. Assumes parameters in x[1,2,3].
          */
-        sub     sp, sp, #16
-        str     lr, [sp]
         mov     lr, x0
         mov     x0, x1
         mov     x1, x2
         mov     x2, x3
         blr     lr
-        ldr     lr, [sp]
-        add     sp, sp, #16
 .endm
 
 ENTRY(__vhe_hyp_call)
+        str     lr, [sp, #-16]!
         do_el2_call
+        ldr     lr, [sp], #16
         /*
          * We used to rely on having an exception return to get
          * an implicit isb. In the E2H case, we don't have it anymore.
@@ -84,8 +82,8 @@ alternative_endif
         /* Here, we're pretty sure the host called HVC. */
         restore_x0_to_x3
 
-        /* Check for __hyp_get_vectors */
-        cbnz    x0, 1f
+        cmp     x0, #HVC_GET_VECTORS
+        b.ne    1f
         mrs     x0, vbar_el2
         b       2f
 
diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
index 9677bf069bcc..b1ad730e1567 100644
--- a/arch/arm64/kvm/reset.c
+++ b/arch/arm64/kvm/reset.c
@@ -29,7 +29,9 @@
 #include <asm/cputype.h>
 #include <asm/ptrace.h>
 #include <asm/kvm_arm.h>
+#include <asm/kvm_asm.h>
 #include <asm/kvm_coproc.h>
+#include <asm/kvm_mmu.h>
 
 /*
  * ARMv8 Reset Values
@@ -130,3 +132,31 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
         /* Reset timer */
         return kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq);
 }
+
+extern char __hyp_idmap_text_start[];
+
+unsigned long kvm_hyp_reset_entry(void)
+{
+        if (!__kvm_cpu_uses_extended_idmap()) {
+                unsigned long offset;
+
+                /*
+                 * Find the address of __kvm_hyp_reset() in the trampoline page.
+                 * This is present in the running page tables, and the boot page
+                 * tables, so we call the code here to start the trampoline
+                 * dance in reverse.
+                 */
+                offset = (unsigned long)__kvm_hyp_reset
+                         - ((unsigned long)__hyp_idmap_text_start & PAGE_MASK);
+
+                return TRAMPOLINE_VA + offset;
+        } else {
+                /*
+                 * KVM is running with merged page tables, which don't have the
+                 * trampoline page mapped. We know the idmap is still mapped,
+                 * but can't be called into directly. Use
+                 * __extended_idmap_trampoline to do the call.
+                 */
+                return (unsigned long)kvm_ksym_ref(__extended_idmap_trampoline);
+        }
+}