Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Will Deacon:

 - virt_to_page/page_address optimisations

 - support for NUMA systems described using device-tree

 - support for hibernate/suspend-to-disk

 - proper support for maxcpus= command line parameter

 - detection and graceful handling of AArch64-only CPUs

 - miscellaneous cleanups and non-critical fixes

* tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (92 commits)
  arm64: do not enforce strict 16 byte alignment to stack pointer
  arm64: kernel: Fix incorrect brk randomization
  arm64: cpuinfo: Missing NULL terminator in compat_hwcap_str
  arm64: secondary_start_kernel: Remove unnecessary barrier
  arm64: Ensure pmd_present() returns false after pmd_mknotpresent()
  arm64: Replace hard-coded values in the pmd/pud_bad() macros
  arm64: Implement pmdp_set_access_flags() for hardware AF/DBM
  arm64: Fix typo in the pmdp_huge_get_and_clear() definition
  arm64: mm: remove unnecessary EXPORT_SYMBOL_GPL
  arm64: always use STRICT_MM_TYPECHECKS
  arm64: kvm: Fix kvm teardown for systems using the extended idmap
  arm64: kaslr: increase randomization granularity
  arm64: kconfig: drop CONFIG_RTC_LIB dependency
  arm64: make ARCH_SUPPORTS_DEBUG_PAGEALLOC depend on !HIBERNATION
  arm64: hibernate: Refuse to hibernate if the boot cpu is offline
  arm64: kernel: Add support for hibernate/suspend-to-disk
  PM / Hibernate: Call flush_icache_range() on pages restored in-place
  arm64: Add new asm macro copy_page
  arm64: Promote KERNEL_START/KERNEL_END definitions to a header file
  arm64: kernel: Include _AC definition in page.h
  ...

2 files changed, 76 insertions, 47 deletions

diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index dded1b763c16..9ef013d86cc5 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -16,7 +16,6 @@
  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-#include <linux/cpu.h>
 #include <linux/cpu_pm.h>
 #include <linux/errno.h>
 #include <linux/err.h>
@@ -66,6 +65,8 @@ static DEFINE_SPINLOCK(kvm_vmid_lock);
 
 static bool vgic_present;
 
+static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled);
+
 static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
 {
         BUG_ON(preemptible());
@@ -90,11 +91,6 @@ struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void)
         return &kvm_arm_running_vcpu;
 }
 
-int kvm_arch_hardware_enable(void)
-{
-        return 0;
-}
-
 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 {
         return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
@@ -1033,11 +1029,6 @@ long kvm_arch_vm_ioctl(struct file *filp,
         }
 }
 
-static void cpu_init_stage2(void *dummy)
-{
-        __cpu_init_stage2();
-}
-
 static void cpu_init_hyp_mode(void *dummy)
 {
         phys_addr_t boot_pgd_ptr;
@@ -1065,43 +1056,87 @@ static void cpu_hyp_reinit(void)
 {
         if (is_kernel_in_hyp_mode()) {
                 /*
-                 * cpu_init_stage2() is safe to call even if the PM
+                 * __cpu_init_stage2() is safe to call even if the PM
                  * event was cancelled before the CPU was reset.
                  */
-                cpu_init_stage2(NULL);
+                __cpu_init_stage2();
         } else {
                 if (__hyp_get_vectors() == hyp_default_vectors)
                         cpu_init_hyp_mode(NULL);
         }
 }
 
-static int hyp_init_cpu_notify(struct notifier_block *self,
-                               unsigned long action, void *cpu)
+static void cpu_hyp_reset(void)
+{
+        phys_addr_t boot_pgd_ptr;
+        phys_addr_t phys_idmap_start;
+
+        if (!is_kernel_in_hyp_mode()) {
+                boot_pgd_ptr = kvm_mmu_get_boot_httbr();
+                phys_idmap_start = kvm_get_idmap_start();
+
+                __cpu_reset_hyp_mode(boot_pgd_ptr, phys_idmap_start);
+        }
+}
+
+static void _kvm_arch_hardware_enable(void *discard)
 {
-        switch (action) {
-        case CPU_STARTING:
-        case CPU_STARTING_FROZEN:
+        if (!__this_cpu_read(kvm_arm_hardware_enabled)) {
                 cpu_hyp_reinit();
+                __this_cpu_write(kvm_arm_hardware_enabled, 1);
         }
+}
+
+int kvm_arch_hardware_enable(void)
+{
+        _kvm_arch_hardware_enable(NULL);
+        return 0;
+}
 
-        return NOTIFY_OK;
+static void _kvm_arch_hardware_disable(void *discard)
+{
+        if (__this_cpu_read(kvm_arm_hardware_enabled)) {
+                cpu_hyp_reset();
+                __this_cpu_write(kvm_arm_hardware_enabled, 0);
+        }
 }
 
-static struct notifier_block hyp_init_cpu_nb = {
-        .notifier_call = hyp_init_cpu_notify,
-};
+void kvm_arch_hardware_disable(void)
+{
+        _kvm_arch_hardware_disable(NULL);
+}
 
 #ifdef CONFIG_CPU_PM
 static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
                                     unsigned long cmd,
                                     void *v)
 {
-        if (cmd == CPU_PM_EXIT) {
-                cpu_hyp_reinit();
+        /*
+         * kvm_arm_hardware_enabled is left with its old value over
+         * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should
+         * re-enable hyp.
+         */
+        switch (cmd) {
+        case CPU_PM_ENTER:
+                if (__this_cpu_read(kvm_arm_hardware_enabled))
+                        /*
+                         * don't update kvm_arm_hardware_enabled here
+                         * so that the hardware will be re-enabled
+                         * when we resume. See below.
+                         */
+                        cpu_hyp_reset();
+
+                return NOTIFY_OK;
+        case CPU_PM_EXIT:
+                if (__this_cpu_read(kvm_arm_hardware_enabled))
+                        /* The hardware was enabled before suspend. */
+                        cpu_hyp_reinit();
+
                 return NOTIFY_OK;
-        }
 
-        return NOTIFY_DONE;
+        default:
+                return NOTIFY_DONE;
+        }
 }
 
 static struct notifier_block hyp_init_cpu_pm_nb = {
@@ -1143,16 +1178,12 @@ static int init_common_resources(void)
 
 static int init_subsystems(void)
 {
-        int err;
+        int err = 0;
 
         /*
-         * Register CPU Hotplug notifier
+         * Enable hardware so that subsystem initialisation can access EL2.
          */
-        err = register_cpu_notifier(&hyp_init_cpu_nb);
-        if (err) {
-                kvm_err("Cannot register KVM init CPU notifier (%d)\n", err);
-                return err;
-        }
+        on_each_cpu(_kvm_arch_hardware_enable, NULL, 1);
 
         /*
          * Register CPU lower-power notifier
@@ -1170,9 +1201,10 @@ static int init_subsystems(void)
         case -ENODEV:
         case -ENXIO:
                 vgic_present = false;
+                err = 0;
                 break;
         default:
-                return err;
+                goto out;
         }
 
         /*
@@ -1180,12 +1212,15 @@ static int init_subsystems(void)
          */
         err = kvm_timer_hyp_init();
         if (err)
-                return err;
+                goto out;
 
         kvm_perf_init();
         kvm_coproc_table_init();
 
-        return 0;
+out:
+        on_each_cpu(_kvm_arch_hardware_disable, NULL, 1);
+
+        return err;
 }
 
 static void teardown_hyp_mode(void)
@@ -1198,17 +1233,11 @@ static void teardown_hyp_mode(void)
         free_hyp_pgds();
         for_each_possible_cpu(cpu)
                 free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
-        unregister_cpu_notifier(&hyp_init_cpu_nb);
         hyp_cpu_pm_exit();
 }
 
 static int init_vhe_mode(void)
 {
-        /*
-         * Execute the init code on each CPU.
-         */
-        on_each_cpu(cpu_init_stage2, NULL, 1);
-
         /* set size of VMID supported by CPU */
         kvm_vmid_bits = kvm_get_vmid_bits();
         kvm_info("%d-bit VMID\n", kvm_vmid_bits);
@@ -1295,11 +1324,6 @@ static int init_hyp_mode(void)
                 }
         }
 
-        /*
-         * Execute the init code on each CPU.
-         */
-        on_each_cpu(cpu_init_hyp_mode, NULL, 1);
-
 #ifndef CONFIG_HOTPLUG_CPU
         free_boot_hyp_pgd();
 #endif
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index d6d4191e68f2..be302128c5d7 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -1666,6 +1666,11 @@ phys_addr_t kvm_get_idmap_vector(void)
         return hyp_idmap_vector;
 }
 
+phys_addr_t kvm_get_idmap_start(void)
+{
+        return hyp_idmap_start;
+}
+
 int kvm_mmu_init(void)
 {
         int err;