Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Catalin Marinas:
 - eBPF JIT compiler for arm64
 - CPU suspend backend for PSCI (firmware interface) with standard idle
   states defined in DT (generic idle driver to be merged via a
   different tree)
 - Support for CONFIG_DEBUG_SET_MODULE_RONX
 - Support for unmapped cpu-release-addr (outside kernel linear mapping)
 - set_arch_dma_coherent_ops() implemented and bus notifiers removed
 - EFI_STUB improvements when base of DRAM is occupied
 - Typos in KGDB macros
 - Clean-up to (partially) allow kernel building with LLVM
 - Other clean-ups (extern keyword, phys_addr_t usage)

* tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (51 commits)
  arm64: Remove unneeded extern keyword
  ARM64: make of_device_ids const
  arm64: Use phys_addr_t type for physical address
  aarch64: filter $x from kallsyms
  arm64: Use DMA_ERROR_CODE to denote failed allocation
  arm64: Fix typos in KGDB macros
  arm64: insn: Add return statements after BUG_ON()
  arm64: debug: don't re-enable debug exceptions on return from el1_dbg
  Revert "arm64: dmi: Add SMBIOS/DMI support"
  arm64: Implement set_arch_dma_coherent_ops() to replace bus notifiers
  of: amba: use of_dma_configure for AMBA devices
  arm64: dmi: Add SMBIOS/DMI support
  arm64: Correct ftrace calls to aarch64_insn_gen_branch_imm()
  arm64:mm: initialize max_mapnr using function set_max_mapnr
  setup: Move unmask of async interrupts after possible earlycon setup
  arm64: LLVMLinux: Fix inline arm64 assembly for use with clang
  arm64: pageattr: Correctly adjust unaligned start addresses
  net: bpf: arm64: fix module memory leak when JIT image build fails
  arm64: add PSCI CPU_SUSPEND based cpu_suspend support
  arm64: kernel: introduce cpu_init_idle CPU operation
  ...

19 files changed, 935 insertions, 106 deletions

diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index df7ef8768fc2..6e9538c2d28a 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -26,6 +26,7 @@ arm64-obj-$(CONFIG_PERF_EVENTS)		+= perf_regs.o
 arm64-obj-$(CONFIG_HW_PERF_EVENTS)      += perf_event.o
 arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)  += hw_breakpoint.o
 arm64-obj-$(CONFIG_ARM64_CPU_SUSPEND)   += sleep.o suspend.o
+arm64-obj-$(CONFIG_CPU_IDLE)            += cpuidle.o
 arm64-obj-$(CONFIG_JUMP_LABEL)          += jump_label.o
 arm64-obj-$(CONFIG_KGDB)                += kgdb.o
 arm64-obj-$(CONFIG_EFI)                 += efi.o efi-stub.o efi-entry.o
diff --git a/arch/arm64/kernel/cpuidle.c b/arch/arm64/kernel/cpuidle.c
new file mode 100644
index 000000000000..19d17f51db37
--- /dev/null
+++ b/arch/arm64/kernel/cpuidle.c
@@ -0,0 +1,31 @@
+/*
+ * ARM64 CPU idle arch support
+ *
+ * Copyright (C) 2014 ARM Ltd.
+ * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/of.h>
+#include <linux/of_device.h>
+
+#include <asm/cpuidle.h>
+#include <asm/cpu_ops.h>
+
+int cpu_init_idle(unsigned int cpu)
+{
+        int ret = -EOPNOTSUPP;
+        struct device_node *cpu_node = of_cpu_device_node_get(cpu);
+
+        if (!cpu_node)
+                return -ENODEV;
+
+        if (cpu_ops[cpu] && cpu_ops[cpu]->cpu_init_idle)
+                ret = cpu_ops[cpu]->cpu_init_idle(cpu_node, cpu);
+
+        of_node_put(cpu_node);
+        return ret;
+}
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index 177169623026..504fdaa8367e 100644
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -20,8 +20,10 @@
 #include <asm/cputype.h>
 
 #include <linux/bitops.h>
+#include <linux/bug.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
+#include <linux/preempt.h>
 #include <linux/printk.h>
 #include <linux/smp.h>
 
@@ -47,8 +49,18 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info)
         unsigned int cpu = smp_processor_id();
         u32 l1ip = CTR_L1IP(info->reg_ctr);
 
-        if (l1ip != ICACHE_POLICY_PIPT)
-                set_bit(ICACHEF_ALIASING, &__icache_flags);
+        if (l1ip != ICACHE_POLICY_PIPT) {
+                /*
+                 * VIPT caches are non-aliasing if the VA always equals the PA
+                 * in all bit positions that are covered by the index. This is
+                 * the case if the size of a way (# of sets * line size) does
+                 * not exceed PAGE_SIZE.
+                 */
+                u32 waysize = icache_get_numsets() * icache_get_linesize();
+
+                if (l1ip != ICACHE_POLICY_VIPT || waysize > PAGE_SIZE)
+                        set_bit(ICACHEF_ALIASING, &__icache_flags);
+        }
         if (l1ip == ICACHE_POLICY_AIVIVT)
                 set_bit(ICACHEF_AIVIVT, &__icache_flags);
 
@@ -190,3 +202,15 @@ void __init cpuinfo_store_boot_cpu(void)
 
         boot_cpu_data = *info;
 }
+
+u64 __attribute_const__ icache_get_ccsidr(void)
+{
+        u64 ccsidr;
+
+        WARN_ON(preemptible());
+
+        /* Select L1 I-cache and read its size ID register */
+        asm("msr csselr_el1, %1; isb; mrs %0, ccsidr_el1"
+            : "=r"(ccsidr) : "r"(1L));
+        return ccsidr;
+}
diff --git a/arch/arm64/kernel/efi-stub.c b/arch/arm64/kernel/efi-stub.c
index 1317fef8dde9..d27dd982ff26 100644
--- a/arch/arm64/kernel/efi-stub.c
+++ b/arch/arm64/kernel/efi-stub.c
@@ -28,20 +28,16 @@ efi_status_t handle_kernel_image(efi_system_table_t *sys_table,
         kernel_size = _edata - _text;
         if (*image_addr != (dram_base + TEXT_OFFSET)) {
                 kernel_memsize = kernel_size + (_end - _edata);
-                status = efi_relocate_kernel(sys_table, image_addr,
-                                             kernel_size, kernel_memsize,
-                                             dram_base + TEXT_OFFSET,
-                                             PAGE_SIZE);
+                status = efi_low_alloc(sys_table, kernel_memsize + TEXT_OFFSET,
+                                       SZ_2M, reserve_addr);
                 if (status != EFI_SUCCESS) {
                         pr_efi_err(sys_table, "Failed to relocate kernel\n");
                         return status;
                 }
-                if (*image_addr != (dram_base + TEXT_OFFSET)) {
-                        pr_efi_err(sys_table, "Failed to alloc kernel memory\n");
-                        efi_free(sys_table, kernel_memsize, *image_addr);
-                        return EFI_LOAD_ERROR;
-                }
-                *image_size = kernel_memsize;
+                memcpy((void *)*reserve_addr + TEXT_OFFSET, (void *)*image_addr,
+                       kernel_size);
+                *image_addr = *reserve_addr + TEXT_OFFSET;
+                *reserve_size = kernel_memsize + TEXT_OFFSET;
         }
 
 
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index f0b5e5120a87..726b910fe6ec 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -324,7 +324,6 @@ el1_dbg:
         mrs     x0, far_el1
         mov     x2, sp                          // struct pt_regs
         bl      do_debug_exception
-        enable_dbg
         kernel_exit 1
 el1_inv:
         // TODO: add support for undefined instructions in kernel mode
diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c
index 7924d73b6476..cf8556ae09d0 100644
--- a/arch/arm64/kernel/ftrace.c
+++ b/arch/arm64/kernel/ftrace.c
@@ -58,7 +58,8 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
         u32 new;
 
         pc = (unsigned long)&ftrace_call;
-        new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func, true);
+        new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func,
+                                          AARCH64_INSN_BRANCH_LINK);
 
         return ftrace_modify_code(pc, 0, new, false);
 }
@@ -72,7 +73,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
         u32 old, new;
 
         old = aarch64_insn_gen_nop();
-        new = aarch64_insn_gen_branch_imm(pc, addr, true);
+        new = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
 
         return ftrace_modify_code(pc, old, new, true);
 }
@@ -86,7 +87,7 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
         unsigned long pc = rec->ip;
         u32 old, new;
 
-        old = aarch64_insn_gen_branch_imm(pc, addr, true);
+        old = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
         new = aarch64_insn_gen_nop();
 
         return ftrace_modify_code(pc, old, new, true);
@@ -154,7 +155,8 @@ static int ftrace_modify_graph_caller(bool enable)
         u32 branch, nop;
 
         branch = aarch64_insn_gen_branch_imm(pc,
-                        (unsigned long)ftrace_graph_caller, false);
+                                             (unsigned long)ftrace_graph_caller,
+                                             AARCH64_INSN_BRANCH_LINK);
         nop = aarch64_insn_gen_nop();
 
         if (enable)
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 873069056229..0a6e4f924df8 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -151,7 +151,7 @@ optional_header:
         .short  0x20b                           // PE32+ format
         .byte   0x02                            // MajorLinkerVersion
         .byte   0x14                            // MinorLinkerVersion
-        .long   _edata - stext                  // SizeOfCode
+        .long   _end - stext                    // SizeOfCode
         .long   0                               // SizeOfInitializedData
         .long   0                               // SizeOfUninitializedData
         .long   efi_stub_entry - efi_head       // AddressOfEntryPoint
@@ -169,7 +169,7 @@ extra_header_fields:
         .short  0                               // MinorSubsystemVersion
         .long   0                               // Win32VersionValue
 
-        .long   _edata - efi_head               // SizeOfImage
+        .long   _end - efi_head                 // SizeOfImage
 
         // Everything before the kernel image is considered part of the header
         .long   stext - efi_head                // SizeOfHeaders
@@ -216,7 +216,7 @@ section_table:
         .byte   0
         .byte   0
         .byte   0                       // end of 0 padding of section name
-        .long   _edata - stext          // VirtualSize
+        .long   _end - stext            // VirtualSize
         .long   stext - efi_head        // VirtualAddress
         .long   _edata - stext          // SizeOfRawData
         .long   stext - efi_head        // PointerToRawData
diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c
index 92f36835486b..e007714ded04 100644
--- a/arch/arm64/kernel/insn.c
+++ b/arch/arm64/kernel/insn.c
@@ -2,6 +2,8 @@
  * Copyright (C) 2013 Huawei Ltd.
  * Author: Jiang Liu <liuj97@gmail.com>
  *
+ * Copyright (C) 2014 Zi Shen Lim <zlim.lnx@gmail.com>
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
@@ -20,9 +22,14 @@
 #include <linux/smp.h>
 #include <linux/stop_machine.h>
 #include <linux/uaccess.h>
+
 #include <asm/cacheflush.h>
+#include <asm/debug-monitors.h>
 #include <asm/insn.h>
 
+#define AARCH64_INSN_SF_BIT     BIT(31)
+#define AARCH64_INSN_N_BIT      BIT(22)
+
 static int aarch64_insn_encoding_class[] = {
         AARCH64_INSN_CLS_UNKNOWN,
         AARCH64_INSN_CLS_UNKNOWN,
@@ -251,6 +258,19 @@ u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
                 mask = BIT(9) - 1;
                 shift = 12;
                 break;
+        case AARCH64_INSN_IMM_7:
+                mask = BIT(7) - 1;
+                shift = 15;
+                break;
+        case AARCH64_INSN_IMM_6:
+        case AARCH64_INSN_IMM_S:
+                mask = BIT(6) - 1;
+                shift = 10;
+                break;
+        case AARCH64_INSN_IMM_R:
+                mask = BIT(6) - 1;
+                shift = 16;
+                break;
         default:
                 pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n",
                         type);
@@ -264,10 +284,76 @@ u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
         return insn;
 }
 
-u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
-                                          enum aarch64_insn_branch_type type)
+static u32 aarch64_insn_encode_register(enum aarch64_insn_register_type type,
+                                        u32 insn,
+                                        enum aarch64_insn_register reg)
+{
+        int shift;
+
+        if (reg < AARCH64_INSN_REG_0 || reg > AARCH64_INSN_REG_SP) {
+                pr_err("%s: unknown register encoding %d\n", __func__, reg);
+                return 0;
+        }
+
+        switch (type) {
+        case AARCH64_INSN_REGTYPE_RT:
+        case AARCH64_INSN_REGTYPE_RD:
+                shift = 0;
+                break;
+        case AARCH64_INSN_REGTYPE_RN:
+                shift = 5;
+                break;
+        case AARCH64_INSN_REGTYPE_RT2:
+        case AARCH64_INSN_REGTYPE_RA:
+                shift = 10;
+                break;
+        case AARCH64_INSN_REGTYPE_RM:
+                shift = 16;
+                break;
+        default:
+                pr_err("%s: unknown register type encoding %d\n", __func__,
+                       type);
+                return 0;
+        }
+
+        insn &= ~(GENMASK(4, 0) << shift);
+        insn |= reg << shift;
+
+        return insn;
+}
+
+static u32 aarch64_insn_encode_ldst_size(enum aarch64_insn_size_type type,
+                                         u32 insn)
+{
+        u32 size;
+
+        switch (type) {
+        case AARCH64_INSN_SIZE_8:
+                size = 0;
+                break;
+        case AARCH64_INSN_SIZE_16:
+                size = 1;
+                break;
+        case AARCH64_INSN_SIZE_32:
+                size = 2;
+                break;
+        case AARCH64_INSN_SIZE_64:
+                size = 3;
+                break;
+        default:
+                pr_err("%s: unknown size encoding %d\n", __func__, type);
+                return 0;
+        }
+
+        insn &= ~GENMASK(31, 30);
+        insn |= size << 30;
+
+        return insn;
+}
+
+static inline long branch_imm_common(unsigned long pc, unsigned long addr,
+                                     long range)
 {
-        u32 insn;
         long offset;
 
         /*
@@ -276,23 +362,97 @@ u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
          */
         BUG_ON((pc & 0x3) || (addr & 0x3));
 
+        offset = ((long)addr - (long)pc);
+        BUG_ON(offset < -range || offset >= range);
+
+        return offset;
+}
+
+u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
+                                          enum aarch64_insn_branch_type type)
+{
+        u32 insn;
+        long offset;
+
         /*
          * B/BL support [-128M, 128M) offset
          * ARM64 virtual address arrangement guarantees all kernel and module
          * texts are within +/-128M.
          */
-        offset = ((long)addr - (long)pc);
-        BUG_ON(offset < -SZ_128M || offset >= SZ_128M);
+        offset = branch_imm_common(pc, addr, SZ_128M);
 
-        if (type == AARCH64_INSN_BRANCH_LINK)
+        switch (type) {
+        case AARCH64_INSN_BRANCH_LINK:
                 insn = aarch64_insn_get_bl_value();
-        else
+                break;
+        case AARCH64_INSN_BRANCH_NOLINK:
                 insn = aarch64_insn_get_b_value();
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
 
         return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
                                              offset >> 2);
 }
 
+u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr,
+                                     enum aarch64_insn_register reg,
+                                     enum aarch64_insn_variant variant,
+                                     enum aarch64_insn_branch_type type)
+{
+        u32 insn;
+        long offset;
+
+        offset = branch_imm_common(pc, addr, SZ_1M);
+
+        switch (type) {
+        case AARCH64_INSN_BRANCH_COMP_ZERO:
+                insn = aarch64_insn_get_cbz_value();
+                break;
+        case AARCH64_INSN_BRANCH_COMP_NONZERO:
+                insn = aarch64_insn_get_cbnz_value();
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        switch (variant) {
+        case AARCH64_INSN_VARIANT_32BIT:
+                break;
+        case AARCH64_INSN_VARIANT_64BIT:
+                insn |= AARCH64_INSN_SF_BIT;
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
+
+        return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
+                                             offset >> 2);
+}
+
+u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr,
+                                     enum aarch64_insn_condition cond)
+{
+        u32 insn;
+        long offset;
+
+        offset = branch_imm_common(pc, addr, SZ_1M);
+
+        insn = aarch64_insn_get_bcond_value();
+
+        BUG_ON(cond < AARCH64_INSN_COND_EQ || cond > AARCH64_INSN_COND_AL);
+        insn |= cond;
+
+        return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
+                                             offset >> 2);
+}
+
 u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_op op)
 {
         return aarch64_insn_get_hint_value() | op;
@@ -302,3 +462,500 @@ u32 __kprobes aarch64_insn_gen_nop(void)
 {
         return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
 }
+
+u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg,
+                                enum aarch64_insn_branch_type type)
+{
+        u32 insn;
+
+        switch (type) {
+        case AARCH64_INSN_BRANCH_NOLINK:
+                insn = aarch64_insn_get_br_value();
+                break;
+        case AARCH64_INSN_BRANCH_LINK:
+                insn = aarch64_insn_get_blr_value();
+                break;
+        case AARCH64_INSN_BRANCH_RETURN:
+                insn = aarch64_insn_get_ret_value();
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, reg);
+}
+
+u32 aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg,
+                                    enum aarch64_insn_register base,
+                                    enum aarch64_insn_register offset,
+                                    enum aarch64_insn_size_type size,
+                                    enum aarch64_insn_ldst_type type)
+{
+        u32 insn;
+
+        switch (type) {
+        case AARCH64_INSN_LDST_LOAD_REG_OFFSET:
+                insn = aarch64_insn_get_ldr_reg_value();
+                break;
+        case AARCH64_INSN_LDST_STORE_REG_OFFSET:
+                insn = aarch64_insn_get_str_reg_value();
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        insn = aarch64_insn_encode_ldst_size(size, insn);
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
+                                            base);
+
+        return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn,
+                                            offset);
+}
+
+u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1,
+                                     enum aarch64_insn_register reg2,
+                                     enum aarch64_insn_register base,
+                                     int offset,
+                                     enum aarch64_insn_variant variant,
+                                     enum aarch64_insn_ldst_type type)
+{
+        u32 insn;
+        int shift;
+
+        switch (type) {
+        case AARCH64_INSN_LDST_LOAD_PAIR_PRE_INDEX:
+                insn = aarch64_insn_get_ldp_pre_value();
+                break;
+        case AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX:
+                insn = aarch64_insn_get_stp_pre_value();
+                break;
+        case AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX:
+                insn = aarch64_insn_get_ldp_post_value();
+                break;
+        case AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX:
+                insn = aarch64_insn_get_stp_post_value();
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        switch (variant) {
+        case AARCH64_INSN_VARIANT_32BIT:
+                /* offset must be multiples of 4 in the range [-256, 252] */
+                BUG_ON(offset & 0x3);
+                BUG_ON(offset < -256 || offset > 252);
+                shift = 2;
+                break;
+        case AARCH64_INSN_VARIANT_64BIT:
+                /* offset must be multiples of 8 in the range [-512, 504] */
+                BUG_ON(offset & 0x7);
+                BUG_ON(offset < -512 || offset > 504);
+                shift = 3;
+                insn |= AARCH64_INSN_SF_BIT;
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
+                                            reg1);
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn,
+                                            reg2);
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
+                                            base);
+
+        return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_7, insn,
+                                             offset >> shift);
+}
+
+u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst,
+                                 enum aarch64_insn_register src,
+                                 int imm, enum aarch64_insn_variant variant,
+                                 enum aarch64_insn_adsb_type type)
+{
+        u32 insn;
+
+        switch (type) {
+        case AARCH64_INSN_ADSB_ADD:
+                insn = aarch64_insn_get_add_imm_value();
+                break;
+        case AARCH64_INSN_ADSB_SUB:
+                insn = aarch64_insn_get_sub_imm_value();
+                break;
+        case AARCH64_INSN_ADSB_ADD_SETFLAGS:
+                insn = aarch64_insn_get_adds_imm_value();
+                break;
+        case AARCH64_INSN_ADSB_SUB_SETFLAGS:
+                insn = aarch64_insn_get_subs_imm_value();
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        switch (variant) {
+        case AARCH64_INSN_VARIANT_32BIT:
+                break;
+        case AARCH64_INSN_VARIANT_64BIT:
+                insn |= AARCH64_INSN_SF_BIT;
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        BUG_ON(imm & ~(SZ_4K - 1));
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
+
+        return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm);
+}
+
+u32 aarch64_insn_gen_bitfield(enum aarch64_insn_register dst,
+                              enum aarch64_insn_register src,
+                              int immr, int imms,
+                              enum aarch64_insn_variant variant,
+                              enum aarch64_insn_bitfield_type type)
+{
+        u32 insn;
+        u32 mask;
+
+        switch (type) {
+        case AARCH64_INSN_BITFIELD_MOVE:
+                insn = aarch64_insn_get_bfm_value();
+                break;
+        case AARCH64_INSN_BITFIELD_MOVE_UNSIGNED:
+                insn = aarch64_insn_get_ubfm_value();
+                break;
+        case AARCH64_INSN_BITFIELD_MOVE_SIGNED:
+                insn = aarch64_insn_get_sbfm_value();
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        switch (variant) {
+        case AARCH64_INSN_VARIANT_32BIT:
+                mask = GENMASK(4, 0);
+                break;
+        case AARCH64_INSN_VARIANT_64BIT:
+                insn |= AARCH64_INSN_SF_BIT | AARCH64_INSN_N_BIT;
+                mask = GENMASK(5, 0);
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        BUG_ON(immr & ~mask);
+        BUG_ON(imms & ~mask);
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
+
+        insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr);
+
+        return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms);
+}
+
+u32 aarch64_insn_gen_movewide(enum aarch64_insn_register dst,
+                              int imm, int shift,
+                              enum aarch64_insn_variant variant,
+                              enum aarch64_insn_movewide_type type)
+{
+        u32 insn;
+
+        switch (type) {
+        case AARCH64_INSN_MOVEWIDE_ZERO:
+                insn = aarch64_insn_get_movz_value();
+                break;
+        case AARCH64_INSN_MOVEWIDE_KEEP:
+                insn = aarch64_insn_get_movk_value();
+                break;
+        case AARCH64_INSN_MOVEWIDE_INVERSE:
+                insn = aarch64_insn_get_movn_value();
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        BUG_ON(imm & ~(SZ_64K - 1));
+
+        switch (variant) {
+        case AARCH64_INSN_VARIANT_32BIT:
+                BUG_ON(shift != 0 && shift != 16);
+                break;
+        case AARCH64_INSN_VARIANT_64BIT:
+                insn |= AARCH64_INSN_SF_BIT;
+                BUG_ON(shift != 0 && shift != 16 && shift != 32 &&
+                       shift != 48);
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        insn |= (shift >> 4) << 21;
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
+
+        return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm);
+}
+
+u32 aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst,
+                                         enum aarch64_insn_register src,
+                                         enum aarch64_insn_register reg,
+                                         int shift,
+                                         enum aarch64_insn_variant variant,
+                                         enum aarch64_insn_adsb_type type)
+{
+        u32 insn;
+
+        switch (type) {
+        case AARCH64_INSN_ADSB_ADD:
+                insn = aarch64_insn_get_add_value();
+                break;
+        case AARCH64_INSN_ADSB_SUB:
+                insn = aarch64_insn_get_sub_value();
+                break;
+        case AARCH64_INSN_ADSB_ADD_SETFLAGS:
+                insn = aarch64_insn_get_adds_value();
+                break;
+        case AARCH64_INSN_ADSB_SUB_SETFLAGS:
+                insn = aarch64_insn_get_subs_value();
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        switch (variant) {
+        case AARCH64_INSN_VARIANT_32BIT:
+                BUG_ON(shift & ~(SZ_32 - 1));
+                break;
+        case AARCH64_INSN_VARIANT_64BIT:
+                insn |= AARCH64_INSN_SF_BIT;
+                BUG_ON(shift & ~(SZ_64 - 1));
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
+
+        return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
+}
+
+u32 aarch64_insn_gen_data1(enum aarch64_insn_register dst,
+                           enum aarch64_insn_register src,
+                           enum aarch64_insn_variant variant,
+                           enum aarch64_insn_data1_type type)
+{
+        u32 insn;
+
+        switch (type) {
+        case AARCH64_INSN_DATA1_REVERSE_16:
+                insn = aarch64_insn_get_rev16_value();
+                break;
+        case AARCH64_INSN_DATA1_REVERSE_32:
+                insn = aarch64_insn_get_rev32_value();
+                break;
+        case AARCH64_INSN_DATA1_REVERSE_64:
+                BUG_ON(variant != AARCH64_INSN_VARIANT_64BIT);
+                insn = aarch64_insn_get_rev64_value();
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        switch (variant) {
+        case AARCH64_INSN_VARIANT_32BIT:
+                break;
+        case AARCH64_INSN_VARIANT_64BIT:
+                insn |= AARCH64_INSN_SF_BIT;
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
+
+        return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
+}
+
+u32 aarch64_insn_gen_data2(enum aarch64_insn_register dst,
+                           enum aarch64_insn_register src,
+                           enum aarch64_insn_register reg,
+                           enum aarch64_insn_variant variant,
+                           enum aarch64_insn_data2_type type)
+{
+        u32 insn;
+
+        switch (type) {
+        case AARCH64_INSN_DATA2_UDIV:
+                insn = aarch64_insn_get_udiv_value();
+                break;
+        case AARCH64_INSN_DATA2_SDIV:
+                insn = aarch64_insn_get_sdiv_value();
+                break;
+        case AARCH64_INSN_DATA2_LSLV:
+                insn = aarch64_insn_get_lslv_value();
+                break;
+        case AARCH64_INSN_DATA2_LSRV:
+                insn = aarch64_insn_get_lsrv_value();
+                break;
+        case AARCH64_INSN_DATA2_ASRV:
+                insn = aarch64_insn_get_asrv_value();
+                break;
+        case AARCH64_INSN_DATA2_RORV:
+                insn = aarch64_insn_get_rorv_value();
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        switch (variant) {
+        case AARCH64_INSN_VARIANT_32BIT:
+                break;
+        case AARCH64_INSN_VARIANT_64BIT:
+                insn |= AARCH64_INSN_SF_BIT;
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
+
+        return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
+}
+
+u32 aarch64_insn_gen_data3(enum aarch64_insn_register dst,
+                           enum aarch64_insn_register src,
+                           enum aarch64_insn_register reg1,
+                           enum aarch64_insn_register reg2,
+                           enum aarch64_insn_variant variant,
+                           enum aarch64_insn_data3_type type)
+{
+        u32 insn;
+
+        switch (type) {
+        case AARCH64_INSN_DATA3_MADD:
+                insn = aarch64_insn_get_madd_value();
+                break;
+        case AARCH64_INSN_DATA3_MSUB:
+                insn = aarch64_insn_get_msub_value();
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        switch (variant) {
+        case AARCH64_INSN_VARIANT_32BIT:
+                break;
+        case AARCH64_INSN_VARIANT_64BIT:
+                insn |= AARCH64_INSN_SF_BIT;
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RA, insn, src);
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
+                                            reg1);
+
+        return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn,
+                                            reg2);
+}
+
+u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst,
+                                         enum aarch64_insn_register src,
+                                         enum aarch64_insn_register reg,
+                                         int shift,
+                                         enum aarch64_insn_variant variant,
+                                         enum aarch64_insn_logic_type type)
+{
+        u32 insn;
+
+        switch (type) {
+        case AARCH64_INSN_LOGIC_AND:
+                insn = aarch64_insn_get_and_value();
+                break;
+        case AARCH64_INSN_LOGIC_BIC:
+                insn = aarch64_insn_get_bic_value();
+                break;
+        case AARCH64_INSN_LOGIC_ORR:
+                insn = aarch64_insn_get_orr_value();
+                break;
+        case AARCH64_INSN_LOGIC_ORN:
+                insn = aarch64_insn_get_orn_value();
+                break;
+        case AARCH64_INSN_LOGIC_EOR:
+                insn = aarch64_insn_get_eor_value();
+                break;
+        case AARCH64_INSN_LOGIC_EON:
+                insn = aarch64_insn_get_eon_value();
+                break;
+        case AARCH64_INSN_LOGIC_AND_SETFLAGS:
+                insn = aarch64_insn_get_ands_value();
+                break;
+        case AARCH64_INSN_LOGIC_BIC_SETFLAGS:
+                insn = aarch64_insn_get_bics_value();
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+        switch (variant) {
+        case AARCH64_INSN_VARIANT_32BIT:
+                BUG_ON(shift & ~(SZ_32 - 1));
+                break;
+        case AARCH64_INSN_VARIANT_64BIT:
+                insn |= AARCH64_INSN_SF_BIT;
+                BUG_ON(shift & ~(SZ_64 - 1));
+                break;
+        default:
+                BUG_ON(1);
+                return AARCH64_BREAK_FAULT;
+        }
+
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
+
+        insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
+
+        return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
+}
diff --git a/arch/arm64/kernel/kgdb.c b/arch/arm64/kernel/kgdb.c
index 75c9cf1aafee..a0d10c55f307 100644
--- a/arch/arm64/kernel/kgdb.c
+++ b/arch/arm64/kernel/kgdb.c
@@ -235,13 +235,13 @@ static int kgdb_step_brk_fn(struct pt_regs *regs, unsigned int esr)
 
 static struct break_hook kgdb_brkpt_hook = {
         .esr_mask       = 0xffffffff,
-        .esr_val        = DBG_ESR_VAL_BRK(KGDB_DYN_DGB_BRK_IMM),
+        .esr_val        = DBG_ESR_VAL_BRK(KGDB_DYN_DBG_BRK_IMM),
         .fn             = kgdb_brk_fn
 };
 
 static struct break_hook kgdb_compiled_brkpt_hook = {
         .esr_mask       = 0xffffffff,
-        .esr_val        = DBG_ESR_VAL_BRK(KDBG_COMPILED_DBG_BRK_IMM),
+        .esr_val        = DBG_ESR_VAL_BRK(KGDB_COMPILED_DBG_BRK_IMM),
         .fn             = kgdb_compiled_brk_fn
 };
 
diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c
index baf5afb7e6a0..aa29ecb4f800 100644
--- a/arch/arm64/kernel/perf_event.c
+++ b/arch/arm64/kernel/perf_event.c
@@ -1276,7 +1276,7 @@ arch_initcall(cpu_pmu_reset);
 /*
  * PMU platform driver and devicetree bindings.
  */
-static struct of_device_id armpmu_of_device_ids[] = {
+static const struct of_device_id armpmu_of_device_ids[] = {
         {.compatible = "arm,armv8-pmuv3"},
         {},
 };
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 29d48690f2ac..89f41f7d27dd 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -57,36 +57,10 @@ unsigned long __stack_chk_guard __read_mostly;
 EXPORT_SYMBOL(__stack_chk_guard);
 #endif
 
-static void setup_restart(void)
-{
-        /*
-         * Tell the mm system that we are going to reboot -
-         * we may need it to insert some 1:1 mappings so that
-         * soft boot works.
-         */
-        setup_mm_for_reboot();
-
-        /* Clean and invalidate caches */
-        flush_cache_all();
-
-        /* Turn D-cache off */
-        cpu_cache_off();
-
-        /* Push out any further dirty data, and ensure cache is empty */
-        flush_cache_all();
-}
-
 void soft_restart(unsigned long addr)
 {
-        typedef void (*phys_reset_t)(unsigned long);
-        phys_reset_t phys_reset;
-
-        setup_restart();
-
-        /* Switch to the identity mapping */
-        phys_reset = (phys_reset_t)virt_to_phys(cpu_reset);
-        phys_reset(addr);
-
+        setup_mm_for_reboot();
+        cpu_soft_restart(virt_to_phys(cpu_reset), addr);
         /* Should never get here */
         BUG();
 }
diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c
index 553954771a67..866c1c821860 100644
--- a/arch/arm64/kernel/psci.c
+++ b/arch/arm64/kernel/psci.c
@@ -21,6 +21,7 @@
 #include <linux/reboot.h>
 #include <linux/pm.h>
 #include <linux/delay.h>
+#include <linux/slab.h>
 #include <uapi/linux/psci.h>
 
 #include <asm/compiler.h>
@@ -28,6 +29,7 @@
 #include <asm/errno.h>
 #include <asm/psci.h>
 #include <asm/smp_plat.h>
+#include <asm/suspend.h>
 #include <asm/system_misc.h>
 
 #define PSCI_POWER_STATE_TYPE_STANDBY           0
@@ -65,6 +67,8 @@ enum psci_function {
         PSCI_FN_MAX,
 };
 
+static DEFINE_PER_CPU_READ_MOSTLY(struct psci_power_state *, psci_power_state);
+
 static u32 psci_function_id[PSCI_FN_MAX];
 
 static int psci_to_linux_errno(int errno)
@@ -93,6 +97,18 @@ static u32 psci_power_state_pack(struct psci_power_state state)
                  & PSCI_0_2_POWER_STATE_AFFL_MASK);
 }
 
+static void psci_power_state_unpack(u32 power_state,
+                                    struct psci_power_state *state)
+{
+        state->id = (power_state & PSCI_0_2_POWER_STATE_ID_MASK) >>
+                        PSCI_0_2_POWER_STATE_ID_SHIFT;
+        state->type = (power_state & PSCI_0_2_POWER_STATE_TYPE_MASK) >>
+                        PSCI_0_2_POWER_STATE_TYPE_SHIFT;
+        state->affinity_level =
+                        (power_state & PSCI_0_2_POWER_STATE_AFFL_MASK) >>
+                        PSCI_0_2_POWER_STATE_AFFL_SHIFT;
+}
+
 /*
  * The following two functions are invoked via the invoke_psci_fn pointer
  * and will not be inlined, allowing us to piggyback on the AAPCS.
@@ -199,6 +215,63 @@ static int psci_migrate_info_type(void)
         return err;
 }
 
+static int __maybe_unused cpu_psci_cpu_init_idle(struct device_node *cpu_node,
+                                                 unsigned int cpu)
+{
+        int i, ret, count = 0;
+        struct psci_power_state *psci_states;
+        struct device_node *state_node;
+
+        /*
+         * If the PSCI cpu_suspend function hook has not been initialized
+         * idle states must not be enabled, so bail out
+         */
+        if (!psci_ops.cpu_suspend)
+                return -EOPNOTSUPP;
+
+        /* Count idle states */
+        while ((state_node = of_parse_phandle(cpu_node, "cpu-idle-states",
+                                              count))) {
+                count++;
+                of_node_put(state_node);
+        }
+
+        if (!count)
+                return -ENODEV;
+
+        psci_states = kcalloc(count, sizeof(*psci_states), GFP_KERNEL);
+        if (!psci_states)
+                return -ENOMEM;
+
+        for (i = 0; i < count; i++) {
+                u32 psci_power_state;
+
+                state_node = of_parse_phandle(cpu_node, "cpu-idle-states", i);
+
+                ret = of_property_read_u32(state_node,
+                                           "arm,psci-suspend-param",
+                                           &psci_power_state);
+                if (ret) {
+                        pr_warn(" * %s missing arm,psci-suspend-param property\n",
+                                state_node->full_name);
+                        of_node_put(state_node);
+                        goto free_mem;
+                }
+
+                of_node_put(state_node);
+                pr_debug("psci-power-state %#x index %d\n", psci_power_state,
+                                                            i);
+                psci_power_state_unpack(psci_power_state, &psci_states[i]);
+        }
+        /* Idle states parsed correctly, initialize per-cpu pointer */
+        per_cpu(psci_power_state, cpu) = psci_states;
+        return 0;
+
+free_mem:
+        kfree(psci_states);
+        return ret;
+}
+
 static int get_set_conduit_method(struct device_node *np)
 {
         const char *method;
@@ -436,8 +509,39 @@ static int cpu_psci_cpu_kill(unsigned int cpu)
 #endif
 #endif
 
+static int psci_suspend_finisher(unsigned long index)
+{
+        struct psci_power_state *state = __get_cpu_var(psci_power_state);
+
+        return psci_ops.cpu_suspend(state[index - 1],
+                                    virt_to_phys(cpu_resume));
+}
+
+static int __maybe_unused cpu_psci_cpu_suspend(unsigned long index)
+{
+        int ret;
+        struct psci_power_state *state = __get_cpu_var(psci_power_state);
+        /*
+         * idle state index 0 corresponds to wfi, should never be called
+         * from the cpu_suspend operations
+         */
+        if (WARN_ON_ONCE(!index))
+                return -EINVAL;
+
+        if (state->type == PSCI_POWER_STATE_TYPE_STANDBY)
+                ret = psci_ops.cpu_suspend(state[index - 1], 0);
+        else
+                ret = __cpu_suspend(index, psci_suspend_finisher);
+
+        return ret;
+}
+
 const struct cpu_operations cpu_psci_ops = {
         .name           = "psci",
+#ifdef CONFIG_CPU_IDLE
+        .cpu_init_idle  = cpu_psci_cpu_init_idle,
+        .cpu_suspend    = cpu_psci_cpu_suspend,
+#endif
 #ifdef CONFIG_SMP
         .cpu_init       = cpu_psci_cpu_init,
         .cpu_prepare    = cpu_psci_cpu_prepare,
diff --git a/arch/arm64/kernel/return_address.c b/arch/arm64/kernel/return_address.c
index 89102a6ffad5..6c4fd2810ecb 100644
--- a/arch/arm64/kernel/return_address.c
+++ b/arch/arm64/kernel/return_address.c
@@ -36,13 +36,12 @@ void *return_address(unsigned int level)
 {
         struct return_address_data data;
         struct stackframe frame;
-        register unsigned long current_sp asm ("sp");
 
         data.level = level + 2;
         data.addr = NULL;
 
         frame.fp = (unsigned long)__builtin_frame_address(0);
-        frame.sp = current_sp;
+        frame.sp = current_stack_pointer;
         frame.pc = (unsigned long)return_address; /* dummy */
 
         walk_stackframe(&frame, save_return_addr, &data);
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index edb146d01857..2437196cc5d4 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -365,11 +365,6 @@ u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID };
 
 void __init setup_arch(char **cmdline_p)
 {
-        /*
-         * Unmask asynchronous aborts early to catch possible system errors.
-         */
-        local_async_enable();
-
         setup_processor();
 
         setup_machine_fdt(__fdt_pointer);
@@ -385,6 +380,12 @@ void __init setup_arch(char **cmdline_p)
 
         parse_early_param();
 
+        /*
+         *  Unmask asynchronous aborts after bringing up possible earlycon.
+         * (Report possible System Errors once we can report this occurred)
+         */
+        local_async_enable();
+
         efi_init();
         arm64_memblock_init();
 
diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S
index b1925729c692..a564b440416a 100644
--- a/arch/arm64/kernel/sleep.S
+++ b/arch/arm64/kernel/sleep.S
@@ -49,28 +49,39 @@
         orr     \dst, \dst, \mask               // dst|=(aff3>>rs3)
         .endm
 /*
- * Save CPU state for a suspend.  This saves callee registers, and allocates
- * space on the kernel stack to save the CPU specific registers + some
- * other data for resume.
+ * Save CPU state for a suspend and execute the suspend finisher.
+ * On success it will return 0 through cpu_resume - ie through a CPU
+ * soft/hard reboot from the reset vector.
+ * On failure it returns the suspend finisher return value or force
+ * -EOPNOTSUPP if the finisher erroneously returns 0 (the suspend finisher
+ * is not allowed to return, if it does this must be considered failure).
+ * It saves callee registers, and allocates space on the kernel stack
+ * to save the CPU specific registers + some other data for resume.
  *
  *  x0 = suspend finisher argument
+ *  x1 = suspend finisher function pointer
  */
-ENTRY(__cpu_suspend)
+ENTRY(__cpu_suspend_enter)
         stp     x29, lr, [sp, #-96]!
         stp     x19, x20, [sp,#16]
         stp     x21, x22, [sp,#32]
         stp     x23, x24, [sp,#48]
         stp     x25, x26, [sp,#64]
         stp     x27, x28, [sp,#80]
+        /*
+         * Stash suspend finisher and its argument in x20 and x19
+         */
+        mov     x19, x0
+        mov     x20, x1
         mov     x2, sp
         sub     sp, sp, #CPU_SUSPEND_SZ // allocate cpu_suspend_ctx
-        mov     x1, sp
+        mov     x0, sp
         /*
-         * x1 now points to struct cpu_suspend_ctx allocated on the stack
+         * x0 now points to struct cpu_suspend_ctx allocated on the stack
          */
-        str     x2, [x1, #CPU_CTX_SP]
-        ldr     x2, =sleep_save_sp
-        ldr     x2, [x2, #SLEEP_SAVE_SP_VIRT]
+        str     x2, [x0, #CPU_CTX_SP]
+        ldr     x1, =sleep_save_sp
+        ldr     x1, [x1, #SLEEP_SAVE_SP_VIRT]
 #ifdef CONFIG_SMP
         mrs     x7, mpidr_el1
         ldr     x9, =mpidr_hash
@@ -82,11 +93,21 @@ ENTRY(__cpu_suspend)
         ldp     w3, w4, [x9, #MPIDR_HASH_SHIFTS]
         ldp     w5, w6, [x9, #(MPIDR_HASH_SHIFTS + 8)]
         compute_mpidr_hash x8, x3, x4, x5, x6, x7, x10
-        add     x2, x2, x8, lsl #3
+        add     x1, x1, x8, lsl #3
 #endif
-        bl      __cpu_suspend_finisher
+        bl      __cpu_suspend_save
+        /*
+         * Grab suspend finisher in x20 and its argument in x19
+         */
+        mov     x0, x19
+        mov     x1, x20
+        /*
+         * We are ready for power down, fire off the suspend finisher
+         * in x1, with argument in x0
+         */
+        blr     x1
         /*
-         * Never gets here, unless suspend fails.
+         * Never gets here, unless suspend finisher fails.
          * Successful cpu_suspend should return from cpu_resume, returning
          * through this code path is considered an error
          * If the return value is set to 0 force x0 = -EOPNOTSUPP
@@ -103,7 +124,7 @@ ENTRY(__cpu_suspend)
         ldp     x27, x28, [sp, #80]
         ldp     x29, lr, [sp], #96
         ret
-ENDPROC(__cpu_suspend)
+ENDPROC(__cpu_suspend_enter)
         .ltorg
 
 /*
diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c
index 0347d38eea29..4f93c67e63de 100644
--- a/arch/arm64/kernel/smp_spin_table.c
+++ b/arch/arm64/kernel/smp_spin_table.c
@@ -20,6 +20,7 @@
 #include <linux/init.h>
 #include <linux/of.h>
 #include <linux/smp.h>
+#include <linux/types.h>
 
 #include <asm/cacheflush.h>
 #include <asm/cpu_ops.h>
@@ -65,12 +66,21 @@ static int smp_spin_table_cpu_init(struct device_node *dn, unsigned int cpu)
 
 static int smp_spin_table_cpu_prepare(unsigned int cpu)
 {
-        void **release_addr;
+        __le64 __iomem *release_addr;
 
         if (!cpu_release_addr[cpu])
                 return -ENODEV;
 
-        release_addr = __va(cpu_release_addr[cpu]);
+        /*
+         * The cpu-release-addr may or may not be inside the linear mapping.
+         * As ioremap_cache will either give us a new mapping or reuse the
+         * existing linear mapping, we can use it to cover both cases. In
+         * either case the memory will be MT_NORMAL.
+         */
+        release_addr = ioremap_cache(cpu_release_addr[cpu],
+                                     sizeof(*release_addr));
+        if (!release_addr)
+                return -ENOMEM;
 
         /*
          * We write the release address as LE regardless of the native
@@ -79,15 +89,17 @@ static int smp_spin_table_cpu_prepare(unsigned int cpu)
          * boot-loader's endianess before jumping. This is mandated by
          * the boot protocol.
          */
-        release_addr[0] = (void *) cpu_to_le64(__pa(secondary_holding_pen));
-
-        __flush_dcache_area(release_addr, sizeof(release_addr[0]));
+        writeq_relaxed(__pa(secondary_holding_pen), release_addr);
+        __flush_dcache_area((__force void *)release_addr,
+                            sizeof(*release_addr));
 
         /*
          * Send an event to wake up the secondary CPU.
          */
         sev();
 
+        iounmap(release_addr);
+
         return 0;
 }
 
diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c
index 55437ba1f5a4..407991bf79f5 100644
--- a/arch/arm64/kernel/stacktrace.c
+++ b/arch/arm64/kernel/stacktrace.c
@@ -111,10 +111,9 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
                 frame.sp = thread_saved_sp(tsk);
                 frame.pc = thread_saved_pc(tsk);
         } else {
-                register unsigned long current_sp asm("sp");
                 data.no_sched_functions = 0;
                 frame.fp = (unsigned long)__builtin_frame_address(0);
-                frame.sp = current_sp;
+                frame.sp = current_stack_pointer;
                 frame.pc = (unsigned long)save_stack_trace_tsk;
         }
 
diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c
index 55a99b9a97e0..13ad4dbb1615 100644
--- a/arch/arm64/kernel/suspend.c
+++ b/arch/arm64/kernel/suspend.c
@@ -9,22 +9,19 @@
 #include <asm/suspend.h>
 #include <asm/tlbflush.h>
 
-extern int __cpu_suspend(unsigned long);
+extern int __cpu_suspend_enter(unsigned long arg, int (*fn)(unsigned long));
 /*
- * This is called by __cpu_suspend() to save the state, and do whatever
+ * This is called by __cpu_suspend_enter() to save the state, and do whatever
  * flushing is required to ensure that when the CPU goes to sleep we have
  * the necessary data available when the caches are not searched.
  *
- * @arg: Argument to pass to suspend operations
- * @ptr: CPU context virtual address
- * @save_ptr: address of the location where the context physical address
- *            must be saved
+ * ptr: CPU context virtual address
+ * save_ptr: address of the location where the context physical address
+ *           must be saved
  */
-int __cpu_suspend_finisher(unsigned long arg, struct cpu_suspend_ctx *ptr,
-                           phys_addr_t *save_ptr)
+void notrace __cpu_suspend_save(struct cpu_suspend_ctx *ptr,
+                                phys_addr_t *save_ptr)
 {
-        int cpu = smp_processor_id();
-
         *save_ptr = virt_to_phys(ptr);
 
         cpu_do_suspend(ptr);
@@ -35,8 +32,6 @@ int __cpu_suspend_finisher(unsigned long arg, struct cpu_suspend_ctx *ptr,
          */
         __flush_dcache_area(ptr, sizeof(*ptr));
         __flush_dcache_area(save_ptr, sizeof(*save_ptr));
-
-        return cpu_ops[cpu]->cpu_suspend(arg);
 }
 
 /*
@@ -56,15 +51,15 @@ void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
 }
 
 /**
- * cpu_suspend
+ * cpu_suspend() - function to enter a low-power state
+ * @arg: argument to pass to CPU suspend operations
  *
- * @arg: argument to pass to the finisher function
+ * Return: 0 on success, -EOPNOTSUPP if CPU suspend hook not initialized, CPU
+ * operations back-end error code otherwise.
  */
 int cpu_suspend(unsigned long arg)
 {
-        struct mm_struct *mm = current->active_mm;
-        int ret, cpu = smp_processor_id();
-        unsigned long flags;
+        int cpu = smp_processor_id();
 
         /*
          * If cpu_ops have not been registered or suspend
@@ -72,6 +67,21 @@ int cpu_suspend(unsigned long arg)
          */
         if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend)
                 return -EOPNOTSUPP;
+        return cpu_ops[cpu]->cpu_suspend(arg);
+}
+
+/*
+ * __cpu_suspend
+ *
+ * arg: argument to pass to the finisher function
+ * fn: finisher function pointer
+ *
+ */
+int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
+{
+        struct mm_struct *mm = current->active_mm;
+        int ret;
+        unsigned long flags;
 
         /*
          * From this point debug exceptions are disabled to prevent
@@ -86,7 +96,7 @@ int cpu_suspend(unsigned long arg)
          * page tables, so that the thread address space is properly
          * set-up on function return.
          */
-        ret = __cpu_suspend(arg);
+        ret = __cpu_suspend_enter(arg, fn);
         if (ret == 0) {
                 cpu_switch_mm(mm->pgd, mm);
                 flush_tlb_all();
@@ -95,7 +105,7 @@ int cpu_suspend(unsigned long arg)
                  * Restore per-cpu offset before any kernel
                  * subsystem relying on it has a chance to run.
                  */
-                set_my_cpu_offset(per_cpu_offset(cpu));
+                set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
 
                 /*
                  * Restore HW breakpoint registers to sane values
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 02cd3f023e9a..de1b085e7963 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -132,7 +132,6 @@ static void dump_instr(const char *lvl, struct pt_regs *regs)
 static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
 {
         struct stackframe frame;
-        const register unsigned long current_sp asm ("sp");
 
         pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);
 
@@ -145,7 +144,7 @@ static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
                 frame.pc = regs->pc;
         } else if (tsk == current) {
                 frame.fp = (unsigned long)__builtin_frame_address(0);
-                frame.sp = current_sp;
+                frame.sp = current_stack_pointer;
                 frame.pc = (unsigned long)dump_backtrace;
         } else {
                 /*