Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Catalin Marinas:
 "arm64 updates for 3.20:

   - reimplementation of the virtual remapping of UEFI Runtime Services
     in a way that is stable across kexec
   - emulation of the "setend" instruction for 32-bit tasks (user
     endianness switching trapped in the kernel, SCTLR_EL1.E0E bit set
     accordingly)
   - compat_sys_call_table implemented in C (from asm) and made it a
     constant array together with sys_call_table
   - export CPU cache information via /sys (like other architectures)
   - DMA API implementation clean-up in preparation for IOMMU support
   - macros clean-up for KVM
   - dropped some unnecessary cache+tlb maintenance
   - CONFIG_ARM64_CPU_SUSPEND clean-up
   - defconfig update (CPU_IDLE)

  The EFI changes going via the arm64 tree have been acked by Matt
  Fleming.  There is also a patch adding sys_*stat64 prototypes to
  include/linux/syscalls.h, acked by Andrew Morton"

* tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (47 commits)
  arm64: compat: Remove incorrect comment in compat_siginfo
  arm64: Fix section mismatch on alloc_init_p[mu]d()
  arm64: Avoid breakage caused by .altmacro in fpsimd save/restore macros
  arm64: mm: use *_sect to check for section maps
  arm64: drop unnecessary cache+tlb maintenance
  arm64:mm: free the useless initial page table
  arm64: Enable CPU_IDLE in defconfig
  arm64: kernel: remove ARM64_CPU_SUSPEND config option
  arm64: make sys_call_table const
  arm64: Remove asm/syscalls.h
  arm64: Implement the compat_sys_call_table in C
  syscalls: Declare sys_*stat64 prototypes if __ARCH_WANT_(COMPAT_)STAT64
  compat: Declare compat_sys_sigpending and compat_sys_sigprocmask prototypes
  arm64: uapi: expose our struct ucontext to the uapi headers
  smp, ARM64: Kill SMP single function call interrupt
  arm64: Emulate SETEND for AArch32 tasks
  arm64: Consolidate hotplug notifier for instruction emulation
  arm64: Track system support for mixed endian EL0
  arm64: implement generic IOMMU configuration
  arm64: Combine coherent and non-coherent swiotlb dma_ops
  ...

8 files changed, 321 insertions, 211 deletions

diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index d92094203913..0a24b9b8c698 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -134,16 +134,17 @@ static void __dma_free_coherent(struct device *dev, size_t size,
                 swiotlb_free_coherent(dev, size, vaddr, dma_handle);
 }
 
-static void *__dma_alloc_noncoherent(struct device *dev, size_t size,
-                                     dma_addr_t *dma_handle, gfp_t flags,
-                                     struct dma_attrs *attrs)
+static void *__dma_alloc(struct device *dev, size_t size,
+                         dma_addr_t *dma_handle, gfp_t flags,
+                         struct dma_attrs *attrs)
 {
         struct page *page;
         void *ptr, *coherent_ptr;
+        bool coherent = is_device_dma_coherent(dev);
 
         size = PAGE_ALIGN(size);
 
-        if (!(flags & __GFP_WAIT)) {
+        if (!coherent && !(flags & __GFP_WAIT)) {
                 struct page *page = NULL;
                 void *addr = __alloc_from_pool(size, &page);
 
@@ -151,13 +152,16 @@ static void *__dma_alloc_noncoherent(struct device *dev, size_t size,
                         *dma_handle = phys_to_dma(dev, page_to_phys(page));
 
                 return addr;
-
         }
 
         ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs);
         if (!ptr)
                 goto no_mem;
 
+        /* no need for non-cacheable mapping if coherent */
+        if (coherent)
+                return ptr;
+
         /* remove any dirty cache lines on the kernel alias */
         __dma_flush_range(ptr, ptr + size);
 
@@ -179,15 +183,17 @@ no_mem:
         return NULL;
 }
 
-static void __dma_free_noncoherent(struct device *dev, size_t size,
-                                   void *vaddr, dma_addr_t dma_handle,
-                                   struct dma_attrs *attrs)
+static void __dma_free(struct device *dev, size_t size,
+                       void *vaddr, dma_addr_t dma_handle,
+                       struct dma_attrs *attrs)
 {
         void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle));
 
-        if (__free_from_pool(vaddr, size))
-                return;
-        vunmap(vaddr);
+        if (!is_device_dma_coherent(dev)) {
+                if (__free_from_pool(vaddr, size))
+                        return;
+                vunmap(vaddr);
+        }
         __dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs);
 }
 
@@ -199,7 +205,8 @@ static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page,
         dma_addr_t dev_addr;
 
         dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs);
-        __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
+        if (!is_device_dma_coherent(dev))
+                __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
 
         return dev_addr;
 }
@@ -209,7 +216,8 @@ static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr,
                                  size_t size, enum dma_data_direction dir,
                                  struct dma_attrs *attrs)
 {
-        __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
+        if (!is_device_dma_coherent(dev))
+                __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
         swiotlb_unmap_page(dev, dev_addr, size, dir, attrs);
 }
 
@@ -221,9 +229,10 @@ static int __swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
         int i, ret;
 
         ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs);
-        for_each_sg(sgl, sg, ret, i)
-                __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
-                               sg->length, dir);
+        if (!is_device_dma_coherent(dev))
+                for_each_sg(sgl, sg, ret, i)
+                        __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
+                                       sg->length, dir);
 
         return ret;
 }
@@ -236,9 +245,10 @@ static void __swiotlb_unmap_sg_attrs(struct device *dev,
         struct scatterlist *sg;
         int i;
 
-        for_each_sg(sgl, sg, nelems, i)
-                __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
-                                 sg->length, dir);
+        if (!is_device_dma_coherent(dev))
+                for_each_sg(sgl, sg, nelems, i)
+                        __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
+                                         sg->length, dir);
         swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs);
 }
 
@@ -246,7 +256,8 @@ static void __swiotlb_sync_single_for_cpu(struct device *dev,
                                           dma_addr_t dev_addr, size_t size,
                                           enum dma_data_direction dir)
 {
-        __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
+        if (!is_device_dma_coherent(dev))
+                __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
         swiotlb_sync_single_for_cpu(dev, dev_addr, size, dir);
 }
 
@@ -255,7 +266,8 @@ static void __swiotlb_sync_single_for_device(struct device *dev,
                                              enum dma_data_direction dir)
 {
         swiotlb_sync_single_for_device(dev, dev_addr, size, dir);
-        __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
+        if (!is_device_dma_coherent(dev))
+                __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
 }
 
 static void __swiotlb_sync_sg_for_cpu(struct device *dev,
@@ -265,9 +277,10 @@ static void __swiotlb_sync_sg_for_cpu(struct device *dev,
         struct scatterlist *sg;
         int i;
 
-        for_each_sg(sgl, sg, nelems, i)
-                __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
-                                 sg->length, dir);
+        if (!is_device_dma_coherent(dev))
+                for_each_sg(sgl, sg, nelems, i)
+                        __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
+                                         sg->length, dir);
         swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir);
 }
 
@@ -279,9 +292,10 @@ static void __swiotlb_sync_sg_for_device(struct device *dev,
         int i;
 
         swiotlb_sync_sg_for_device(dev, sgl, nelems, dir);
-        for_each_sg(sgl, sg, nelems, i)
-                __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
-                               sg->length, dir);
+        if (!is_device_dma_coherent(dev))
+                for_each_sg(sgl, sg, nelems, i)
+                        __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
+                                       sg->length, dir);
 }
 
 /* vma->vm_page_prot must be set appropriately before calling this function */
@@ -308,28 +322,20 @@ static int __dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
         return ret;
 }
 
-static int __swiotlb_mmap_noncoherent(struct device *dev,
-                struct vm_area_struct *vma,
-                void *cpu_addr, dma_addr_t dma_addr, size_t size,
-                struct dma_attrs *attrs)
-{
-        vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, false);
-        return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
-}
-
-static int __swiotlb_mmap_coherent(struct device *dev,
-                struct vm_area_struct *vma,
-                void *cpu_addr, dma_addr_t dma_addr, size_t size,
-                struct dma_attrs *attrs)
+static int __swiotlb_mmap(struct device *dev,
+                          struct vm_area_struct *vma,
+                          void *cpu_addr, dma_addr_t dma_addr, size_t size,
+                          struct dma_attrs *attrs)
 {
-        /* Just use whatever page_prot attributes were specified */
+        vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
+                                             is_device_dma_coherent(dev));
         return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
 }
 
-struct dma_map_ops noncoherent_swiotlb_dma_ops = {
-        .alloc = __dma_alloc_noncoherent,
-        .free = __dma_free_noncoherent,
-        .mmap = __swiotlb_mmap_noncoherent,
+static struct dma_map_ops swiotlb_dma_ops = {
+        .alloc = __dma_alloc,
+        .free = __dma_free,
+        .mmap = __swiotlb_mmap,
         .map_page = __swiotlb_map_page,
         .unmap_page = __swiotlb_unmap_page,
         .map_sg = __swiotlb_map_sg_attrs,
@@ -341,24 +347,6 @@ struct dma_map_ops noncoherent_swiotlb_dma_ops = {
         .dma_supported = swiotlb_dma_supported,
         .mapping_error = swiotlb_dma_mapping_error,
 };
-EXPORT_SYMBOL(noncoherent_swiotlb_dma_ops);
-
-struct dma_map_ops coherent_swiotlb_dma_ops = {
-        .alloc = __dma_alloc_coherent,
-        .free = __dma_free_coherent,
-        .mmap = __swiotlb_mmap_coherent,
-        .map_page = swiotlb_map_page,
-        .unmap_page = swiotlb_unmap_page,
-        .map_sg = swiotlb_map_sg_attrs,
-        .unmap_sg = swiotlb_unmap_sg_attrs,
-        .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
-        .sync_single_for_device = swiotlb_sync_single_for_device,
-        .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
-        .sync_sg_for_device = swiotlb_sync_sg_for_device,
-        .dma_supported = swiotlb_dma_supported,
-        .mapping_error = swiotlb_dma_mapping_error,
-};
-EXPORT_SYMBOL(coherent_swiotlb_dma_ops);
 
 extern int swiotlb_late_init_with_default_size(size_t default_size);
 
@@ -427,7 +415,7 @@ static int __init swiotlb_late_init(void)
 {
         size_t swiotlb_size = min(SZ_64M, MAX_ORDER_NR_PAGES << PAGE_SHIFT);
 
-        dma_ops = &noncoherent_swiotlb_dma_ops;
+        dma_ops = &swiotlb_dma_ops;
 
         return swiotlb_late_init_with_default_size(swiotlb_size);
 }
diff --git a/arch/arm64/mm/dump.c b/arch/arm64/mm/dump.c
index d54dc9ac4b70..74c256744b25 100644
--- a/arch/arm64/mm/dump.c
+++ b/arch/arm64/mm/dump.c
@@ -14,14 +14,18 @@
  * of the License.
  */
 #include <linux/debugfs.h>
+#include <linux/errno.h>
 #include <linux/fs.h>
 #include <linux/io.h>
+#include <linux/init.h>
 #include <linux/mm.h>
 #include <linux/sched.h>
 #include <linux/seq_file.h>
 
 #include <asm/fixmap.h>
+#include <asm/memory.h>
 #include <asm/pgtable.h>
+#include <asm/pgtable-hwdef.h>
 
 #define LOWEST_ADDR     (UL(0xffffffffffffffff) << VA_BITS)
 
@@ -37,10 +41,10 @@ enum address_markers_idx {
         VMEMMAP_START_NR,
         VMEMMAP_END_NR,
 #endif
-        PCI_START_NR,
-        PCI_END_NR,
         FIXADDR_START_NR,
         FIXADDR_END_NR,
+        PCI_START_NR,
+        PCI_END_NR,
         MODULES_START_NR,
         MODUELS_END_NR,
         KERNEL_SPACE_NR,
@@ -53,10 +57,10 @@ static struct addr_marker address_markers[] = {
         { 0,                    "vmemmap start" },
         { 0,                    "vmemmap end" },
 #endif
-        { (unsigned long) PCI_IOBASE,           "PCI I/O start" },
-        { (unsigned long) PCI_IOBASE + SZ_16M,  "PCI I/O end" },
         { FIXADDR_START,        "Fixmap start" },
         { FIXADDR_TOP,          "Fixmap end" },
+        { PCI_IO_START,         "PCI I/O start" },
+        { PCI_IO_END,           "PCI I/O end" },
         { MODULES_VADDR,        "Modules start" },
         { MODULES_END,          "Modules end" },
         { PAGE_OFFSET,          "Kernel Mapping" },
@@ -246,10 +250,12 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start)
 
         for (i = 0; i < PTRS_PER_PMD; i++, pmd++) {
                 addr = start + i * PMD_SIZE;
-                if (pmd_none(*pmd) || pmd_sect(*pmd) || pmd_bad(*pmd))
+                if (pmd_none(*pmd) || pmd_sect(*pmd)) {
                         note_page(st, addr, 3, pmd_val(*pmd));
-                else
+                } else {
+                        BUG_ON(pmd_bad(*pmd));
                         walk_pte(st, pmd, addr);
+                }
         }
 }
 
@@ -261,10 +267,12 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
 
         for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
                 addr = start + i * PUD_SIZE;
-                if (pud_none(*pud) || pud_sect(*pud) || pud_bad(*pud))
+                if (pud_none(*pud) || pud_sect(*pud)) {
                         note_page(st, addr, 2, pud_val(*pud));
-                else
+                } else {
+                        BUG_ON(pud_bad(*pud));
                         walk_pmd(st, pud, addr);
+                }
         }
 }
 
@@ -276,10 +284,12 @@ static void walk_pgd(struct pg_state *st, struct mm_struct *mm, unsigned long st
 
         for (i = 0; i < PTRS_PER_PGD; i++, pgd++) {
                 addr = start + i * PGDIR_SIZE;
-                if (pgd_none(*pgd) || pgd_bad(*pgd))
+                if (pgd_none(*pgd)) {
                         note_page(st, addr, 1, pgd_val(*pgd));
-                else
+                } else {
+                        BUG_ON(pgd_bad(*pgd));
                         walk_pud(st, pgd, addr);
+                }
         }
 }
 
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index c11cd27ca8f5..96da13167d4a 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -219,7 +219,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
 
         if (esr & ESR_LNX_EXEC) {
                 vm_flags = VM_EXEC;
-        } else if ((esr & ESR_EL1_WRITE) && !(esr & ESR_EL1_CM)) {
+        } else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) {
                 vm_flags = VM_WRITE;
                 mm_flags |= FAULT_FLAG_WRITE;
         }
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index c95464a33f36..71145f952070 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -35,6 +35,7 @@
 #include <linux/efi.h>
 
 #include <asm/fixmap.h>
+#include <asm/memory.h>
 #include <asm/sections.h>
 #include <asm/setup.h>
 #include <asm/sizes.h>
@@ -136,10 +137,29 @@ static void arm64_memory_present(void)
 }
 #endif
 
+static phys_addr_t memory_limit = (phys_addr_t)ULLONG_MAX;
+
+/*
+ * Limit the memory size that was specified via FDT.
+ */
+static int __init early_mem(char *p)
+{
+        if (!p)
+                return 1;
+
+        memory_limit = memparse(p, &p) & PAGE_MASK;
+        pr_notice("Memory limited to %lldMB\n", memory_limit >> 20);
+
+        return 0;
+}
+early_param("mem", early_mem);
+
 void __init arm64_memblock_init(void)
 {
         phys_addr_t dma_phys_limit = 0;
 
+        memblock_enforce_memory_limit(memory_limit);
+
         /*
          * Register the kernel text, kernel data, initrd, and initial
          * pagetables with memblock.
@@ -277,8 +297,8 @@ void __init mem_init(void)
                   "    vmemmap : 0x%16lx - 0x%16lx   (%6ld GB maximum)\n"
                   "              0x%16lx - 0x%16lx   (%6ld MB actual)\n"
 #endif
-                  "    PCI I/O : 0x%16lx - 0x%16lx   (%6ld MB)\n"
                   "    fixed   : 0x%16lx - 0x%16lx   (%6ld KB)\n"
+                  "    PCI I/O : 0x%16lx - 0x%16lx   (%6ld MB)\n"
                   "    modules : 0x%16lx - 0x%16lx   (%6ld MB)\n"
                   "    memory  : 0x%16lx - 0x%16lx   (%6ld MB)\n"
                   "      .init : 0x%p" " - 0x%p" "   (%6ld KB)\n"
@@ -291,8 +311,8 @@ void __init mem_init(void)
                   MLM((unsigned long)virt_to_page(PAGE_OFFSET),
                       (unsigned long)virt_to_page(high_memory)),
 #endif
-                  MLM((unsigned long)PCI_IOBASE, (unsigned long)PCI_IOBASE + SZ_16M),
                   MLK(FIXADDR_START, FIXADDR_TOP),
+                  MLM(PCI_IO_START, PCI_IO_END),
                   MLM(MODULES_VADDR, MODULES_END),
                   MLM(PAGE_OFFSET, (unsigned long)high_memory),
                   MLK_ROUNDUP(__init_begin, __init_end),
@@ -325,6 +345,7 @@ void __init mem_init(void)
 
 void free_initmem(void)
 {
+        fixup_init();
         free_initmem_default(0);
         free_alternatives_memory();
 }
diff --git a/arch/arm64/mm/ioremap.c b/arch/arm64/mm/ioremap.c
index cbb99c8f1e04..01e88c8bcab0 100644
--- a/arch/arm64/mm/ioremap.c
+++ b/arch/arm64/mm/ioremap.c
@@ -62,6 +62,7 @@ static void __iomem *__ioremap_caller(phys_addr_t phys_addr, size_t size,
         if (!area)
                 return NULL;
         addr = (unsigned long)area->addr;
+        area->phys_addr = phys_addr;
 
         err = ioremap_page_range(addr, addr + size, phys_addr, prot);
         if (err) {
diff --git a/arch/arm64/mm/mm.h b/arch/arm64/mm/mm.h
index 50c3351df9c7..ef47d99b5cbc 100644
--- a/arch/arm64/mm/mm.h
+++ b/arch/arm64/mm/mm.h
@@ -1 +1,3 @@
 extern void __init bootmem_init(void);
+
+void fixup_init(void);
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 6032f3e3056a..c6daaf6c6f97 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -26,6 +26,8 @@
 #include <linux/memblock.h>
 #include <linux/fs.h>
 #include <linux/io.h>
+#include <linux/slab.h>
+#include <linux/stop_machine.h>
 
 #include <asm/cputype.h>
 #include <asm/fixmap.h>
@@ -45,80 +47,6 @@
 struct page *empty_zero_page;
 EXPORT_SYMBOL(empty_zero_page);
 
-struct cachepolicy {
-        const char      policy[16];
-        u64             mair;
-        u64             tcr;
-};
-
-static struct cachepolicy cache_policies[] __initdata = {
-        {
-                .policy         = "uncached",
-                .mair           = 0x44,                 /* inner, outer non-cacheable */
-                .tcr            = TCR_IRGN_NC | TCR_ORGN_NC,
-        }, {
-                .policy         = "writethrough",
-                .mair           = 0xaa,                 /* inner, outer write-through, read-allocate */
-                .tcr            = TCR_IRGN_WT | TCR_ORGN_WT,
-        }, {
-                .policy         = "writeback",
-                .mair           = 0xee,                 /* inner, outer write-back, read-allocate */
-                .tcr            = TCR_IRGN_WBnWA | TCR_ORGN_WBnWA,
-        }
-};
-
-/*
- * These are useful for identifying cache coherency problems by allowing the
- * cache or the cache and writebuffer to be turned off. It changes the Normal
- * memory caching attributes in the MAIR_EL1 register.
- */
-static int __init early_cachepolicy(char *p)
-{
-        int i;
-        u64 tmp;
-
-        for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
-                int len = strlen(cache_policies[i].policy);
-
-                if (memcmp(p, cache_policies[i].policy, len) == 0)
-                        break;
-        }
-        if (i == ARRAY_SIZE(cache_policies)) {
-                pr_err("ERROR: unknown or unsupported cache policy: %s\n", p);
-                return 0;
-        }
-
-        flush_cache_all();
-
-        /*
-         * Modify MT_NORMAL attributes in MAIR_EL1.
-         */
-        asm volatile(
-        "       mrs     %0, mair_el1\n"
-        "       bfi     %0, %1, %2, #8\n"
-        "       msr     mair_el1, %0\n"
-        "       isb\n"
-        : "=&r" (tmp)
-        : "r" (cache_policies[i].mair), "i" (MT_NORMAL * 8));
-
-        /*
-         * Modify TCR PTW cacheability attributes.
-         */
-        asm volatile(
-        "       mrs     %0, tcr_el1\n"
-        "       bic     %0, %0, %2\n"
-        "       orr     %0, %0, %1\n"
-        "       msr     tcr_el1, %0\n"
-        "       isb\n"
-        : "=&r" (tmp)
-        : "r" (cache_policies[i].tcr), "r" (TCR_IRGN_MASK | TCR_ORGN_MASK));
-
-        flush_cache_all();
-
-        return 0;
-}
-early_param("cachepolicy", early_cachepolicy);
-
 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
                               unsigned long size, pgprot_t vma_prot)
 {
@@ -133,19 +61,42 @@ EXPORT_SYMBOL(phys_mem_access_prot);
 static void __init *early_alloc(unsigned long sz)
 {
         void *ptr = __va(memblock_alloc(sz, sz));
+        BUG_ON(!ptr);
         memset(ptr, 0, sz);
         return ptr;
 }
 
-static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
+/*
+ * remap a PMD into pages
+ */
+static void split_pmd(pmd_t *pmd, pte_t *pte)
+{
+        unsigned long pfn = pmd_pfn(*pmd);
+        int i = 0;
+
+        do {
+                /*
+                 * Need to have the least restrictive permissions available
+                 * permissions will be fixed up later
+                 */
+                set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC));
+                pfn++;
+        } while (pte++, i++, i < PTRS_PER_PTE);
+}
+
+static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
                                   unsigned long end, unsigned long pfn,
-                                  pgprot_t prot)
+                                  pgprot_t prot,
+                                  void *(*alloc)(unsigned long size))
 {
         pte_t *pte;
 
-        if (pmd_none(*pmd)) {
-                pte = early_alloc(PTRS_PER_PTE * sizeof(pte_t));
+        if (pmd_none(*pmd) || pmd_sect(*pmd)) {
+                pte = alloc(PTRS_PER_PTE * sizeof(pte_t));
+                if (pmd_sect(*pmd))
+                        split_pmd(pmd, pte);
                 __pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE);
+                flush_tlb_all();
         }
         BUG_ON(pmd_bad(*pmd));
 
@@ -156,30 +107,42 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
         } while (pte++, addr += PAGE_SIZE, addr != end);
 }
 
-static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
-                                  unsigned long end, phys_addr_t phys,
-                                  int map_io)
+void split_pud(pud_t *old_pud, pmd_t *pmd)
+{
+        unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT;
+        pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr);
+        int i = 0;
+
+        do {
+                set_pmd(pmd, __pmd(addr | prot));
+                addr += PMD_SIZE;
+        } while (pmd++, i++, i < PTRS_PER_PMD);
+}
+
+static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud,
+                                  unsigned long addr, unsigned long end,
+                                  phys_addr_t phys, pgprot_t prot,
+                                  void *(*alloc)(unsigned long size))
 {
         pmd_t *pmd;
         unsigned long next;
-        pmdval_t prot_sect;
-        pgprot_t prot_pte;
-
-        if (map_io) {
-                prot_sect = PROT_SECT_DEVICE_nGnRE;
-                prot_pte = __pgprot(PROT_DEVICE_nGnRE);
-        } else {
-                prot_sect = PROT_SECT_NORMAL_EXEC;
-                prot_pte = PAGE_KERNEL_EXEC;
-        }
 
         /*
          * Check for initial section mappings in the pgd/pud and remove them.
          */
-        if (pud_none(*pud) || pud_bad(*pud)) {
-                pmd = early_alloc(PTRS_PER_PMD * sizeof(pmd_t));
-                pud_populate(&init_mm, pud, pmd);
+        if (pud_none(*pud) || pud_sect(*pud)) {
+                pmd = alloc(PTRS_PER_PMD * sizeof(pmd_t));
+                if (pud_sect(*pud)) {
+                        /*
+                         * need to have the 1G of mappings continue to be
+                         * present
+                         */
+                        split_pud(pud, pmd);
+                }
+                pud_populate(mm, pud, pmd);
+                flush_tlb_all();
         }
+        BUG_ON(pud_bad(*pud));
 
         pmd = pmd_offset(pud, addr);
         do {
@@ -187,31 +150,51 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
                 /* try section mapping first */
                 if (((addr | next | phys) & ~SECTION_MASK) == 0) {
                         pmd_t old_pmd =*pmd;
-                        set_pmd(pmd, __pmd(phys | prot_sect));
+                        set_pmd(pmd, __pmd(phys |
+                                           pgprot_val(mk_sect_prot(prot))));
                         /*
                          * Check for previous table entries created during
                          * boot (__create_page_tables) and flush them.
                          */
-                        if (!pmd_none(old_pmd))
+                        if (!pmd_none(old_pmd)) {
                                 flush_tlb_all();
+                                if (pmd_table(old_pmd)) {
+                                        phys_addr_t table = __pa(pte_offset_map(&old_pmd, 0));
+                                        if (!WARN_ON_ONCE(slab_is_available()))
+                                                memblock_free(table, PAGE_SIZE);
+                                }
+                        }
                 } else {
                         alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys),
-                                       prot_pte);
+                                       prot, alloc);
                 }
                 phys += next - addr;
         } while (pmd++, addr = next, addr != end);
 }
 
-static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
-                                  unsigned long end, phys_addr_t phys,
-                                  int map_io)
+static inline bool use_1G_block(unsigned long addr, unsigned long next,
+                        unsigned long phys)
+{
+        if (PAGE_SHIFT != 12)
+                return false;
+
+        if (((addr | next | phys) & ~PUD_MASK) != 0)
+                return false;
+
+        return true;
+}
+
+static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd,
+                                  unsigned long addr, unsigned long end,
+                                  phys_addr_t phys, pgprot_t prot,
+                                  void *(*alloc)(unsigned long size))
 {
         pud_t *pud;
         unsigned long next;
 
         if (pgd_none(*pgd)) {
-                pud = early_alloc(PTRS_PER_PUD * sizeof(pud_t));
-                pgd_populate(&init_mm, pgd, pud);
+                pud = alloc(PTRS_PER_PUD * sizeof(pud_t));
+                pgd_populate(mm, pgd, pud);
         }
         BUG_ON(pgd_bad(*pgd));
 
@@ -222,10 +205,10 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
                 /*
                  * For 4K granule only, attempt to put down a 1GB block
                  */
-                if (!map_io && (PAGE_SHIFT == 12) &&
-                    ((addr | next | phys) & ~PUD_MASK) == 0) {
+                if (use_1G_block(addr, next, phys)) {
                         pud_t old_pud = *pud;
-                        set_pud(pud, __pud(phys | PROT_SECT_NORMAL_EXEC));
+                        set_pud(pud, __pud(phys |
+                                           pgprot_val(mk_sect_prot(prot))));
 
                         /*
                          * If we have an old value for a pud, it will
@@ -235,12 +218,15 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
                          * Look up the old pmd table and free it.
                          */
                         if (!pud_none(old_pud)) {
-                                phys_addr_t table = __pa(pmd_offset(&old_pud, 0));
-                                memblock_free(table, PAGE_SIZE);
                                 flush_tlb_all();
+                                if (pud_table(old_pud)) {
+                                        phys_addr_t table = __pa(pmd_offset(&old_pud, 0));
+                                        if (!WARN_ON_ONCE(slab_is_available()))
+                                                memblock_free(table, PAGE_SIZE);
+                                }
                         }
                 } else {
-                        alloc_init_pmd(pud, addr, next, phys, map_io);
+                        alloc_init_pmd(mm, pud, addr, next, phys, prot, alloc);
                 }
                 phys += next - addr;
         } while (pud++, addr = next, addr != end);
@@ -250,9 +236,10 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
  * Create the page directory entries and any necessary page tables for the
  * mapping specified by 'md'.
  */
-static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys,
-                                    unsigned long virt, phys_addr_t size,
-                                    int map_io)
+static void  __create_mapping(struct mm_struct *mm, pgd_t *pgd,
+                                    phys_addr_t phys, unsigned long virt,
+                                    phys_addr_t size, pgprot_t prot,
+                                    void *(*alloc)(unsigned long size))
 {
         unsigned long addr, length, end, next;
 
@@ -262,31 +249,95 @@ static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys,
         end = addr + length;
         do {
                 next = pgd_addr_end(addr, end);
-                alloc_init_pud(pgd, addr, next, phys, map_io);
+                alloc_init_pud(mm, pgd, addr, next, phys, prot, alloc);
                 phys += next - addr;
         } while (pgd++, addr = next, addr != end);
 }
 
-static void __init create_mapping(phys_addr_t phys, unsigned long virt,
-                                  phys_addr_t size)
+static void *late_alloc(unsigned long size)
+{
+        void *ptr;
+
+        BUG_ON(size > PAGE_SIZE);
+        ptr = (void *)__get_free_page(PGALLOC_GFP);
+        BUG_ON(!ptr);
+        return ptr;
+}
+
+static void __ref create_mapping(phys_addr_t phys, unsigned long virt,
+                                  phys_addr_t size, pgprot_t prot)
 {
         if (virt < VMALLOC_START) {
                 pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
                         &phys, virt);
                 return;
         }
-        __create_mapping(pgd_offset_k(virt & PAGE_MASK), phys, virt, size, 0);
+        __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), phys, virt,
+                         size, prot, early_alloc);
+}
+
+void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
+                               unsigned long virt, phys_addr_t size,
+                               pgprot_t prot)
+{
+        __create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot,
+                                late_alloc);
 }
 
-void __init create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io)
+static void create_mapping_late(phys_addr_t phys, unsigned long virt,
+                                  phys_addr_t size, pgprot_t prot)
 {
-        if ((addr >> PGDIR_SHIFT) >= ARRAY_SIZE(idmap_pg_dir)) {
-                pr_warn("BUG: not creating id mapping for %pa\n", &addr);
+        if (virt < VMALLOC_START) {
+                pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
+                        &phys, virt);
                 return;
         }
-        __create_mapping(&idmap_pg_dir[pgd_index(addr)],
-                         addr, addr, size, map_io);
+
+        return __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK),
+                                phys, virt, size, prot, late_alloc);
+}
+
+#ifdef CONFIG_DEBUG_RODATA
+static void __init __map_memblock(phys_addr_t start, phys_addr_t end)
+{
+        /*
+         * Set up the executable regions using the existing section mappings
+         * for now. This will get more fine grained later once all memory
+         * is mapped
+         */
+        unsigned long kernel_x_start = round_down(__pa(_stext), SECTION_SIZE);
+        unsigned long kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE);
+
+        if (end < kernel_x_start) {
+                create_mapping(start, __phys_to_virt(start),
+                        end - start, PAGE_KERNEL);
+        } else if (start >= kernel_x_end) {
+                create_mapping(start, __phys_to_virt(start),
+                        end - start, PAGE_KERNEL);
+        } else {
+                if (start < kernel_x_start)
+                        create_mapping(start, __phys_to_virt(start),
+                                kernel_x_start - start,
+                                PAGE_KERNEL);
+                create_mapping(kernel_x_start,
+                                __phys_to_virt(kernel_x_start),
+                                kernel_x_end - kernel_x_start,
+                                PAGE_KERNEL_EXEC);
+                if (kernel_x_end < end)
+                        create_mapping(kernel_x_end,
+                                __phys_to_virt(kernel_x_end),
+                                end - kernel_x_end,
+                                PAGE_KERNEL);
+        }
+
 }
+#else
+static void __init __map_memblock(phys_addr_t start, phys_addr_t end)
+{
+        create_mapping(start, __phys_to_virt(start), end - start,
+                        PAGE_KERNEL_EXEC);
+}
+#endif
 
 static void __init map_mem(void)
 {
@@ -332,14 +383,53 @@ static void __init map_mem(void)
                         memblock_set_current_limit(limit);
                 }
 #endif
-
-                create_mapping(start, __phys_to_virt(start), end - start);
+                __map_memblock(start, end);
         }
 
         /* Limit no longer required. */
         memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
 }
 
+void __init fixup_executable(void)
+{
+#ifdef CONFIG_DEBUG_RODATA
+        /* now that we are actually fully mapped, make the start/end more fine grained */
+        if (!IS_ALIGNED((unsigned long)_stext, SECTION_SIZE)) {
+                unsigned long aligned_start = round_down(__pa(_stext),
+                                                        SECTION_SIZE);
+
+                create_mapping(aligned_start, __phys_to_virt(aligned_start),
+                                __pa(_stext) - aligned_start,
+                                PAGE_KERNEL);
+        }
+
+        if (!IS_ALIGNED((unsigned long)__init_end, SECTION_SIZE)) {
+                unsigned long aligned_end = round_up(__pa(__init_end),
+                                                        SECTION_SIZE);
+                create_mapping(__pa(__init_end), (unsigned long)__init_end,
+                                aligned_end - __pa(__init_end),
+                                PAGE_KERNEL);
+        }
+#endif
+}
+
+#ifdef CONFIG_DEBUG_RODATA
+void mark_rodata_ro(void)
+{
+        create_mapping_late(__pa(_stext), (unsigned long)_stext,
+                                (unsigned long)_etext - (unsigned long)_stext,
+                                PAGE_KERNEL_EXEC | PTE_RDONLY);
+
+}
+#endif
+
+void fixup_init(void)
+{
+        create_mapping_late(__pa(__init_begin), (unsigned long)__init_begin,
+                        (unsigned long)__init_end - (unsigned long)__init_begin,
+                        PAGE_KERNEL);
+}
+
 /*
  * paging_init() sets up the page tables, initialises the zone memory
  * maps and sets up the zero page.
@@ -349,13 +439,7 @@ void __init paging_init(void)
         void *zero_page;
 
         map_mem();
-
-        /*
-         * Finally flush the caches and tlb to ensure that we're in a
-         * consistent state.
-         */
-        flush_cache_all();
-        flush_tlb_all();
+        fixup_executable();
 
         /* allocate the zero page. */
         zero_page = early_alloc(PAGE_SIZE);
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 4e778b13291b..28eebfb6af76 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -102,7 +102,7 @@ ENTRY(cpu_do_idle)
         ret
 ENDPROC(cpu_do_idle)
 
-#ifdef CONFIG_ARM64_CPU_SUSPEND
+#ifdef CONFIG_CPU_PM
 /**
  * cpu_do_suspend - save CPU registers context
  *
@@ -244,14 +244,18 @@ ENTRY(__cpu_setup)
 ENDPROC(__cpu_setup)
 
         /*
+         * We set the desired value explicitly, including those of the
+         * reserved bits. The values of bits EE & E0E were set early in
+         * el2_setup, which are left untouched below.
+         *
          *                 n n            T
          *       U E      WT T UD     US IHBS
          *       CE0      XWHW CZ     ME TEEA S
          * .... .IEE .... NEAI TE.I ..AD DEN0 ACAM
-         * 0011 0... 1101 ..0. ..0. 10.. .... .... < hardware reserved
-         * .... .1.. .... 01.1 11.1 ..01 0001 1101 < software settings
+         * 0011 0... 1101 ..0. ..0. 10.. .0.. .... < hardware reserved
+         * .... .1.. .... 01.1 11.1 ..01 0.01 1101 < software settings
          */
         .type   crval, #object
 crval:
-        .word   0x000802e2                      // clear
-        .word   0x0405d11d                      // set
+        .word   0xfcffffff                      // clear
+        .word   0x34d5d91d                      // set