s390/pageattr: allow kernel page table splitting

set_memory_ro() and set_memory_rw() currently only work on 4k
mappings, which is good enough for module code aka the vmalloc area.

However we stumbled already twice into the need to make this also work
on larger mappings:
- the ro after init patch set
- the crash kernel resize code

Therefore this patch implements automatic kernel page table splitting
if e.g. set_memory_ro() would be called on parts of a 2G mapping.
This works quite the same as the x86 code, but is much simpler.

In order to make this work and to be architecturally compliant we now
always use the csp, cspg or crdte instructions to replace valid page
table entries. This means that set_memory_ro() and set_memory_rw()
will be much more expensive than before. In order to avoid huge
latencies the code contains a couple of cond_resched() calls.

The current code only splits page tables, but does not merge them if
it would be possible.  The reason for this is that currently there is
no real life scenarion where this would really happen. All current use
cases that I know of only change access rights once during the life
time. If that should change we can still implement kernel page table
merging at a later time.

Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

2 files changed, 214 insertions, 33 deletions

diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c
index e67a8f712e19..91e5e29c1f5c 100644
--- a/arch/s390/mm/pageattr.c
+++ b/arch/s390/mm/pageattr.c
@@ -40,54 +40,235 @@ void __storage_key_init_range(unsigned long start, unsigned long end)
 }
 #endif
 
-static pte_t *walk_page_table(unsigned long addr)
+static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr,
+                    unsigned long dtt)
 {
-        pgd_t *pgdp;
-        pud_t *pudp;
+        unsigned long table, mask;
+
+        mask = 0;
+        if (MACHINE_HAS_EDAT2) {
+                switch (dtt) {
+                case CRDTE_DTT_REGION3:
+                        mask = ~(PTRS_PER_PUD * sizeof(pud_t) - 1);
+                        break;
+                case CRDTE_DTT_SEGMENT:
+                        mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
+                        break;
+                case CRDTE_DTT_PAGE:
+                        mask = ~(PTRS_PER_PTE * sizeof(pte_t) - 1);
+                        break;
+                }
+                table = (unsigned long)old & mask;
+                crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce);
+        } else if (MACHINE_HAS_IDTE) {
+                cspg(old, *old, new);
+        } else {
+                csp((unsigned int *)old + 1, *old, new);
+        }
+}
+
+struct cpa {
+        unsigned int set_ro     : 1;
+        unsigned int clear_ro   : 1;
+};
+
+static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end,
+                          struct cpa cpa)
+{
+        pte_t *ptep, new;
+
+        ptep = pte_offset(pmdp, addr);
+        do {
+                if (pte_none(*ptep))
+                        return -EINVAL;
+                if (cpa.set_ro)
+                        new = pte_wrprotect(*ptep);
+                else if (cpa.clear_ro)
+                        new = pte_mkwrite(pte_mkdirty(*ptep));
+                pgt_set((unsigned long *)ptep, pte_val(new), addr, CRDTE_DTT_PAGE);
+                ptep++;
+                addr += PAGE_SIZE;
+                cond_resched();
+        } while (addr < end);
+        return 0;
+}
+
+static int split_pmd_page(pmd_t *pmdp, unsigned long addr)
+{
+        unsigned long pte_addr, prot;
+        pte_t *pt_dir, *ptep;
+        pmd_t new;
+        int i, ro;
+
+        pt_dir = vmem_pte_alloc();
+        if (!pt_dir)
+                return -ENOMEM;
+        pte_addr = pmd_pfn(*pmdp) << PAGE_SHIFT;
+        ro = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT);
+        prot = pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL);
+        ptep = pt_dir;
+        for (i = 0; i < PTRS_PER_PTE; i++) {
+                pte_val(*ptep) = pte_addr | prot;
+                pte_addr += PAGE_SIZE;
+                ptep++;
+        }
+        pmd_val(new) = __pa(pt_dir) | _SEGMENT_ENTRY;
+        pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT);
+        return 0;
+}
+
+static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, struct cpa cpa)
+{
+        pmd_t new;
+
+        if (cpa.set_ro)
+                new = pmd_wrprotect(*pmdp);
+        else if (cpa.clear_ro)
+                new = pmd_mkwrite(pmd_mkdirty(*pmdp));
+        pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT);
+}
+
+static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end,
+                          struct cpa cpa)
+{
+        unsigned long next;
         pmd_t *pmdp;
-        pte_t *ptep;
+        int rc = 0;
 
-        pgdp = pgd_offset_k(addr);
-        if (pgd_none(*pgdp))
-                return NULL;
-        pudp = pud_offset(pgdp, addr);
-        if (pud_none(*pudp) || pud_large(*pudp))
-                return NULL;
         pmdp = pmd_offset(pudp, addr);
-        if (pmd_none(*pmdp) || pmd_large(*pmdp))
-                return NULL;
-        ptep = pte_offset_kernel(pmdp, addr);
-        if (pte_none(*ptep))
-                return NULL;
-        return ptep;
+        do {
+                if (pmd_none(*pmdp))
+                        return -EINVAL;
+                next = pmd_addr_end(addr, end);
+                if (pmd_large(*pmdp)) {
+                        if (addr & ~PMD_MASK || addr + PMD_SIZE > next) {
+                                rc = split_pmd_page(pmdp, addr);
+                                if (rc)
+                                        return rc;
+                                continue;
+                        }
+                        modify_pmd_page(pmdp, addr, cpa);
+                } else {
+                        rc = walk_pte_level(pmdp, addr, next, cpa);
+                        if (rc)
+                                return rc;
+                }
+                pmdp++;
+                addr = next;
+                cond_resched();
+        } while (addr < end);
+        return rc;
 }
 
-static void change_page_attr(unsigned long addr, int numpages,
-                             pte_t (*set) (pte_t))
+static int split_pud_page(pud_t *pudp, unsigned long addr)
 {
-        pte_t *ptep;
-        int i;
+        unsigned long pmd_addr, prot;
+        pmd_t *pm_dir, *pmdp;
+        pud_t new;
+        int i, ro;
 
-        for (i = 0; i < numpages; i++) {
-                ptep = walk_page_table(addr);
-                if (WARN_ON_ONCE(!ptep))
-                        break;
-                *ptep = set(*ptep);
-                addr += PAGE_SIZE;
+        pm_dir = vmem_pmd_alloc();
+        if (!pm_dir)
+                return -ENOMEM;
+        pmd_addr = pud_pfn(*pudp) << PAGE_SHIFT;
+        ro = !!(pud_val(*pudp) & _REGION_ENTRY_PROTECT);
+        prot = pgprot_val(ro ? SEGMENT_KERNEL_RO : SEGMENT_KERNEL);
+        pmdp = pm_dir;
+        for (i = 0; i < PTRS_PER_PMD; i++) {
+                pmd_val(*pmdp) = pmd_addr | prot;
+                pmd_addr += PMD_SIZE;
+                pmdp++;
         }
-        __tlb_flush_kernel();
+        pud_val(new) = __pa(pm_dir) | _REGION3_ENTRY;
+        pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3);
+        return 0;
+}
+
+static void modify_pud_page(pud_t *pudp, unsigned long addr, struct cpa cpa)
+{
+        pud_t new;
+
+        if (cpa.set_ro)
+                new = pud_wrprotect(*pudp);
+        else if (cpa.clear_ro)
+                new = pud_mkwrite(pud_mkdirty(*pudp));
+        pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3);
+}
+
+static int walk_pud_level(pgd_t *pgd, unsigned long addr, unsigned long end,
+                          struct cpa cpa)
+{
+        unsigned long next;
+        pud_t *pudp;
+        int rc = 0;
+
+        pudp = pud_offset(pgd, addr);
+        do {
+                if (pud_none(*pudp))
+                        return -EINVAL;
+                next = pud_addr_end(addr, end);
+                if (pud_large(*pudp)) {
+                        if (addr & ~PUD_MASK || addr + PUD_SIZE > next) {
+                                rc = split_pud_page(pudp, addr);
+                                if (rc)
+                                        break;
+                                continue;
+                        }
+                        modify_pud_page(pudp, addr, cpa);
+                } else {
+                        rc = walk_pmd_level(pudp, addr, next, cpa);
+                }
+                pudp++;
+                addr = next;
+                cond_resched();
+        } while (addr < end && !rc);
+        return rc;
+}
+
+static DEFINE_MUTEX(cpa_mutex);
+
+static int change_page_attr(unsigned long addr, unsigned long end,
+                            struct cpa cpa)
+{
+        unsigned long next;
+        int rc = -EINVAL;
+        pgd_t *pgdp;
+
+        if (end >= MODULES_END)
+                return -EINVAL;
+        mutex_lock(&cpa_mutex);
+        pgdp = pgd_offset_k(addr);
+        do {
+                if (pgd_none(*pgdp))
+                        break;
+                next = pgd_addr_end(addr, end);
+                rc = walk_pud_level(pgdp, addr, next, cpa);
+                if (rc)
+                        break;
+                cond_resched();
+        } while (pgdp++, addr = next, addr < end && !rc);
+        mutex_unlock(&cpa_mutex);
+        return rc;
 }
 
 int set_memory_ro(unsigned long addr, int numpages)
 {
-        change_page_attr(addr, numpages, pte_wrprotect);
-        return 0;
+        struct cpa cpa = {
+                .set_ro = 1,
+        };
+
+        addr &= PAGE_MASK;
+        return change_page_attr(addr, addr + numpages * PAGE_SIZE, cpa);
 }
 
 int set_memory_rw(unsigned long addr, int numpages)
 {
-        change_page_attr(addr, numpages, pte_mkwrite);
-        return 0;
+        struct cpa cpa = {
+                .clear_ro = 1,
+        };
+
+        addr &= PAGE_MASK;
+        return change_page_attr(addr, addr + numpages * PAGE_SIZE, cpa);
 }
 
 /* not possible */
diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c
index 4badd8252e3c..0a7b03496f67 100644
--- a/arch/s390/mm/vmem.c
+++ b/arch/s390/mm/vmem.c
@@ -47,7 +47,7 @@ static inline pud_t *vmem_pud_alloc(void)
         return pud;
 }
 
-static inline pmd_t *vmem_pmd_alloc(void)
+pmd_t *vmem_pmd_alloc(void)
 {
         pmd_t *pmd = NULL;
 
@@ -58,7 +58,7 @@ static inline pmd_t *vmem_pmd_alloc(void)
         return pmd;
 }
 
-static pte_t __ref *vmem_pte_alloc(void)
+pte_t __ref *vmem_pte_alloc(void)
 {
         pte_t *pte;