1 files changed, 32 insertions, 13 deletions

diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 0f0de30e3e3f..e8b6e5b8932c 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -623,7 +623,7 @@ static inline pgste_t pgste_get_lock(pte_t *ptep)
                 "       csg     %0,%1,%2\n"
                 "       jl      0b\n"
                 : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE])
-                : "Q" (ptep[PTRS_PER_PTE]) : "cc");
+                : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory");
 #endif
         return __pgste(new);
 }
@@ -635,18 +635,26 @@ static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
                 "       nihh    %1,0xff7f\n"    /* clear RCP_PCL_BIT */
                 "       stg     %1,%0\n"
                 : "=Q" (ptep[PTRS_PER_PTE])
-                : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) : "cc");
+                : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE])
+                : "cc", "memory");
         preempt_enable();
 #endif
 }
 
+static inline void pgste_set(pte_t *ptep, pgste_t pgste)
+{
+#ifdef CONFIG_PGSTE
+        *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste;
+#endif
+}
+
 static inline pgste_t pgste_update_all(pte_t *ptep, pgste_t pgste)
 {
 #ifdef CONFIG_PGSTE
         unsigned long address, bits;
         unsigned char skey;
 
-        if (!pte_present(*ptep))
+        if (pte_val(*ptep) & _PAGE_INVALID)
                 return pgste;
         address = pte_val(*ptep) & PAGE_MASK;
         skey = page_get_storage_key(address);
@@ -680,7 +688,7 @@ static inline pgste_t pgste_update_young(pte_t *ptep, pgste_t pgste)
 #ifdef CONFIG_PGSTE
         int young;
 
-        if (!pte_present(*ptep))
+        if (pte_val(*ptep) & _PAGE_INVALID)
                 return pgste;
         /* Get referenced bit from storage key */
         young = page_reset_referenced(pte_val(*ptep) & PAGE_MASK);
@@ -704,17 +712,19 @@ static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry)
 {
 #ifdef CONFIG_PGSTE
         unsigned long address;
-        unsigned long okey, nkey;
+        unsigned long nkey;
 
-        if (!pte_present(entry))
+        if (pte_val(entry) & _PAGE_INVALID)
                 return;
+        VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
         address = pte_val(entry) & PAGE_MASK;
-        okey = nkey = page_get_storage_key(address);
-        nkey &= ~(_PAGE_ACC_BITS | _PAGE_FP_BIT);
-        /* Set page access key and fetch protection bit from pgste */
-        nkey |= (pgste_val(pgste) & (RCP_ACC_BITS | RCP_FP_BIT)) >> 56;
-        if (okey != nkey)
-                page_set_storage_key(address, nkey, 0);
+        /*
+         * Set page access key and fetch protection bit from pgste.
+         * The guest C/R information is still in the PGSTE, set real
+         * key C/R to 0.
+         */
+        nkey = (pgste_val(pgste) & (RCP_ACC_BITS | RCP_FP_BIT)) >> 56;
+        page_set_storage_key(address, nkey, 0);
 #endif
 }
 
@@ -1098,6 +1108,11 @@ static inline pte_t ptep_modify_prot_start(struct mm_struct *mm,
         pte = *ptep;
         if (!mm_exclusive(mm))
                 __ptep_ipte(address, ptep);
+
+        if (mm_has_pgste(mm)) {
+                pgste = pgste_update_all(&pte, pgste);
+                pgste_set(ptep, pgste);
+        }
         return pte;
 }
 
@@ -1105,9 +1120,13 @@ static inline void ptep_modify_prot_commit(struct mm_struct *mm,
                                            unsigned long address,
                                            pte_t *ptep, pte_t pte)
 {
+        pgste_t pgste;
+
         if (mm_has_pgste(mm)) {
+                pgste = *(pgste_t *)(ptep + PTRS_PER_PTE);
+                pgste_set_key(ptep, pgste, pte);
                 pgste_set_pte(ptep, pte);
-                pgste_set_unlock(ptep, *(pgste_t *)(ptep + PTRS_PER_PTE));
+                pgste_set_unlock(ptep, pgste);
         } else
                 *ptep = pte;
 }