1 files changed, 9 insertions, 5 deletions

diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 71baed7e592a..c0af47617299 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -205,7 +205,8 @@ ENDPROC(idmap_cpu_replace_ttbr1)
         dc      cvac, cur_\()\type\()p          // Ensure any existing dirty
         dmb     sy                              // lines are written back before
         ldr     \type, [cur_\()\type\()p]       // loading the entry
-        tbz     \type, #0, next_\()\type        // Skip invalid entries
+        tbz     \type, #0, skip_\()\type        // Skip invalid and
+        tbnz    \type, #11, skip_\()\type       // non-global entries
         .endm
 
         .macro __idmap_kpti_put_pgtable_ent_ng, type
@@ -265,8 +266,9 @@ ENTRY(idmap_kpti_install_ng_mappings)
         add     end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8)
 do_pgd: __idmap_kpti_get_pgtable_ent    pgd
         tbnz    pgd, #1, walk_puds
-        __idmap_kpti_put_pgtable_ent_ng pgd
 next_pgd:
+        __idmap_kpti_put_pgtable_ent_ng pgd
+skip_pgd:
         add     cur_pgdp, cur_pgdp, #8
         cmp     cur_pgdp, end_pgdp
         b.ne    do_pgd
@@ -294,8 +296,9 @@ walk_puds:
         add     end_pudp, cur_pudp, #(PTRS_PER_PUD * 8)
 do_pud: __idmap_kpti_get_pgtable_ent    pud
         tbnz    pud, #1, walk_pmds
-        __idmap_kpti_put_pgtable_ent_ng pud
 next_pud:
+        __idmap_kpti_put_pgtable_ent_ng pud
+skip_pud:
         add     cur_pudp, cur_pudp, 8
         cmp     cur_pudp, end_pudp
         b.ne    do_pud
@@ -314,8 +317,9 @@ walk_pmds:
         add     end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8)
 do_pmd: __idmap_kpti_get_pgtable_ent    pmd
         tbnz    pmd, #1, walk_ptes
-        __idmap_kpti_put_pgtable_ent_ng pmd
 next_pmd:
+        __idmap_kpti_put_pgtable_ent_ng pmd
+skip_pmd:
         add     cur_pmdp, cur_pmdp, #8
         cmp     cur_pmdp, end_pmdp
         b.ne    do_pmd
@@ -333,7 +337,7 @@ walk_ptes:
         add     end_ptep, cur_ptep, #(PTRS_PER_PTE * 8)
 do_pte: __idmap_kpti_get_pgtable_ent    pte
         __idmap_kpti_put_pgtable_ent_ng pte
-next_pte:
+skip_pte:
         add     cur_ptep, cur_ptep, #8
         cmp     cur_ptep, end_ptep
         b.ne    do_pte