aboutsummaryrefslogtreecommitdiffstats
path: root/arch/s390/include/asm/pgtable.h
diff options
context:
space:
mode:
Diffstat (limited to 'arch/s390/include/asm/pgtable.h')
-rw-r--r--arch/s390/include/asm/pgtable.h636
1 files changed, 121 insertions, 515 deletions
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 64ead8091248..2f66645587a2 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -298,15 +298,15 @@ static inline int is_module_addr(void *addr)
298 298
299/* 299/*
300 * Segment table entry encoding (R = read-only, I = invalid, y = young bit): 300 * Segment table entry encoding (R = read-only, I = invalid, y = young bit):
301 * dy..R...I...wr 301 * dy..R...I...rw
302 * prot-none, clean, old 00..1...1...00 302 * prot-none, clean, old 00..1...1...00
303 * prot-none, clean, young 01..1...1...00 303 * prot-none, clean, young 01..1...1...00
304 * prot-none, dirty, old 10..1...1...00 304 * prot-none, dirty, old 10..1...1...00
305 * prot-none, dirty, young 11..1...1...00 305 * prot-none, dirty, young 11..1...1...00
306 * read-only, clean, old 00..1...1...01 306 * read-only, clean, old 00..1...1...10
307 * read-only, clean, young 01..1...0...01 307 * read-only, clean, young 01..1...0...10
308 * read-only, dirty, old 10..1...1...01 308 * read-only, dirty, old 10..1...1...10
309 * read-only, dirty, young 11..1...0...01 309 * read-only, dirty, young 11..1...0...10
310 * read-write, clean, old 00..1...1...11 310 * read-write, clean, old 00..1...1...11
311 * read-write, clean, young 01..1...0...11 311 * read-write, clean, young 01..1...0...11
312 * read-write, dirty, old 10..0...1...11 312 * read-write, dirty, old 10..0...1...11
@@ -520,15 +520,6 @@ static inline int pmd_bad(pmd_t pmd)
520 return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0; 520 return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0;
521} 521}
522 522
523#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
524extern int pmdp_set_access_flags(struct vm_area_struct *vma,
525 unsigned long address, pmd_t *pmdp,
526 pmd_t entry, int dirty);
527
528#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
529extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
530 unsigned long address, pmd_t *pmdp);
531
532#define __HAVE_ARCH_PMD_WRITE 523#define __HAVE_ARCH_PMD_WRITE
533static inline int pmd_write(pmd_t pmd) 524static inline int pmd_write(pmd_t pmd)
534{ 525{
@@ -631,208 +622,6 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
631 return pmd; 622 return pmd;
632} 623}
633 624
634static inline pgste_t pgste_get_lock(pte_t *ptep)
635{
636 unsigned long new = 0;
637#ifdef CONFIG_PGSTE
638 unsigned long old;
639
640 preempt_disable();
641 asm(
642 " lg %0,%2\n"
643 "0: lgr %1,%0\n"
644 " nihh %0,0xff7f\n" /* clear PCL bit in old */
645 " oihh %1,0x0080\n" /* set PCL bit in new */
646 " csg %0,%1,%2\n"
647 " jl 0b\n"
648 : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE])
649 : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory");
650#endif
651 return __pgste(new);
652}
653
654static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
655{
656#ifdef CONFIG_PGSTE
657 asm(
658 " nihh %1,0xff7f\n" /* clear PCL bit */
659 " stg %1,%0\n"
660 : "=Q" (ptep[PTRS_PER_PTE])
661 : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE])
662 : "cc", "memory");
663 preempt_enable();
664#endif
665}
666
667static inline pgste_t pgste_get(pte_t *ptep)
668{
669 unsigned long pgste = 0;
670#ifdef CONFIG_PGSTE
671 pgste = *(unsigned long *)(ptep + PTRS_PER_PTE);
672#endif
673 return __pgste(pgste);
674}
675
676static inline void pgste_set(pte_t *ptep, pgste_t pgste)
677{
678#ifdef CONFIG_PGSTE
679 *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste;
680#endif
681}
682
683static inline pgste_t pgste_update_all(pte_t *ptep, pgste_t pgste,
684 struct mm_struct *mm)
685{
686#ifdef CONFIG_PGSTE
687 unsigned long address, bits, skey;
688
689 if (!mm_use_skey(mm) || pte_val(*ptep) & _PAGE_INVALID)
690 return pgste;
691 address = pte_val(*ptep) & PAGE_MASK;
692 skey = (unsigned long) page_get_storage_key(address);
693 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
694 /* Transfer page changed & referenced bit to guest bits in pgste */
695 pgste_val(pgste) |= bits << 48; /* GR bit & GC bit */
696 /* Copy page access key and fetch protection bit to pgste */
697 pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT);
698 pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
699#endif
700 return pgste;
701
702}
703
704static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,
705 struct mm_struct *mm)
706{
707#ifdef CONFIG_PGSTE
708 unsigned long address;
709 unsigned long nkey;
710
711 if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID)
712 return;
713 VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
714 address = pte_val(entry) & PAGE_MASK;
715 /*
716 * Set page access key and fetch protection bit from pgste.
717 * The guest C/R information is still in the PGSTE, set real
718 * key C/R to 0.
719 */
720 nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
721 nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
722 page_set_storage_key(address, nkey, 0);
723#endif
724}
725
726static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)
727{
728 if ((pte_val(entry) & _PAGE_PRESENT) &&
729 (pte_val(entry) & _PAGE_WRITE) &&
730 !(pte_val(entry) & _PAGE_INVALID)) {
731 if (!MACHINE_HAS_ESOP) {
732 /*
733 * Without enhanced suppression-on-protection force
734 * the dirty bit on for all writable ptes.
735 */
736 pte_val(entry) |= _PAGE_DIRTY;
737 pte_val(entry) &= ~_PAGE_PROTECT;
738 }
739 if (!(pte_val(entry) & _PAGE_PROTECT))
740 /* This pte allows write access, set user-dirty */
741 pgste_val(pgste) |= PGSTE_UC_BIT;
742 }
743 *ptep = entry;
744 return pgste;
745}
746
747/**
748 * struct gmap_struct - guest address space
749 * @crst_list: list of all crst tables used in the guest address space
750 * @mm: pointer to the parent mm_struct
751 * @guest_to_host: radix tree with guest to host address translation
752 * @host_to_guest: radix tree with pointer to segment table entries
753 * @guest_table_lock: spinlock to protect all entries in the guest page table
754 * @table: pointer to the page directory
755 * @asce: address space control element for gmap page table
756 * @pfault_enabled: defines if pfaults are applicable for the guest
757 */
758struct gmap {
759 struct list_head list;
760 struct list_head crst_list;
761 struct mm_struct *mm;
762 struct radix_tree_root guest_to_host;
763 struct radix_tree_root host_to_guest;
764 spinlock_t guest_table_lock;
765 unsigned long *table;
766 unsigned long asce;
767 unsigned long asce_end;
768 void *private;
769 bool pfault_enabled;
770};
771
772/**
773 * struct gmap_notifier - notify function block for page invalidation
774 * @notifier_call: address of callback function
775 */
776struct gmap_notifier {
777 struct list_head list;
778 void (*notifier_call)(struct gmap *gmap, unsigned long gaddr);
779};
780
781struct gmap *gmap_alloc(struct mm_struct *mm, unsigned long limit);
782void gmap_free(struct gmap *gmap);
783void gmap_enable(struct gmap *gmap);
784void gmap_disable(struct gmap *gmap);
785int gmap_map_segment(struct gmap *gmap, unsigned long from,
786 unsigned long to, unsigned long len);
787int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len);
788unsigned long __gmap_translate(struct gmap *, unsigned long gaddr);
789unsigned long gmap_translate(struct gmap *, unsigned long gaddr);
790int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr);
791int gmap_fault(struct gmap *, unsigned long gaddr, unsigned int fault_flags);
792void gmap_discard(struct gmap *, unsigned long from, unsigned long to);
793void __gmap_zap(struct gmap *, unsigned long gaddr);
794bool gmap_test_and_clear_dirty(unsigned long address, struct gmap *);
795
796
797void gmap_register_ipte_notifier(struct gmap_notifier *);
798void gmap_unregister_ipte_notifier(struct gmap_notifier *);
799int gmap_ipte_notify(struct gmap *, unsigned long start, unsigned long len);
800void gmap_do_ipte_notify(struct mm_struct *, unsigned long addr, pte_t *);
801
802static inline pgste_t pgste_ipte_notify(struct mm_struct *mm,
803 unsigned long addr,
804 pte_t *ptep, pgste_t pgste)
805{
806#ifdef CONFIG_PGSTE
807 if (pgste_val(pgste) & PGSTE_IN_BIT) {
808 pgste_val(pgste) &= ~PGSTE_IN_BIT;
809 gmap_do_ipte_notify(mm, addr, ptep);
810 }
811#endif
812 return pgste;
813}
814
815/*
816 * Certain architectures need to do special things when PTEs
817 * within a page table are directly modified. Thus, the following
818 * hook is made available.
819 */
820static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
821 pte_t *ptep, pte_t entry)
822{
823 pgste_t pgste;
824
825 if (mm_has_pgste(mm)) {
826 pgste = pgste_get_lock(ptep);
827 pgste_val(pgste) &= ~_PGSTE_GPS_ZERO;
828 pgste_set_key(ptep, pgste, entry, mm);
829 pgste = pgste_set_pte(ptep, pgste, entry);
830 pgste_set_unlock(ptep, pgste);
831 } else {
832 *ptep = entry;
833 }
834}
835
836/* 625/*
837 * query functions pte_write/pte_dirty/pte_young only work if 626 * query functions pte_write/pte_dirty/pte_young only work if
838 * pte_present() is true. Undefined behaviour if not.. 627 * pte_present() is true. Undefined behaviour if not..
@@ -998,96 +787,30 @@ static inline void __ptep_ipte_range(unsigned long address, int nr, pte_t *ptep)
998 } while (nr != 255); 787 } while (nr != 255);
999} 788}
1000 789
1001static inline void ptep_flush_direct(struct mm_struct *mm,
1002 unsigned long address, pte_t *ptep)
1003{
1004 int active, count;
1005
1006 if (pte_val(*ptep) & _PAGE_INVALID)
1007 return;
1008 active = (mm == current->active_mm) ? 1 : 0;
1009 count = atomic_add_return(0x10000, &mm->context.attach_count);
1010 if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
1011 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
1012 __ptep_ipte_local(address, ptep);
1013 else
1014 __ptep_ipte(address, ptep);
1015 atomic_sub(0x10000, &mm->context.attach_count);
1016}
1017
1018static inline void ptep_flush_lazy(struct mm_struct *mm,
1019 unsigned long address, pte_t *ptep)
1020{
1021 int active, count;
1022
1023 if (pte_val(*ptep) & _PAGE_INVALID)
1024 return;
1025 active = (mm == current->active_mm) ? 1 : 0;
1026 count = atomic_add_return(0x10000, &mm->context.attach_count);
1027 if ((count & 0xffff) <= active) {
1028 pte_val(*ptep) |= _PAGE_INVALID;
1029 mm->context.flush_mm = 1;
1030 } else
1031 __ptep_ipte(address, ptep);
1032 atomic_sub(0x10000, &mm->context.attach_count);
1033}
1034
1035/* 790/*
1036 * Get (and clear) the user dirty bit for a pte. 791 * This is hard to understand. ptep_get_and_clear and ptep_clear_flush
792 * both clear the TLB for the unmapped pte. The reason is that
793 * ptep_get_and_clear is used in common code (e.g. change_pte_range)
794 * to modify an active pte. The sequence is
795 * 1) ptep_get_and_clear
796 * 2) set_pte_at
797 * 3) flush_tlb_range
798 * On s390 the tlb needs to get flushed with the modification of the pte
799 * if the pte is active. The only way how this can be implemented is to
800 * have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range
801 * is a nop.
1037 */ 802 */
1038static inline int ptep_test_and_clear_user_dirty(struct mm_struct *mm, 803pte_t ptep_xchg_direct(struct mm_struct *, unsigned long, pte_t *, pte_t);
1039 unsigned long addr, 804pte_t ptep_xchg_lazy(struct mm_struct *, unsigned long, pte_t *, pte_t);
1040 pte_t *ptep)
1041{
1042 pgste_t pgste;
1043 pte_t pte;
1044 int dirty;
1045
1046 if (!mm_has_pgste(mm))
1047 return 0;
1048 pgste = pgste_get_lock(ptep);
1049 dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT);
1050 pgste_val(pgste) &= ~PGSTE_UC_BIT;
1051 pte = *ptep;
1052 if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
1053 pgste = pgste_ipte_notify(mm, addr, ptep, pgste);
1054 __ptep_ipte(addr, ptep);
1055 if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE))
1056 pte_val(pte) |= _PAGE_PROTECT;
1057 else
1058 pte_val(pte) |= _PAGE_INVALID;
1059 *ptep = pte;
1060 }
1061 pgste_set_unlock(ptep, pgste);
1062 return dirty;
1063}
1064 805
1065#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG 806#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1066static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, 807static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
1067 unsigned long addr, pte_t *ptep) 808 unsigned long addr, pte_t *ptep)
1068{ 809{
1069 pgste_t pgste; 810 pte_t pte = *ptep;
1070 pte_t pte, oldpte;
1071 int young;
1072
1073 if (mm_has_pgste(vma->vm_mm)) {
1074 pgste = pgste_get_lock(ptep);
1075 pgste = pgste_ipte_notify(vma->vm_mm, addr, ptep, pgste);
1076 }
1077
1078 oldpte = pte = *ptep;
1079 ptep_flush_direct(vma->vm_mm, addr, ptep);
1080 young = pte_young(pte);
1081 pte = pte_mkold(pte);
1082
1083 if (mm_has_pgste(vma->vm_mm)) {
1084 pgste = pgste_update_all(&oldpte, pgste, vma->vm_mm);
1085 pgste = pgste_set_pte(ptep, pgste, pte);
1086 pgste_set_unlock(ptep, pgste);
1087 } else
1088 *ptep = pte;
1089 811
1090 return young; 812 pte = ptep_xchg_direct(vma->vm_mm, addr, ptep, pte_mkold(pte));
813 return pte_young(pte);
1091} 814}
1092 815
1093#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH 816#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
@@ -1097,104 +820,22 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
1097 return ptep_test_and_clear_young(vma, address, ptep); 820 return ptep_test_and_clear_young(vma, address, ptep);
1098} 821}
1099 822
1100/*
1101 * This is hard to understand. ptep_get_and_clear and ptep_clear_flush
1102 * both clear the TLB for the unmapped pte. The reason is that
1103 * ptep_get_and_clear is used in common code (e.g. change_pte_range)
1104 * to modify an active pte. The sequence is
1105 * 1) ptep_get_and_clear
1106 * 2) set_pte_at
1107 * 3) flush_tlb_range
1108 * On s390 the tlb needs to get flushed with the modification of the pte
1109 * if the pte is active. The only way how this can be implemented is to
1110 * have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range
1111 * is a nop.
1112 */
1113#define __HAVE_ARCH_PTEP_GET_AND_CLEAR 823#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
1114static inline pte_t ptep_get_and_clear(struct mm_struct *mm, 824static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
1115 unsigned long address, pte_t *ptep) 825 unsigned long addr, pte_t *ptep)
1116{ 826{
1117 pgste_t pgste; 827 return ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
1118 pte_t pte;
1119
1120 if (mm_has_pgste(mm)) {
1121 pgste = pgste_get_lock(ptep);
1122 pgste = pgste_ipte_notify(mm, address, ptep, pgste);
1123 }
1124
1125 pte = *ptep;
1126 ptep_flush_lazy(mm, address, ptep);
1127 pte_val(*ptep) = _PAGE_INVALID;
1128
1129 if (mm_has_pgste(mm)) {
1130 pgste = pgste_update_all(&pte, pgste, mm);
1131 pgste_set_unlock(ptep, pgste);
1132 }
1133 return pte;
1134} 828}
1135 829
1136#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION 830#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
1137static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, 831pte_t ptep_modify_prot_start(struct mm_struct *, unsigned long, pte_t *);
1138 unsigned long address, 832void ptep_modify_prot_commit(struct mm_struct *, unsigned long, pte_t *, pte_t);
1139 pte_t *ptep)
1140{
1141 pgste_t pgste;
1142 pte_t pte;
1143
1144 if (mm_has_pgste(mm)) {
1145 pgste = pgste_get_lock(ptep);
1146 pgste_ipte_notify(mm, address, ptep, pgste);
1147 }
1148
1149 pte = *ptep;
1150 ptep_flush_lazy(mm, address, ptep);
1151
1152 if (mm_has_pgste(mm)) {
1153 pgste = pgste_update_all(&pte, pgste, mm);
1154 pgste_set(ptep, pgste);
1155 }
1156 return pte;
1157}
1158
1159static inline void ptep_modify_prot_commit(struct mm_struct *mm,
1160 unsigned long address,
1161 pte_t *ptep, pte_t pte)
1162{
1163 pgste_t pgste;
1164
1165 if (mm_has_pgste(mm)) {
1166 pgste = pgste_get(ptep);
1167 pgste_set_key(ptep, pgste, pte, mm);
1168 pgste = pgste_set_pte(ptep, pgste, pte);
1169 pgste_set_unlock(ptep, pgste);
1170 } else
1171 *ptep = pte;
1172}
1173 833
1174#define __HAVE_ARCH_PTEP_CLEAR_FLUSH 834#define __HAVE_ARCH_PTEP_CLEAR_FLUSH
1175static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, 835static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
1176 unsigned long address, pte_t *ptep) 836 unsigned long addr, pte_t *ptep)
1177{ 837{
1178 pgste_t pgste; 838 return ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID));
1179 pte_t pte;
1180
1181 if (mm_has_pgste(vma->vm_mm)) {
1182 pgste = pgste_get_lock(ptep);
1183 pgste = pgste_ipte_notify(vma->vm_mm, address, ptep, pgste);
1184 }
1185
1186 pte = *ptep;
1187 ptep_flush_direct(vma->vm_mm, address, ptep);
1188 pte_val(*ptep) = _PAGE_INVALID;
1189
1190 if (mm_has_pgste(vma->vm_mm)) {
1191 if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) ==
1192 _PGSTE_GPS_USAGE_UNUSED)
1193 pte_val(pte) |= _PAGE_UNUSED;
1194 pgste = pgste_update_all(&pte, pgste, vma->vm_mm);
1195 pgste_set_unlock(ptep, pgste);
1196 }
1197 return pte;
1198} 839}
1199 840
1200/* 841/*
@@ -1206,80 +847,66 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
1206 */ 847 */
1207#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL 848#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
1208static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, 849static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
1209 unsigned long address, 850 unsigned long addr,
1210 pte_t *ptep, int full) 851 pte_t *ptep, int full)
1211{ 852{
1212 pgste_t pgste; 853 if (full) {
1213 pte_t pte; 854 pte_t pte = *ptep;
1214 855 *ptep = __pte(_PAGE_INVALID);
1215 if (!full && mm_has_pgste(mm)) { 856 return pte;
1216 pgste = pgste_get_lock(ptep);
1217 pgste = pgste_ipte_notify(mm, address, ptep, pgste);
1218 }
1219
1220 pte = *ptep;
1221 if (!full)
1222 ptep_flush_lazy(mm, address, ptep);
1223 pte_val(*ptep) = _PAGE_INVALID;
1224
1225 if (!full && mm_has_pgste(mm)) {
1226 pgste = pgste_update_all(&pte, pgste, mm);
1227 pgste_set_unlock(ptep, pgste);
1228 } 857 }
1229 return pte; 858 return ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
1230} 859}
1231 860
1232#define __HAVE_ARCH_PTEP_SET_WRPROTECT 861#define __HAVE_ARCH_PTEP_SET_WRPROTECT
1233static inline pte_t ptep_set_wrprotect(struct mm_struct *mm, 862static inline void ptep_set_wrprotect(struct mm_struct *mm,
1234 unsigned long address, pte_t *ptep) 863 unsigned long addr, pte_t *ptep)
1235{ 864{
1236 pgste_t pgste;
1237 pte_t pte = *ptep; 865 pte_t pte = *ptep;
1238 866
1239 if (pte_write(pte)) { 867 if (pte_write(pte))
1240 if (mm_has_pgste(mm)) { 868 ptep_xchg_lazy(mm, addr, ptep, pte_wrprotect(pte));
1241 pgste = pgste_get_lock(ptep);
1242 pgste = pgste_ipte_notify(mm, address, ptep, pgste);
1243 }
1244
1245 ptep_flush_lazy(mm, address, ptep);
1246 pte = pte_wrprotect(pte);
1247
1248 if (mm_has_pgste(mm)) {
1249 pgste = pgste_set_pte(ptep, pgste, pte);
1250 pgste_set_unlock(ptep, pgste);
1251 } else
1252 *ptep = pte;
1253 }
1254 return pte;
1255} 869}
1256 870
1257#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS 871#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
1258static inline int ptep_set_access_flags(struct vm_area_struct *vma, 872static inline int ptep_set_access_flags(struct vm_area_struct *vma,
1259 unsigned long address, pte_t *ptep, 873 unsigned long addr, pte_t *ptep,
1260 pte_t entry, int dirty) 874 pte_t entry, int dirty)
1261{ 875{
1262 pgste_t pgste; 876 if (pte_same(*ptep, entry))
1263 pte_t oldpte;
1264
1265 oldpte = *ptep;
1266 if (pte_same(oldpte, entry))
1267 return 0; 877 return 0;
1268 if (mm_has_pgste(vma->vm_mm)) { 878 ptep_xchg_direct(vma->vm_mm, addr, ptep, entry);
1269 pgste = pgste_get_lock(ptep); 879 return 1;
1270 pgste = pgste_ipte_notify(vma->vm_mm, address, ptep, pgste); 880}
1271 }
1272 881
1273 ptep_flush_direct(vma->vm_mm, address, ptep); 882/*
883 * Additional functions to handle KVM guest page tables
884 */
885void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
886 pte_t *ptep, pte_t entry);
887void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
888void ptep_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
889void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
890 pte_t *ptep , int reset);
891void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
892
893bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long address);
894int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
895 unsigned char key, bool nq);
896unsigned char get_guest_storage_key(struct mm_struct *mm, unsigned long addr);
1274 897
1275 if (mm_has_pgste(vma->vm_mm)) { 898/*
1276 if (pte_val(oldpte) & _PAGE_INVALID) 899 * Certain architectures need to do special things when PTEs
1277 pgste_set_key(ptep, pgste, entry, vma->vm_mm); 900 * within a page table are directly modified. Thus, the following
1278 pgste = pgste_set_pte(ptep, pgste, entry); 901 * hook is made available.
1279 pgste_set_unlock(ptep, pgste); 902 */
1280 } else 903static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
904 pte_t *ptep, pte_t entry)
905{
906 if (mm_has_pgste(mm))
907 ptep_set_pte_at(mm, addr, ptep, entry);
908 else
1281 *ptep = entry; 909 *ptep = entry;
1282 return 1;
1283} 910}
1284 911
1285/* 912/*
@@ -1476,54 +1103,51 @@ static inline void __pmdp_idte_local(unsigned long address, pmd_t *pmdp)
1476 : "cc" ); 1103 : "cc" );
1477} 1104}
1478 1105
1479static inline void pmdp_flush_direct(struct mm_struct *mm, 1106pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
1480 unsigned long address, pmd_t *pmdp) 1107pmd_t pmdp_xchg_lazy(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
1481{
1482 int active, count;
1483 1108
1484 if (pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID) 1109#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1485 return;
1486 if (!MACHINE_HAS_IDTE) {
1487 __pmdp_csp(pmdp);
1488 return;
1489 }
1490 active = (mm == current->active_mm) ? 1 : 0;
1491 count = atomic_add_return(0x10000, &mm->context.attach_count);
1492 if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
1493 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
1494 __pmdp_idte_local(address, pmdp);
1495 else
1496 __pmdp_idte(address, pmdp);
1497 atomic_sub(0x10000, &mm->context.attach_count);
1498}
1499 1110
1500static inline void pmdp_flush_lazy(struct mm_struct *mm, 1111#define __HAVE_ARCH_PGTABLE_DEPOSIT
1501 unsigned long address, pmd_t *pmdp) 1112void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
1113 pgtable_t pgtable);
1114
1115#define __HAVE_ARCH_PGTABLE_WITHDRAW
1116pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
1117
1118#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
1119static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
1120 unsigned long addr, pmd_t *pmdp,
1121 pmd_t entry, int dirty)
1502{ 1122{
1503 int active, count; 1123 VM_BUG_ON(addr & ~HPAGE_MASK);
1504 1124
1505 if (pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID) 1125 entry = pmd_mkyoung(entry);
1506 return; 1126 if (dirty)
1507 active = (mm == current->active_mm) ? 1 : 0; 1127 entry = pmd_mkdirty(entry);
1508 count = atomic_add_return(0x10000, &mm->context.attach_count); 1128 if (pmd_val(*pmdp) == pmd_val(entry))
1509 if ((count & 0xffff) <= active) { 1129 return 0;
1510 pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID; 1130 pmdp_xchg_direct(vma->vm_mm, addr, pmdp, entry);
1511 mm->context.flush_mm = 1; 1131 return 1;
1512 } else if (MACHINE_HAS_IDTE)
1513 __pmdp_idte(address, pmdp);
1514 else
1515 __pmdp_csp(pmdp);
1516 atomic_sub(0x10000, &mm->context.attach_count);
1517} 1132}
1518 1133
1519#ifdef CONFIG_TRANSPARENT_HUGEPAGE 1134#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1135static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1136 unsigned long addr, pmd_t *pmdp)
1137{
1138 pmd_t pmd = *pmdp;
1520 1139
1521#define __HAVE_ARCH_PGTABLE_DEPOSIT 1140 pmd = pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd_mkold(pmd));
1522extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 1141 return pmd_young(pmd);
1523 pgtable_t pgtable); 1142}
1524 1143
1525#define __HAVE_ARCH_PGTABLE_WITHDRAW 1144#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
1526extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); 1145static inline int pmdp_clear_flush_young(struct vm_area_struct *vma,
1146 unsigned long addr, pmd_t *pmdp)
1147{
1148 VM_BUG_ON(addr & ~HPAGE_MASK);
1149 return pmdp_test_and_clear_young(vma, addr, pmdp);
1150}
1527 1151
1528static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, 1152static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1529 pmd_t *pmdp, pmd_t entry) 1153 pmd_t *pmdp, pmd_t entry)
@@ -1539,66 +1163,48 @@ static inline pmd_t pmd_mkhuge(pmd_t pmd)
1539 return pmd; 1163 return pmd;
1540} 1164}
1541 1165
1542#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1543static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1544 unsigned long address, pmd_t *pmdp)
1545{
1546 pmd_t pmd;
1547
1548 pmd = *pmdp;
1549 pmdp_flush_direct(vma->vm_mm, address, pmdp);
1550 *pmdp = pmd_mkold(pmd);
1551 return pmd_young(pmd);
1552}
1553
1554#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR 1166#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
1555static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, 1167static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
1556 unsigned long address, pmd_t *pmdp) 1168 unsigned long addr, pmd_t *pmdp)
1557{ 1169{
1558 pmd_t pmd = *pmdp; 1170 return pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID));
1559
1560 pmdp_flush_direct(mm, address, pmdp);
1561 pmd_clear(pmdp);
1562 return pmd;
1563} 1171}
1564 1172
1565#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL 1173#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
1566static inline pmd_t pmdp_huge_get_and_clear_full(struct mm_struct *mm, 1174static inline pmd_t pmdp_huge_get_and_clear_full(struct mm_struct *mm,
1567 unsigned long address, 1175 unsigned long addr,
1568 pmd_t *pmdp, int full) 1176 pmd_t *pmdp, int full)
1569{ 1177{
1570 pmd_t pmd = *pmdp; 1178 if (full) {
1571 1179 pmd_t pmd = *pmdp;
1572 if (!full) 1180 *pmdp = __pmd(_SEGMENT_ENTRY_INVALID);
1573 pmdp_flush_lazy(mm, address, pmdp); 1181 return pmd;
1574 pmd_clear(pmdp); 1182 }
1575 return pmd; 1183 return pmdp_xchg_lazy(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID));
1576} 1184}
1577 1185
1578#define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH 1186#define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
1579static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, 1187static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
1580 unsigned long address, pmd_t *pmdp) 1188 unsigned long addr, pmd_t *pmdp)
1581{ 1189{
1582 return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); 1190 return pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp);
1583} 1191}
1584 1192
1585#define __HAVE_ARCH_PMDP_INVALIDATE 1193#define __HAVE_ARCH_PMDP_INVALIDATE
1586static inline void pmdp_invalidate(struct vm_area_struct *vma, 1194static inline void pmdp_invalidate(struct vm_area_struct *vma,
1587 unsigned long address, pmd_t *pmdp) 1195 unsigned long addr, pmd_t *pmdp)
1588{ 1196{
1589 pmdp_flush_direct(vma->vm_mm, address, pmdp); 1197 pmdp_xchg_direct(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID));
1590} 1198}
1591 1199
1592#define __HAVE_ARCH_PMDP_SET_WRPROTECT 1200#define __HAVE_ARCH_PMDP_SET_WRPROTECT
1593static inline void pmdp_set_wrprotect(struct mm_struct *mm, 1201static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1594 unsigned long address, pmd_t *pmdp) 1202 unsigned long addr, pmd_t *pmdp)
1595{ 1203{
1596 pmd_t pmd = *pmdp; 1204 pmd_t pmd = *pmdp;
1597 1205
1598 if (pmd_write(pmd)) { 1206 if (pmd_write(pmd))
1599 pmdp_flush_direct(mm, address, pmdp); 1207 pmd = pmdp_xchg_lazy(mm, addr, pmdp, pmd_wrprotect(pmd));
1600 set_pmd_at(mm, address, pmdp, pmd_wrprotect(pmd));
1601 }
1602} 1208}
1603 1209
1604static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, 1210static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
generated by cgit v1.2.2 (git 2.25.0) at 2026-04-25 15:54:46 -0400