aboutsummaryrefslogtreecommitdiffstats
path: root/mm/mprotect.c
diff options
context:
space:
mode:
Diffstat (limited to 'mm/mprotect.c')
-rw-r--r--mm/mprotect.c76
1 files changed, 28 insertions, 48 deletions
diff --git a/mm/mprotect.c b/mm/mprotect.c
index 94722a4d6b43..26667971c824 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -37,14 +37,12 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
37 37
38static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 38static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
39 unsigned long addr, unsigned long end, pgprot_t newprot, 39 unsigned long addr, unsigned long end, pgprot_t newprot,
40 int dirty_accountable, int prot_numa, bool *ret_all_same_node) 40 int dirty_accountable, int prot_numa)
41{ 41{
42 struct mm_struct *mm = vma->vm_mm; 42 struct mm_struct *mm = vma->vm_mm;
43 pte_t *pte, oldpte; 43 pte_t *pte, oldpte;
44 spinlock_t *ptl; 44 spinlock_t *ptl;
45 unsigned long pages = 0; 45 unsigned long pages = 0;
46 bool all_same_node = true;
47 int last_nid = -1;
48 46
49 pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 47 pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
50 arch_enter_lazy_mmu_mode(); 48 arch_enter_lazy_mmu_mode();
@@ -63,15 +61,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
63 61
64 page = vm_normal_page(vma, addr, oldpte); 62 page = vm_normal_page(vma, addr, oldpte);
65 if (page) { 63 if (page) {
66 int this_nid = page_to_nid(page); 64 if (!pte_numa(oldpte)) {
67 if (last_nid == -1)
68 last_nid = this_nid;
69 if (last_nid != this_nid)
70 all_same_node = false;
71
72 /* only check non-shared pages */
73 if (!pte_numa(oldpte) &&
74 page_mapcount(page) == 1) {
75 ptent = pte_mknuma(ptent); 65 ptent = pte_mknuma(ptent);
76 updated = true; 66 updated = true;
77 } 67 }
@@ -94,40 +84,27 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
94 swp_entry_t entry = pte_to_swp_entry(oldpte); 84 swp_entry_t entry = pte_to_swp_entry(oldpte);
95 85
96 if (is_write_migration_entry(entry)) { 86 if (is_write_migration_entry(entry)) {
87 pte_t newpte;
97 /* 88 /*
98 * A protection check is difficult so 89 * A protection check is difficult so
99 * just be safe and disable write 90 * just be safe and disable write
100 */ 91 */
101 make_migration_entry_read(&entry); 92 make_migration_entry_read(&entry);
102 set_pte_at(mm, addr, pte, 93 newpte = swp_entry_to_pte(entry);
103 swp_entry_to_pte(entry)); 94 if (pte_swp_soft_dirty(oldpte))
95 newpte = pte_swp_mksoft_dirty(newpte);
96 set_pte_at(mm, addr, pte, newpte);
97
98 pages++;
104 } 99 }
105 pages++;
106 } 100 }
107 } while (pte++, addr += PAGE_SIZE, addr != end); 101 } while (pte++, addr += PAGE_SIZE, addr != end);
108 arch_leave_lazy_mmu_mode(); 102 arch_leave_lazy_mmu_mode();
109 pte_unmap_unlock(pte - 1, ptl); 103 pte_unmap_unlock(pte - 1, ptl);
110 104
111 *ret_all_same_node = all_same_node;
112 return pages; 105 return pages;
113} 106}
114 107
115#ifdef CONFIG_NUMA_BALANCING
116static inline void change_pmd_protnuma(struct mm_struct *mm, unsigned long addr,
117 pmd_t *pmd)
118{
119 spin_lock(&mm->page_table_lock);
120 set_pmd_at(mm, addr & PMD_MASK, pmd, pmd_mknuma(*pmd));
121 spin_unlock(&mm->page_table_lock);
122}
123#else
124static inline void change_pmd_protnuma(struct mm_struct *mm, unsigned long addr,
125 pmd_t *pmd)
126{
127 BUG();
128}
129#endif /* CONFIG_NUMA_BALANCING */
130
131static inline unsigned long change_pmd_range(struct vm_area_struct *vma, 108static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
132 pud_t *pud, unsigned long addr, unsigned long end, 109 pud_t *pud, unsigned long addr, unsigned long end,
133 pgprot_t newprot, int dirty_accountable, int prot_numa) 110 pgprot_t newprot, int dirty_accountable, int prot_numa)
@@ -135,36 +112,39 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
135 pmd_t *pmd; 112 pmd_t *pmd;
136 unsigned long next; 113 unsigned long next;
137 unsigned long pages = 0; 114 unsigned long pages = 0;
138 bool all_same_node; 115 unsigned long nr_huge_updates = 0;
139 116
140 pmd = pmd_offset(pud, addr); 117 pmd = pmd_offset(pud, addr);
141 do { 118 do {
119 unsigned long this_pages;
120
142 next = pmd_addr_end(addr, end); 121 next = pmd_addr_end(addr, end);
143 if (pmd_trans_huge(*pmd)) { 122 if (pmd_trans_huge(*pmd)) {
144 if (next - addr != HPAGE_PMD_SIZE) 123 if (next - addr != HPAGE_PMD_SIZE)
145 split_huge_page_pmd(vma, addr, pmd); 124 split_huge_page_pmd(vma, addr, pmd);
146 else if (change_huge_pmd(vma, pmd, addr, newprot, 125 else {
147 prot_numa)) { 126 int nr_ptes = change_huge_pmd(vma, pmd, addr,
148 pages += HPAGE_PMD_NR; 127 newprot, prot_numa);
149 continue; 128
129 if (nr_ptes) {
130 if (nr_ptes == HPAGE_PMD_NR) {
131 pages += HPAGE_PMD_NR;
132 nr_huge_updates++;
133 }
134 continue;
135 }
150 } 136 }
151 /* fall through */ 137 /* fall through */
152 } 138 }
153 if (pmd_none_or_clear_bad(pmd)) 139 if (pmd_none_or_clear_bad(pmd))
154 continue; 140 continue;
155 pages += change_pte_range(vma, pmd, addr, next, newprot, 141 this_pages = change_pte_range(vma, pmd, addr, next, newprot,
156 dirty_accountable, prot_numa, &all_same_node); 142 dirty_accountable, prot_numa);
157 143 pages += this_pages;
158 /*
159 * If we are changing protections for NUMA hinting faults then
160 * set pmd_numa if the examined pages were all on the same
161 * node. This allows a regular PMD to be handled as one fault
162 * and effectively batches the taking of the PTL
163 */
164 if (prot_numa && all_same_node)
165 change_pmd_protnuma(vma->vm_mm, addr, pmd);
166 } while (pmd++, addr = next, addr != end); 144 } while (pmd++, addr = next, addr != end);
167 145
146 if (nr_huge_updates)
147 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);
168 return pages; 148 return pages;
169} 149}
170 150
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-01 15:22:01 -0400