diff options
Diffstat (limited to 'arch/sh64/mm/hugetlbpage.c')
| -rw-r--r-- | arch/sh64/mm/hugetlbpage.c | 188 |
1 files changed, 12 insertions, 176 deletions
diff --git a/arch/sh64/mm/hugetlbpage.c b/arch/sh64/mm/hugetlbpage.c index dcd9c8a8baf8..ed6a505b3ee2 100644 --- a/arch/sh64/mm/hugetlbpage.c +++ b/arch/sh64/mm/hugetlbpage.c | |||
| @@ -54,41 +54,31 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) | |||
| 54 | return pte; | 54 | return pte; |
| 55 | } | 55 | } |
| 56 | 56 | ||
| 57 | #define mk_pte_huge(entry) do { pte_val(entry) |= _PAGE_SZHUGE; } while (0) | 57 | void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, |
| 58 | 58 | pte_t *ptep, pte_t entry) | |
| 59 | static void set_huge_pte(struct mm_struct *mm, struct vm_area_struct *vma, | ||
| 60 | struct page *page, pte_t * page_table, int write_access) | ||
| 61 | { | 59 | { |
| 62 | unsigned long i; | 60 | int i; |
| 63 | pte_t entry; | ||
| 64 | |||
| 65 | add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE); | ||
| 66 | |||
| 67 | if (write_access) | ||
| 68 | entry = pte_mkwrite(pte_mkdirty(mk_pte(page, | ||
| 69 | vma->vm_page_prot))); | ||
| 70 | else | ||
| 71 | entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot)); | ||
| 72 | entry = pte_mkyoung(entry); | ||
| 73 | mk_pte_huge(entry); | ||
| 74 | 61 | ||
| 75 | for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { | 62 | for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { |
| 76 | set_pte(page_table, entry); | 63 | set_pte_at(mm, addr, ptep, entry); |
| 77 | page_table++; | 64 | ptep++; |
| 78 | 65 | addr += PAGE_SIZE; | |
| 79 | pte_val(entry) += PAGE_SIZE; | 66 | pte_val(entry) += PAGE_SIZE; |
| 80 | } | 67 | } |
| 81 | } | 68 | } |
| 82 | 69 | ||
| 83 | pte_t huge_ptep_get_and_clear(pte_t *ptep) | 70 | pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, |
| 71 | pte_t *ptep) | ||
| 84 | { | 72 | { |
| 85 | pte_t entry; | 73 | pte_t entry; |
| 74 | int i; | ||
| 86 | 75 | ||
| 87 | entry = *ptep; | 76 | entry = *ptep; |
| 88 | 77 | ||
| 89 | for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { | 78 | for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { |
| 90 | pte_clear(pte); | 79 | pte_clear(mm, addr, ptep); |
| 91 | pte++; | 80 | addr += PAGE_SIZE; |
| 81 | ptep++; | ||
| 92 | } | 82 | } |
| 93 | 83 | ||
| 94 | return entry; | 84 | return entry; |
| @@ -106,79 +96,6 @@ int is_aligned_hugepage_range(unsigned long addr, unsigned long len) | |||
| 106 | return 0; | 96 | return 0; |
| 107 | } | 97 | } |
| 108 | 98 | ||
| 109 | int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, | ||
| 110 | struct vm_area_struct *vma) | ||
| 111 | { | ||
| 112 | pte_t *src_pte, *dst_pte, entry; | ||
| 113 | struct page *ptepage; | ||
| 114 | unsigned long addr = vma->vm_start; | ||
| 115 | unsigned long end = vma->vm_end; | ||
| 116 | int i; | ||
| 117 | |||
| 118 | while (addr < end) { | ||
| 119 | dst_pte = huge_pte_alloc(dst, addr); | ||
| 120 | if (!dst_pte) | ||
| 121 | goto nomem; | ||
| 122 | src_pte = huge_pte_offset(src, addr); | ||
| 123 | BUG_ON(!src_pte || pte_none(*src_pte)); | ||
| 124 | entry = *src_pte; | ||
| 125 | ptepage = pte_page(entry); | ||
| 126 | get_page(ptepage); | ||
| 127 | for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { | ||
| 128 | set_pte(dst_pte, entry); | ||
| 129 | pte_val(entry) += PAGE_SIZE; | ||
| 130 | dst_pte++; | ||
| 131 | } | ||
| 132 | add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE); | ||
| 133 | addr += HPAGE_SIZE; | ||
| 134 | } | ||
| 135 | return 0; | ||
| 136 | |||
| 137 | nomem: | ||
| 138 | return -ENOMEM; | ||
| 139 | } | ||
| 140 | |||
| 141 | int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, | ||
| 142 | struct page **pages, struct vm_area_struct **vmas, | ||
| 143 | unsigned long *position, int *length, int i) | ||
| 144 | { | ||
| 145 | unsigned long vaddr = *position; | ||
| 146 | int remainder = *length; | ||
| 147 | |||
| 148 | WARN_ON(!is_vm_hugetlb_page(vma)); | ||
| 149 | |||
| 150 | while (vaddr < vma->vm_end && remainder) { | ||
| 151 | if (pages) { | ||
| 152 | pte_t *pte; | ||
| 153 | struct page *page; | ||
| 154 | |||
| 155 | pte = huge_pte_offset(mm, vaddr); | ||
| 156 | |||
| 157 | /* hugetlb should be locked, and hence, prefaulted */ | ||
| 158 | BUG_ON(!pte || pte_none(*pte)); | ||
| 159 | |||
| 160 | page = pte_page(*pte); | ||
| 161 | |||
| 162 | WARN_ON(!PageCompound(page)); | ||
| 163 | |||
| 164 | get_page(page); | ||
| 165 | pages[i] = page; | ||
| 166 | } | ||
| 167 | |||
| 168 | if (vmas) | ||
| 169 | vmas[i] = vma; | ||
| 170 | |||
| 171 | vaddr += PAGE_SIZE; | ||
| 172 | --remainder; | ||
| 173 | ++i; | ||
| 174 | } | ||
| 175 | |||
| 176 | *length = remainder; | ||
| 177 | *position = vaddr; | ||
| 178 | |||
| 179 | return i; | ||
| 180 | } | ||
| 181 | |||
| 182 | struct page *follow_huge_addr(struct mm_struct *mm, | 99 | struct page *follow_huge_addr(struct mm_struct *mm, |
| 183 | unsigned long address, int write) | 100 | unsigned long address, int write) |
| 184 | { | 101 | { |
| @@ -195,84 +112,3 @@ struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, | |||
| 195 | { | 112 | { |
| 196 | return NULL; | 113 | return NULL; |
| 197 | } | 114 | } |
| 198 | |||
| 199 | void unmap_hugepage_range(struct vm_area_struct *vma, | ||
| 200 | unsigned long start, unsigned long end) | ||
| 201 | { | ||
| 202 | struct mm_struct *mm = vma->vm_mm; | ||
| 203 | unsigned long address; | ||
| 204 | pte_t *pte; | ||
| 205 | struct page *page; | ||
| 206 | int i; | ||
| 207 | |||
| 208 | BUG_ON(start & (HPAGE_SIZE - 1)); | ||
| 209 | BUG_ON(end & (HPAGE_SIZE - 1)); | ||
| 210 | |||
| 211 | for (address = start; address < end; address += HPAGE_SIZE) { | ||
| 212 | pte = huge_pte_offset(mm, address); | ||
| 213 | BUG_ON(!pte); | ||
| 214 | if (pte_none(*pte)) | ||
| 215 | continue; | ||
| 216 | page = pte_page(*pte); | ||
| 217 | put_page(page); | ||
| 218 | for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { | ||
| 219 | pte_clear(mm, address+(i*PAGE_SIZE), pte); | ||
| 220 | pte++; | ||
| 221 | } | ||
| 222 | } | ||
| 223 | add_mm_counter(mm, rss, -((end - start) >> PAGE_SHIFT)); | ||
| 224 | flush_tlb_range(vma, start, end); | ||
| 225 | } | ||
| 226 | |||
| 227 | int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma) | ||
| 228 | { | ||
| 229 | struct mm_struct *mm = current->mm; | ||
| 230 | unsigned long addr; | ||
| 231 | int ret = 0; | ||
| 232 | |||
| 233 | BUG_ON(vma->vm_start & ~HPAGE_MASK); | ||
| 234 | BUG_ON(vma->vm_end & ~HPAGE_MASK); | ||
| 235 | |||
| 236 | spin_lock(&mm->page_table_lock); | ||
| 237 | for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) { | ||
| 238 | unsigned long idx; | ||
| 239 | pte_t *pte = huge_pte_alloc(mm, addr); | ||
| 240 | struct page *page; | ||
| 241 | |||
| 242 | if (!pte) { | ||
| 243 | ret = -ENOMEM; | ||
| 244 | goto out; | ||
| 245 | } | ||
| 246 | if (!pte_none(*pte)) | ||
| 247 | continue; | ||
| 248 | |||
| 249 | idx = ((addr - vma->vm_start) >> HPAGE_SHIFT) | ||
| 250 | + (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT)); | ||
| 251 | page = find_get_page(mapping, idx); | ||
| 252 | if (!page) { | ||
| 253 | /* charge the fs quota first */ | ||
| 254 | if (hugetlb_get_quota(mapping)) { | ||
| 255 | ret = -ENOMEM; | ||
| 256 | goto out; | ||
| 257 | } | ||
| 258 | page = alloc_huge_page(); | ||
| 259 | if (!page) { | ||
| 260 | hugetlb_put_quota(mapping); | ||
| 261 | ret = -ENOMEM; | ||
| 262 | goto out; | ||
| 263 | } | ||
| 264 | ret = add_to_page_cache(page, mapping, idx, GFP_ATOMIC); | ||
| 265 | if (! ret) { | ||
| 266 | unlock_page(page); | ||
| 267 | } else { | ||
| 268 | hugetlb_put_quota(mapping); | ||
| 269 | free_huge_page(page); | ||
| 270 | goto out; | ||
| 271 | } | ||
| 272 | } | ||
| 273 | set_huge_pte(mm, vma, page, pte, vma->vm_flags & VM_WRITE); | ||
| 274 | } | ||
| 275 | out: | ||
| 276 | spin_unlock(&mm->page_table_lock); | ||
| 277 | return ret; | ||
| 278 | } | ||
