diff options
Diffstat (limited to 'mm/mremap.c')
-rw-r--r-- | mm/mremap.c | 241 |
1 files changed, 156 insertions, 85 deletions
diff --git a/mm/mremap.c b/mm/mremap.c index 97bff2547719..845190898d59 100644 --- a/mm/mremap.c +++ b/mm/mremap.c | |||
@@ -261,6 +261,137 @@ static unsigned long move_vma(struct vm_area_struct *vma, | |||
261 | return new_addr; | 261 | return new_addr; |
262 | } | 262 | } |
263 | 263 | ||
264 | static struct vm_area_struct *vma_to_resize(unsigned long addr, | ||
265 | unsigned long old_len, unsigned long new_len, unsigned long *p) | ||
266 | { | ||
267 | struct mm_struct *mm = current->mm; | ||
268 | struct vm_area_struct *vma = find_vma(mm, addr); | ||
269 | |||
270 | if (!vma || vma->vm_start > addr) | ||
271 | goto Efault; | ||
272 | |||
273 | if (is_vm_hugetlb_page(vma)) | ||
274 | goto Einval; | ||
275 | |||
276 | /* We can't remap across vm area boundaries */ | ||
277 | if (old_len > vma->vm_end - addr) | ||
278 | goto Efault; | ||
279 | |||
280 | if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) { | ||
281 | if (new_len > old_len) | ||
282 | goto Efault; | ||
283 | } | ||
284 | |||
285 | if (vma->vm_flags & VM_LOCKED) { | ||
286 | unsigned long locked, lock_limit; | ||
287 | locked = mm->locked_vm << PAGE_SHIFT; | ||
288 | lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur; | ||
289 | locked += new_len - old_len; | ||
290 | if (locked > lock_limit && !capable(CAP_IPC_LOCK)) | ||
291 | goto Eagain; | ||
292 | } | ||
293 | |||
294 | if (!may_expand_vm(mm, (new_len - old_len) >> PAGE_SHIFT)) | ||
295 | goto Enomem; | ||
296 | |||
297 | if (vma->vm_flags & VM_ACCOUNT) { | ||
298 | unsigned long charged = (new_len - old_len) >> PAGE_SHIFT; | ||
299 | if (security_vm_enough_memory(charged)) | ||
300 | goto Efault; | ||
301 | *p = charged; | ||
302 | } | ||
303 | |||
304 | return vma; | ||
305 | |||
306 | Efault: /* very odd choice for most of the cases, but... */ | ||
307 | return ERR_PTR(-EFAULT); | ||
308 | Einval: | ||
309 | return ERR_PTR(-EINVAL); | ||
310 | Enomem: | ||
311 | return ERR_PTR(-ENOMEM); | ||
312 | Eagain: | ||
313 | return ERR_PTR(-EAGAIN); | ||
314 | } | ||
315 | |||
316 | static unsigned long mremap_to(unsigned long addr, | ||
317 | unsigned long old_len, unsigned long new_addr, | ||
318 | unsigned long new_len) | ||
319 | { | ||
320 | struct mm_struct *mm = current->mm; | ||
321 | struct vm_area_struct *vma; | ||
322 | unsigned long ret = -EINVAL; | ||
323 | unsigned long charged = 0; | ||
324 | unsigned long map_flags; | ||
325 | |||
326 | if (new_addr & ~PAGE_MASK) | ||
327 | goto out; | ||
328 | |||
329 | if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len) | ||
330 | goto out; | ||
331 | |||
332 | /* Check if the location we're moving into overlaps the | ||
333 | * old location at all, and fail if it does. | ||
334 | */ | ||
335 | if ((new_addr <= addr) && (new_addr+new_len) > addr) | ||
336 | goto out; | ||
337 | |||
338 | if ((addr <= new_addr) && (addr+old_len) > new_addr) | ||
339 | goto out; | ||
340 | |||
341 | ret = security_file_mmap(NULL, 0, 0, 0, new_addr, 1); | ||
342 | if (ret) | ||
343 | goto out; | ||
344 | |||
345 | ret = do_munmap(mm, new_addr, new_len); | ||
346 | if (ret) | ||
347 | goto out; | ||
348 | |||
349 | if (old_len >= new_len) { | ||
350 | ret = do_munmap(mm, addr+new_len, old_len - new_len); | ||
351 | if (ret && old_len != new_len) | ||
352 | goto out; | ||
353 | old_len = new_len; | ||
354 | } | ||
355 | |||
356 | vma = vma_to_resize(addr, old_len, new_len, &charged); | ||
357 | if (IS_ERR(vma)) { | ||
358 | ret = PTR_ERR(vma); | ||
359 | goto out; | ||
360 | } | ||
361 | |||
362 | map_flags = MAP_FIXED; | ||
363 | if (vma->vm_flags & VM_MAYSHARE) | ||
364 | map_flags |= MAP_SHARED; | ||
365 | |||
366 | ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff + | ||
367 | ((addr - vma->vm_start) >> PAGE_SHIFT), | ||
368 | map_flags); | ||
369 | if (ret & ~PAGE_MASK) | ||
370 | goto out1; | ||
371 | |||
372 | ret = move_vma(vma, addr, old_len, new_len, new_addr); | ||
373 | if (!(ret & ~PAGE_MASK)) | ||
374 | goto out; | ||
375 | out1: | ||
376 | vm_unacct_memory(charged); | ||
377 | |||
378 | out: | ||
379 | return ret; | ||
380 | } | ||
381 | |||
382 | static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) | ||
383 | { | ||
384 | unsigned long end = vma->vm_end + delta; | ||
385 | if (end < vma->vm_end) /* overflow */ | ||
386 | return 0; | ||
387 | if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */ | ||
388 | return 0; | ||
389 | if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, | ||
390 | 0, MAP_FIXED) & ~PAGE_MASK) | ||
391 | return 0; | ||
392 | return 1; | ||
393 | } | ||
394 | |||
264 | /* | 395 | /* |
265 | * Expand (or shrink) an existing mapping, potentially moving it at the | 396 | * Expand (or shrink) an existing mapping, potentially moving it at the |
266 | * same time (controlled by the MREMAP_MAYMOVE flag and available VM space) | 397 | * same time (controlled by the MREMAP_MAYMOVE flag and available VM space) |
@@ -294,32 +425,10 @@ unsigned long do_mremap(unsigned long addr, | |||
294 | if (!new_len) | 425 | if (!new_len) |
295 | goto out; | 426 | goto out; |
296 | 427 | ||
297 | /* new_addr is only valid if MREMAP_FIXED is specified */ | ||
298 | if (flags & MREMAP_FIXED) { | 428 | if (flags & MREMAP_FIXED) { |
299 | if (new_addr & ~PAGE_MASK) | 429 | if (flags & MREMAP_MAYMOVE) |
300 | goto out; | 430 | ret = mremap_to(addr, old_len, new_addr, new_len); |
301 | if (!(flags & MREMAP_MAYMOVE)) | 431 | goto out; |
302 | goto out; | ||
303 | |||
304 | if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len) | ||
305 | goto out; | ||
306 | |||
307 | /* Check if the location we're moving into overlaps the | ||
308 | * old location at all, and fail if it does. | ||
309 | */ | ||
310 | if ((new_addr <= addr) && (new_addr+new_len) > addr) | ||
311 | goto out; | ||
312 | |||
313 | if ((addr <= new_addr) && (addr+old_len) > new_addr) | ||
314 | goto out; | ||
315 | |||
316 | ret = security_file_mmap(NULL, 0, 0, 0, new_addr, 1); | ||
317 | if (ret) | ||
318 | goto out; | ||
319 | |||
320 | ret = do_munmap(mm, new_addr, new_len); | ||
321 | if (ret) | ||
322 | goto out; | ||
323 | } | 432 | } |
324 | 433 | ||
325 | /* | 434 | /* |
@@ -332,60 +441,23 @@ unsigned long do_mremap(unsigned long addr, | |||
332 | if (ret && old_len != new_len) | 441 | if (ret && old_len != new_len) |
333 | goto out; | 442 | goto out; |
334 | ret = addr; | 443 | ret = addr; |
335 | if (!(flags & MREMAP_FIXED) || (new_addr == addr)) | 444 | goto out; |
336 | goto out; | ||
337 | old_len = new_len; | ||
338 | } | 445 | } |
339 | 446 | ||
340 | /* | 447 | /* |
341 | * Ok, we need to grow.. or relocate. | 448 | * Ok, we need to grow.. |
342 | */ | 449 | */ |
343 | ret = -EFAULT; | 450 | vma = vma_to_resize(addr, old_len, new_len, &charged); |
344 | vma = find_vma(mm, addr); | 451 | if (IS_ERR(vma)) { |
345 | if (!vma || vma->vm_start > addr) | 452 | ret = PTR_ERR(vma); |
346 | goto out; | ||
347 | if (is_vm_hugetlb_page(vma)) { | ||
348 | ret = -EINVAL; | ||
349 | goto out; | ||
350 | } | ||
351 | /* We can't remap across vm area boundaries */ | ||
352 | if (old_len > vma->vm_end - addr) | ||
353 | goto out; | ||
354 | if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) { | ||
355 | if (new_len > old_len) | ||
356 | goto out; | ||
357 | } | ||
358 | if (vma->vm_flags & VM_LOCKED) { | ||
359 | unsigned long locked, lock_limit; | ||
360 | locked = mm->locked_vm << PAGE_SHIFT; | ||
361 | lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur; | ||
362 | locked += new_len - old_len; | ||
363 | ret = -EAGAIN; | ||
364 | if (locked > lock_limit && !capable(CAP_IPC_LOCK)) | ||
365 | goto out; | ||
366 | } | ||
367 | if (!may_expand_vm(mm, (new_len - old_len) >> PAGE_SHIFT)) { | ||
368 | ret = -ENOMEM; | ||
369 | goto out; | 453 | goto out; |
370 | } | 454 | } |
371 | 455 | ||
372 | if (vma->vm_flags & VM_ACCOUNT) { | ||
373 | charged = (new_len - old_len) >> PAGE_SHIFT; | ||
374 | if (security_vm_enough_memory(charged)) | ||
375 | goto out_nc; | ||
376 | } | ||
377 | |||
378 | /* old_len exactly to the end of the area.. | 456 | /* old_len exactly to the end of the area.. |
379 | * And we're not relocating the area. | ||
380 | */ | 457 | */ |
381 | if (old_len == vma->vm_end - addr && | 458 | if (old_len == vma->vm_end - addr) { |
382 | !((flags & MREMAP_FIXED) && (addr != new_addr)) && | ||
383 | (old_len != new_len || !(flags & MREMAP_MAYMOVE))) { | ||
384 | unsigned long max_addr = TASK_SIZE; | ||
385 | if (vma->vm_next) | ||
386 | max_addr = vma->vm_next->vm_start; | ||
387 | /* can we just expand the current mapping? */ | 459 | /* can we just expand the current mapping? */ |
388 | if (max_addr - addr >= new_len) { | 460 | if (vma_expandable(vma, new_len - old_len)) { |
389 | int pages = (new_len - old_len) >> PAGE_SHIFT; | 461 | int pages = (new_len - old_len) >> PAGE_SHIFT; |
390 | 462 | ||
391 | vma_adjust(vma, vma->vm_start, | 463 | vma_adjust(vma, vma->vm_start, |
@@ -409,28 +481,27 @@ unsigned long do_mremap(unsigned long addr, | |||
409 | */ | 481 | */ |
410 | ret = -ENOMEM; | 482 | ret = -ENOMEM; |
411 | if (flags & MREMAP_MAYMOVE) { | 483 | if (flags & MREMAP_MAYMOVE) { |
412 | if (!(flags & MREMAP_FIXED)) { | 484 | unsigned long map_flags = 0; |
413 | unsigned long map_flags = 0; | 485 | if (vma->vm_flags & VM_MAYSHARE) |
414 | if (vma->vm_flags & VM_MAYSHARE) | 486 | map_flags |= MAP_SHARED; |
415 | map_flags |= MAP_SHARED; | 487 | |
416 | 488 | new_addr = get_unmapped_area(vma->vm_file, 0, new_len, | |
417 | new_addr = get_unmapped_area(vma->vm_file, 0, new_len, | 489 | vma->vm_pgoff + |
418 | vma->vm_pgoff, map_flags); | 490 | ((addr - vma->vm_start) >> PAGE_SHIFT), |
419 | if (new_addr & ~PAGE_MASK) { | 491 | map_flags); |
420 | ret = new_addr; | 492 | if (new_addr & ~PAGE_MASK) { |
421 | goto out; | 493 | ret = new_addr; |
422 | } | 494 | goto out; |
423 | |||
424 | ret = security_file_mmap(NULL, 0, 0, 0, new_addr, 1); | ||
425 | if (ret) | ||
426 | goto out; | ||
427 | } | 495 | } |
496 | |||
497 | ret = security_file_mmap(NULL, 0, 0, 0, new_addr, 1); | ||
498 | if (ret) | ||
499 | goto out; | ||
428 | ret = move_vma(vma, addr, old_len, new_len, new_addr); | 500 | ret = move_vma(vma, addr, old_len, new_len, new_addr); |
429 | } | 501 | } |
430 | out: | 502 | out: |
431 | if (ret & ~PAGE_MASK) | 503 | if (ret & ~PAGE_MASK) |
432 | vm_unacct_memory(charged); | 504 | vm_unacct_memory(charged); |
433 | out_nc: | ||
434 | return ret; | 505 | return ret; |
435 | } | 506 | } |
436 | 507 | ||